diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.45668233713901946,
+  "epoch": 0.4557868815760018,
   "eval_steps": 500,
-  "global_step": 510,
+  "global_step": 509,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -24,10 +24,10 @@
       "completions/min_terminated_length": 212.0,
       "epoch": 0.0008954555630176853,
       "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.2952850427924405,
-      "kl": 0.0005941390991210938,
+      "grad_norm": 1.324537021697667,
+      "kl": 0.0518798828125,
       "learning_rate": 0.0,
-      "loss": 0.016,
+      "loss": 0.0165,
       "num_tokens": 614764.0,
       "reward": 0.05976562947034836,
       "reward_std": 0.0456097275018692,
@@ -52,10 +52,10 @@
       "completions/min_terminated_length": 144.0,
       "epoch": 0.0017909111260353706,
       "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.1654835256642486,
-      "kl": 0.0006380081176757812,
+      "grad_norm": 1.1898987030436239,
+      "kl": 0.0516357421875,
       "learning_rate": 3.3333333333333335e-07,
-      "loss": 0.0213,
+      "loss": 0.0218,
       "num_tokens": 1251315.0,
       "reward": 0.06015624850988388,
       "reward_std": 0.04386558383703232,
@@ -73,24 +73,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1898.0,
-      "completions/mean_length": 633.169921875,
-      "completions/mean_terminated_length": 630.4011840820312,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_terminated_length": 1477.0,
+      "completions/mean_length": 619.98828125,
+      "completions/mean_terminated_length": 617.1937255859375,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.002686366689053056,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 1.1432983033631348,
-      "kl": 0.0006580352783203125,
+      "grad_norm": 1.161155142512,
+      "kl": 0.05609130859375,
       "learning_rate": 6.666666666666667e-07,
-      "loss": 0.0288,
-      "num_tokens": 1902698.0,
-      "reward": 0.05878906697034836,
-      "reward_std": 0.04603128135204315,
+      "loss": 0.0261,
+      "num_tokens": 1895949.0,
+      "reward": 0.06425781548023224,
+      "reward_std": 0.04574279859662056,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.587890625,
-      "rewards/format_reward/std": 0.49269601702690125,
+      "rewards/format_reward/mean": 0.642578125,
+      "rewards/format_reward/std": 0.4797092080116272,
       "step": 3
     },
     {
@@ -99,26 +99,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1800.0,
-      "completions/mean_length": 631.861328125,
-      "completions/mean_terminated_length": 629.0900268554688,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/max_terminated_length": 2034.0,
+      "completions/mean_length": 641.892578125,
+      "completions/mean_terminated_length": 632.3051147460938,
+      "completions/min_length": 244.0,
+      "completions/min_terminated_length": 244.0,
       "epoch": 0.003581822252070741,
-      "frac_reward_zero_std": 0.0625,
-      "grad_norm": 0.8408256552350213,
-      "kl": 0.0007734298706054688,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.062290227835526,
+      "kl": 0.15771484375,
       "learning_rate": 1.0000000000000002e-06,
-      "loss": 0.0369,
-      "num_tokens": 2551539.0,
-      "reward": 0.07343749701976776,
-      "reward_std": 0.0411534309387207,
+      "loss": 0.0292,
+      "num_tokens": 2549926.0,
+      "reward": 0.06679688394069672,
+      "reward_std": 0.04454650729894638,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.734375,
-      "rewards/format_reward/std": 0.44209739565849304,
+      "rewards/format_reward/mean": 0.66796875,
+      "rewards/format_reward/std": 0.47140273451805115,
       "step": 4
     },
     {
@@ -127,26 +127,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1767.0,
-      "completions/max_terminated_length": 1767.0,
-      "completions/mean_length": 579.943359375,
-      "completions/mean_terminated_length": 579.943359375,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1832.0,
+      "completions/mean_length": 602.048828125,
+      "completions/mean_terminated_length": 598.4530029296875,
+      "completions/min_length": 271.0,
+      "completions/min_terminated_length": 271.0,
       "epoch": 0.0044772778150884264,
-      "frac_reward_zero_std": 0.21875,
-      "grad_norm": 0.5439705548720983,
-      "kl": 0.0013751983642578125,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 1.0733703140487003,
+      "kl": 0.07025146484375,
       "learning_rate": 1.3333333333333334e-06,
-      "loss": 0.0314,
-      "num_tokens": 3130982.0,
-      "reward": 0.08925781399011612,
-      "reward_std": 0.026584036648273468,
+      "loss": 0.0339,
+      "num_tokens": 3140687.0,
+      "reward": 0.07949218899011612,
+      "reward_std": 0.03763638436794281,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.892578125,
-      "rewards/format_reward/std": 0.30995169281959534,
+      "rewards/format_reward/mean": 0.794921875,
+      "rewards/format_reward/std": 0.4041535556316376,
       "step": 5
     },
     {
@@ -155,26 +155,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1988.0,
-      "completions/mean_length": 599.826171875,
-      "completions/mean_terminated_length": 591.290771484375,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/max_terminated_length": 1348.0,
+      "completions/mean_length": 593.43359375,
+      "completions/mean_terminated_length": 590.5870971679688,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.005372733378106112,
-      "frac_reward_zero_std": 0.28125,
-      "grad_norm": 0.35924800717698446,
-      "kl": 0.001567840576171875,
+      "frac_reward_zero_std": 0.15625,
+      "grad_norm": 0.46792041518870797,
+      "kl": 0.05133056640625,
       "learning_rate": 1.6666666666666667e-06,
-      "loss": 0.03,
-      "num_tokens": 3753453.0,
-      "reward": 0.08671875298023224,
-      "reward_std": 0.02724354714155197,
+      "loss": 0.0476,
+      "num_tokens": 3759885.0,
+      "reward": 0.08847656846046448,
+      "reward_std": 0.028867140412330627,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.8671875,
-      "rewards/format_reward/std": 0.33970388770103455,
+      "rewards/format_reward/mean": 0.884765625,
+      "rewards/format_reward/std": 0.3196168541908264,
       "step": 6
     },
     {
@@ -185,24 +185,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1959.0,
-      "completions/mean_length": 629.521484375,
-      "completions/mean_terminated_length": 626.74560546875,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_terminated_length": 1760.0,
+      "completions/mean_length": 622.640625,
+      "completions/mean_terminated_length": 619.8512573242188,
+      "completions/min_length": 148.0,
+      "completions/min_terminated_length": 148.0,
       "epoch": 0.006268188941123797,
       "frac_reward_zero_std": 0.3125,
-      "grad_norm": 0.35624534475901054,
-      "kl": 0.00205230712890625,
+      "grad_norm": 0.3715829196297699,
+      "kl": 0.0545654296875,
       "learning_rate": 2.0000000000000003e-06,
-      "loss": 0.0283,
-      "num_tokens": 4472840.0,
-      "reward": 0.09335938096046448,
-      "reward_std": 0.0204335805028677,
+      "loss": 0.0305,
+      "num_tokens": 4475749.0,
+      "reward": 0.09160156548023224,
+      "reward_std": 0.022055473178625107,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.93359375,
-      "rewards/format_reward/std": 0.2492343932390213,
+      "rewards/format_reward/mean": 0.916015625,
+      "rewards/format_reward/std": 0.2776356339454651,
       "step": 7
     },
     {
@@ -212,25 +212,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1685.0,
-      "completions/max_terminated_length": 1685.0,
-      "completions/mean_length": 544.69921875,
-      "completions/mean_terminated_length": 544.69921875,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/max_length": 1935.0,
+      "completions/max_terminated_length": 1935.0,
+      "completions/mean_length": 581.716796875,
+      "completions/mean_terminated_length": 581.716796875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.007163644504141482,
-      "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.3450929543712783,
-      "kl": 0.00217437744140625,
+      "frac_reward_zero_std": 0.375,
+      "grad_norm": 0.31874462958935845,
+      "kl": 0.05224609375,
       "learning_rate": 2.3333333333333336e-06,
-      "loss": 0.0152,
-      "num_tokens": 5063230.0,
-      "reward": 0.09531249105930328,
-      "reward_std": 0.014789125882089138,
+      "loss": 0.0226,
+      "num_tokens": 5085092.0,
+      "reward": 0.09453125298023224,
+      "reward_std": 0.017671994864940643,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.953125,
-      "rewards/format_reward/std": 0.21157780289649963,
+      "rewards/format_reward/mean": 0.9453125,
+      "rewards/format_reward/std": 0.2275916188955307,
       "step": 8
     },
     {
@@ -239,26 +239,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1935.0,
-      "completions/mean_length": 532.423828125,
-      "completions/mean_terminated_length": 526.4804077148438,
-      "completions/min_length": 158.0,
-      "completions/min_terminated_length": 158.0,
+      "completions/max_terminated_length": 1669.0,
+      "completions/mean_length": 538.501953125,
+      "completions/mean_terminated_length": 535.5479125976562,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.008059100067159167,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.1839405208353318,
-      "kl": 0.00618743896484375,
+      "frac_reward_zero_std": 0.71875,
+      "grad_norm": 0.22707584816837714,
+      "kl": 0.05548095703125,
       "learning_rate": 2.666666666666667e-06,
-      "loss": 0.0253,
-      "num_tokens": 5628855.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0026298905722796917,
+      "loss": 0.0184,
+      "num_tokens": 5653829.0,
+      "reward": 0.09804687649011612,
+      "reward_std": 0.007317390292882919,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
       "step": 9
     },
     {
@@ -268,19 +268,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1702.0,
-      "completions/max_terminated_length": 1702.0,
-      "completions/mean_length": 487.248046875,
-      "completions/mean_terminated_length": 487.248046875,
-      "completions/min_length": 97.0,
-      "completions/min_terminated_length": 97.0,
+      "completions/max_length": 1623.0,
+      "completions/max_terminated_length": 1623.0,
+      "completions/mean_length": 508.15625,
+      "completions/mean_terminated_length": 508.15625,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
       "epoch": 0.008954555630176853,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.12236768536967425,
-      "kl": 0.0093231201171875,
+      "grad_norm": 0.10154622002931386,
+      "kl": 0.0634765625,
       "learning_rate": 3e-06,
-      "loss": -0.0057,
-      "num_tokens": 6163494.0,
+      "loss": 0.0023,
+      "num_tokens": 6199173.0,
       "reward": 0.099609375,
       "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
@@ -295,20 +295,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1580.0,
-      "completions/max_terminated_length": 1580.0,
-      "completions/mean_length": 545.07421875,
-      "completions/mean_terminated_length": 545.07421875,
-      "completions/min_length": 131.0,
-      "completions/min_terminated_length": 131.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1765.0,
+      "completions/mean_length": 553.5234375,
+      "completions/mean_terminated_length": 549.9627685546875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.009850011193194537,
       "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.12766105862016214,
-      "kl": 0.009033203125,
+      "grad_norm": 0.14394814844886103,
+      "kl": 0.06744384765625,
       "learning_rate": 3.3333333333333333e-06,
-      "loss": -0.0035,
-      "num_tokens": 6763004.0,
+      "loss": 0.0189,
+      "num_tokens": 6803009.0,
       "reward": 0.09941406548023224,
       "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
@@ -324,25 +324,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1583.0,
-      "completions/max_terminated_length": 1583.0,
-      "completions/mean_length": 458.548828125,
-      "completions/mean_terminated_length": 458.548828125,
-      "completions/min_length": 104.0,
-      "completions/min_terminated_length": 104.0,
+      "completions/max_length": 1550.0,
+      "completions/max_terminated_length": 1550.0,
+      "completions/mean_length": 454.251953125,
+      "completions/mean_terminated_length": 454.251953125,
+      "completions/min_length": 78.0,
+      "completions/min_terminated_length": 78.0,
       "epoch": 0.010745466756212223,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.2226030448008434,
-      "kl": 0.012603759765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.16220747501541533,
+      "kl": 0.0635986328125,
       "learning_rate": 3.6666666666666666e-06,
-      "loss": 0.0003,
-      "num_tokens": 7302997.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0039,
+      "num_tokens": 7340802.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 12
     },
     {
@@ -352,25 +352,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1771.0,
-      "completions/max_terminated_length": 1771.0,
-      "completions/mean_length": 478.6875,
-      "completions/mean_terminated_length": 478.6875,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_length": 1469.0,
+      "completions/max_terminated_length": 1469.0,
+      "completions/mean_length": 469.560546875,
+      "completions/mean_terminated_length": 469.560546875,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.011640922319229908,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10169433318921767,
-      "kl": 0.0145111083984375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.1336388368219293,
+      "kl": 0.0775146484375,
       "learning_rate": 4.000000000000001e-06,
-      "loss": -0.0032,
-      "num_tokens": 7862917.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0062,
+      "num_tokens": 7896049.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 13
     },
     {
@@ -379,26 +379,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.703125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1984.0,
-      "completions/mean_length": 593.982421875,
-      "completions/mean_terminated_length": 537.9451904296875,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/max_terminated_length": 1832.0,
+      "completions/mean_length": 472.15234375,
+      "completions/mean_terminated_length": 469.0684814453125,
+      "completions/min_length": 100.0,
+      "completions/min_terminated_length": 100.0,
       "epoch": 0.012536377882247594,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.12608218316972675,
-      "kl": 0.034210205078125,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 0.2405259644807092,
+      "kl": 0.097412109375,
       "learning_rate": 4.333333333333334e-06,
-      "loss": 0.0026,
-      "num_tokens": 8522044.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0011,
+      "num_tokens": 8492799.0,
+      "reward": 0.09882812947034836,
+      "reward_std": 0.004192390479147434,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 14
     },
     {
@@ -407,26 +407,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.5,
+      "completions/clipped_ratio": -6.875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2048.0,
-      "completions/mean_length": 627.28515625,
-      "completions/mean_terminated_length": 532.5708618164062,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
+      "completions/max_terminated_length": 1911.0,
+      "completions/mean_length": 452.572265625,
+      "completions/mean_terminated_length": 433.5972595214844,
+      "completions/min_length": 98.0,
+      "completions/min_terminated_length": 98.0,
       "epoch": 0.013431833445265278,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 15.775692571832229,
-      "kl": 0.079345703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.1753416604203705,
+      "kl": 0.171630859375,
       "learning_rate": 4.666666666666667e-06,
-      "loss": 0.0104,
-      "num_tokens": 9162638.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0017,
+      "num_tokens": 9043940.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 15
     },
     {
@@ -435,26 +435,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.546875,
+      "completions/clipped_ratio": -6.921875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2002.0,
-      "completions/mean_length": 607.8515625,
-      "completions/mean_terminated_length": 526.5134887695312,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
+      "completions/max_terminated_length": 1929.0,
+      "completions/mean_length": 476.982421875,
+      "completions/mean_terminated_length": 464.00592041015625,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.014327289008282965,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 18276.22528094759,
-      "kl": 717.5,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 1.0552577658632907,
+      "kl": 0.169189453125,
       "learning_rate": 5e-06,
-      "loss": 7.1764,
-      "num_tokens": 9798962.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0024,
+      "num_tokens": 9613259.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 16
     },
     {
@@ -463,20 +463,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.5625,
+      "completions/clipped_ratio": -6.90625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1943.0,
-      "completions/mean_length": 605.16015625,
-      "completions/mean_terminated_length": 522.7809448242188,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
+      "completions/max_terminated_length": 1642.0,
+      "completions/mean_length": 417.689453125,
+      "completions/mean_terminated_length": 406.4466552734375,
+      "completions/min_length": 57.0,
+      "completions/min_terminated_length": 57.0,
       "epoch": 0.015222744571300649,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 784.6537568296061,
-      "kl": 15.96484375,
+      "grad_norm": 0.26917824685344643,
+      "kl": 0.1390380859375,
       "learning_rate": 4.999952797253148e-06,
-      "loss": 0.1602,
-      "num_tokens": 10393940.0,
+      "loss": 0.0027,
+      "num_tokens": 10112252.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -491,26 +491,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.703125,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1984.0,
-      "completions/mean_length": 598.755859375,
-      "completions/mean_terminated_length": 543.7788696289062,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/max_terminated_length": 1843.0,
+      "completions/mean_length": 462.291015625,
+      "completions/mean_terminated_length": 452.94500732421875,
+      "completions/min_length": 110.0,
+      "completions/min_terminated_length": 110.0,
       "epoch": 0.016118200134318333,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 20.79955935344091,
-      "kl": 1.15966796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.2947535239705964,
+      "kl": 0.117431640625,
       "learning_rate": 4.9998111909931225e-06,
-      "loss": 0.0205,
-      "num_tokens": 11043431.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0012,
+      "num_tokens": 10691873.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 18
     },
     {
@@ -519,20 +519,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.40625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1975.0,
-      "completions/mean_length": 595.953125,
-      "completions/mean_terminated_length": 481.9282531738281,
-      "completions/min_length": 131.0,
-      "completions/min_terminated_length": 131.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 901.0,
+      "completions/max_terminated_length": 901.0,
+      "completions/mean_length": 376.89453125,
+      "completions/mean_terminated_length": 376.89453125,
+      "completions/min_length": 114.0,
+      "completions/min_terminated_length": 114.0,
       "epoch": 0.01701365569733602,
       "frac_reward_zero_std": 0.90625,
-      "grad_norm": 30.502487683490465,
-      "kl": 4.1640625,
+      "grad_norm": 0.21872537582484822,
+      "kl": 0.0916748046875,
       "learning_rate": 4.999575187161439e-06,
-      "loss": 0.0478,
-      "num_tokens": 11634959.0,
+      "loss": 0.0064,
+      "num_tokens": 11171243.0,
       "reward": 0.09941406548023224,
       "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
@@ -547,26 +547,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.078125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2045.0,
-      "completions/mean_length": 666.94140625,
-      "completions/mean_terminated_length": 487.06842041015625,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1383.0,
+      "completions/max_terminated_length": 1383.0,
+      "completions/mean_length": 386.00390625,
+      "completions/mean_terminated_length": 386.00390625,
+      "completions/min_length": 72.0,
+      "completions/min_terminated_length": 72.0,
       "epoch": 0.017909111260353706,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 9.128603733807386,
-      "kl": 1.1630859375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.13652379466986567,
+      "kl": 0.0869140625,
       "learning_rate": 4.9992447956603455e-06,
-      "loss": 0.0073,
-      "num_tokens": 12270049.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0022,
+      "num_tokens": 11662493.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 20
     },
     {
@@ -575,26 +575,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -5.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1939.0,
-      "completions/mean_length": 685.447265625,
-      "completions/mean_terminated_length": 487.8009033203125,
-      "completions/min_length": 102.0,
-      "completions/min_terminated_length": 102.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 893.0,
+      "completions/max_terminated_length": 893.0,
+      "completions/mean_length": 333.017578125,
+      "completions/mean_terminated_length": 333.017578125,
+      "completions/min_length": 70.0,
+      "completions/min_terminated_length": 70.0,
       "epoch": 0.01880456682337139,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 2.0053797855307804,
-      "kl": 0.22998046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.04417150040050226,
+      "kl": 0.095703125,
       "learning_rate": 4.998820030352409e-06,
-      "loss": -0.0081,
-      "num_tokens": 12901686.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.001,
+      "num_tokens": 12113686.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 21
     },
     {
@@ -603,26 +603,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.265625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2000.0,
-      "completions/mean_length": 641.787109375,
-      "completions/mean_terminated_length": 501.0129089355469,
-      "completions/min_length": 130.0,
-      "completions/min_terminated_length": 130.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1031.0,
+      "completions/max_terminated_length": 1031.0,
+      "completions/mean_length": 387.27734375,
+      "completions/mean_terminated_length": 387.27734375,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
       "epoch": 0.019700022386389075,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 12.479987888813367,
-      "kl": 0.74072265625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.19366315929923666,
+      "kl": 0.0830078125,
       "learning_rate": 4.998300909059929e-06,
-      "loss": 0.034,
-      "num_tokens": 13527305.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0031250000465661287,
+      "loss": 0.0035,
+      "num_tokens": 12608996.0,
+      "reward": 0.09902343899011612,
+      "reward_std": 0.0028222277760505676,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "step": 22
     },
     {
@@ -631,26 +631,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1882.0,
-      "completions/mean_length": 565.125,
-      "completions/mean_terminated_length": 445.1525573730469,
-      "completions/min_length": 79.0,
-      "completions/min_terminated_length": 79.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 900.0,
+      "completions/max_terminated_length": 900.0,
+      "completions/mean_length": 393.1796875,
+      "completions/mean_terminated_length": 393.1796875,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
       "epoch": 0.02059547794940676,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 6.963355920650139,
-      "kl": 0.6787109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08926508511842418,
+      "kl": 0.081298828125,
       "learning_rate": 4.997687453564198e-06,
-      "loss": 0.0195,
-      "num_tokens": 14087097.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0004,
+      "num_tokens": 13080752.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 23
     },
     {
@@ -659,26 +659,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.640625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2022.0,
-      "completions/mean_length": 528.884765625,
-      "completions/mean_terminated_length": 466.11041259765625,
-      "completions/min_length": 123.0,
-      "completions/min_terminated_length": 123.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1882.0,
+      "completions/max_terminated_length": 1882.0,
+      "completions/mean_length": 444.2734375,
+      "completions/mean_terminated_length": 444.2734375,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.021490933512424447,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.23487749119933354,
-      "kl": 0.29736328125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.12692069614371276,
+      "kl": 0.085693359375,
       "learning_rate": 4.9969796896045775e-06,
-      "loss": 0.003,
-      "num_tokens": 14711134.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0076,
+      "num_tokens": 13661468.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 24
     },
     {
@@ -687,26 +687,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1501.0,
-      "completions/mean_length": 411.025390625,
-      "completions/mean_terminated_length": 393.637451171875,
-      "completions/min_length": 123.0,
-      "completions/min_terminated_length": 123.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1259.0,
+      "completions/max_terminated_length": 1259.0,
+      "completions/mean_length": 439.455078125,
+      "completions/mean_terminated_length": 436.3902282714844,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.02238638907544213,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 7.283851009962948,
-      "kl": 0.1751708984375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 20.877169149767465,
+      "kl": 0.09375,
       "learning_rate": 4.996177646877426e-06,
-      "loss": 0.0117,
-      "num_tokens": 15189179.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0144,
+      "num_tokens": 14154069.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 25
     },
     {
@@ -715,20 +715,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.78125,
+      "completions/clipped_ratio": -6.90625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1775.0,
-      "completions/mean_length": 471.109375,
-      "completions/mean_terminated_length": 441.7851257324219,
-      "completions/min_length": 131.0,
-      "completions/min_terminated_length": 131.0,
+      "completions/max_terminated_length": 1895.0,
+      "completions/mean_length": 474.58984375,
+      "completions/mean_terminated_length": 462.3340148925781,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.023281844638459816,
       "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.595269313741645,
-      "kl": 0.2216796875,
+      "grad_norm": 92.43206042257317,
+      "kl": 7.2659912109375,
       "learning_rate": 4.995281359034851e-06,
-      "loss": 0.0083,
-      "num_tokens": 15734499.0,
+      "loss": 0.0913,
+      "num_tokens": 14701171.0,
       "reward": 0.09941406548023224,
       "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
@@ -743,26 +743,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.859375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 926.0,
-      "completions/mean_length": 366.5703125,
-      "completions/mean_terminated_length": 358.7117004394531,
-      "completions/min_length": 68.0,
-      "completions/min_terminated_length": 68.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1354.0,
+      "completions/max_terminated_length": 1354.0,
+      "completions/mean_length": 451.283203125,
+      "completions/mean_terminated_length": 449.9921569824219,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
       "epoch": 0.0241773002014775,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 3.211143343202539,
-      "kl": 0.07269287109375,
+      "grad_norm": 0.15669283865685832,
+      "kl": 0.0904541015625,
       "learning_rate": 4.994290863683296e-06,
-      "loss": 0.0146,
-      "num_tokens": 16192391.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0103,
+      "num_tokens": 15202436.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 27
     },
     {
@@ -771,26 +771,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1249.0,
-      "completions/max_terminated_length": 1249.0,
-      "completions/mean_length": 425.619140625,
-      "completions/mean_terminated_length": 424.0589599609375,
-      "completions/min_length": 74.0,
-      "completions/min_terminated_length": 74.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1410.0,
+      "completions/max_terminated_length": 1410.0,
+      "completions/mean_length": 478.384765625,
+      "completions/mean_terminated_length": 476.5616455078125,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.025072755764495188,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.041783599036276396,
-      "kl": 0.03924560546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07431876710651732,
+      "kl": 0.088623046875,
       "learning_rate": 4.99320620238196e-06,
-      "loss": 0.0004,
-      "num_tokens": 16695076.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0051,
+      "num_tokens": 15732137.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 28
     },
     {
@@ -799,20 +799,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1408.0,
-      "completions/mean_length": 464.380859375,
-      "completions/mean_terminated_length": 461.28179931640625,
-      "completions/min_length": 79.0,
-      "completions/min_terminated_length": 79.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 1981.0,
+      "completions/max_terminated_length": 1981.0,
+      "completions/mean_length": 495.1640625,
+      "completions/mean_terminated_length": 489.8369445800781,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.025968211327512872,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07454353504766084,
-      "kl": 0.02850341796875,
+      "grad_norm": 1.419703912441118,
+      "kl": 0.1038818359375,
       "learning_rate": 4.99202742064106e-06,
-      "loss": 0.0192,
-      "num_tokens": 17286791.0,
+      "loss": -0.0017,
+      "num_tokens": 16339613.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -827,26 +827,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1740.0,
-      "completions/max_terminated_length": 1740.0,
-      "completions/mean_length": 448.654296875,
-      "completions/mean_terminated_length": 448.654296875,
-      "completions/min_length": 63.0,
-      "completions/min_terminated_length": 63.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 1830.0,
+      "completions/max_terminated_length": 1830.0,
+      "completions/mean_length": 483.849609375,
+      "completions/mean_terminated_length": 479.77606201171875,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.026863666890530557,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.022840042186935416,
-      "kl": 0.025787353515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 20.526980124430374,
+      "kl": 0.110595703125,
       "learning_rate": 4.990754567919917e-06,
-      "loss": 0.0003,
-      "num_tokens": 17819094.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0162,
+      "num_tokens": 16889936.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 30
     },
     {
@@ -855,20 +855,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1376.0,
-      "completions/max_terminated_length": 1376.0,
-      "completions/mean_length": 435.240234375,
-      "completions/mean_terminated_length": 435.240234375,
-      "completions/min_length": 69.0,
-      "completions/min_terminated_length": 69.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1573.0,
+      "completions/max_terminated_length": 1573.0,
+      "completions/mean_length": 477.513671875,
+      "completions/mean_terminated_length": 475.369873046875,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.02775912245354824,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0201442073656093,
-      "kl": 0.023193359375,
+      "grad_norm": 0.3115792345317698,
+      "kl": 0.1025390625,
       "learning_rate": 4.989387697624881e-06,
-      "loss": 0.0002,
-      "num_tokens": 18344689.0,
+      "loss": 0.001,
+      "num_tokens": 17437175.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -883,20 +883,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1879.0,
-      "completions/mean_length": 529.171875,
-      "completions/mean_terminated_length": 524.1078491210938,
-      "completions/min_length": 130.0,
-      "completions/min_terminated_length": 130.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1254.0,
+      "completions/max_terminated_length": 1254.0,
+      "completions/mean_length": 497.666015625,
+      "completions/mean_terminated_length": 496.96673583984375,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.02865457801656593,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 11726.835967429213,
-      "kl": 544.0148620605469,
+      "grad_norm": 0.4880352589785924,
+      "kl": 0.1168212890625,
       "learning_rate": 4.987926867107095e-06,
-      "loss": 5.464,
-      "num_tokens": 18960393.0,
+      "loss": 0.0045,
+      "num_tokens": 18036748.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -911,26 +911,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1324.0,
-      "completions/max_terminated_length": 1324.0,
-      "completions/mean_length": 500.341796875,
-      "completions/mean_terminated_length": 497.57257080078125,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1841.0,
+      "completions/max_terminated_length": 1841.0,
+      "completions/mean_length": 492.521484375,
+      "completions/mean_terminated_length": 484.14959716796875,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.029550033579583614,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 4.4073495095910635,
-      "kl": 0.8916015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 5.555072095753837,
+      "kl": 0.425537109375,
       "learning_rate": 4.986372137660078e-06,
-      "loss": 0.0089,
-      "num_tokens": 19533640.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0105,
+      "num_tokens": 18605991.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 33
     },
     {
@@ -939,26 +939,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1168.0,
-      "completions/max_terminated_length": 1168.0,
-      "completions/mean_length": 430.51953125,
-      "completions/mean_terminated_length": 429.8884582519531,
-      "completions/min_length": 83.0,
-      "completions/min_terminated_length": 83.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 1231.0,
+      "completions/max_terminated_length": 1231.0,
+      "completions/mean_length": 470.818359375,
+      "completions/mean_terminated_length": 466.25048828125,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.030445489142601298,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09236567784912826,
-      "kl": 0.028533935546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.061669261382861086,
+      "kl": 0.1256103515625,
       "learning_rate": 4.984723574517165e-06,
-      "loss": -0.0062,
-      "num_tokens": 20032834.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0013,
+      "num_tokens": 19125818.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 34
     },
     {
@@ -967,26 +967,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1237.0,
-      "completions/max_terminated_length": 1237.0,
-      "completions/mean_length": 499.416015625,
-      "completions/mean_terminated_length": 498.78277587890625,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 1387.0,
+      "completions/max_terminated_length": 1387.0,
+      "completions/mean_length": 532.9296875,
+      "completions/mean_terminated_length": 525.10498046875,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.03134094470561898,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01651332440268518,
-      "kl": 0.02386474609375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 17.57107852556159,
+      "kl": 0.1639404296875,
       "learning_rate": 4.9829812468487655e-06,
-      "loss": 0.0002,
-      "num_tokens": 20608647.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0174,
+      "num_tokens": 19718790.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 35
     },
     {
@@ -996,21 +996,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.734375,
-      "completions/max_length": 1281.0,
-      "completions/max_terminated_length": 1281.0,
-      "completions/mean_length": 500.65234375,
-      "completions/mean_terminated_length": 480.89093017578125,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
+      "completions/max_length": 1184.0,
+      "completions/max_terminated_length": 1184.0,
+      "completions/mean_length": 516.59765625,
+      "completions/mean_terminated_length": 496.25860595703125,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.03223640026863667,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2529.782303345979,
-      "kl": 48.9375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 17.78488865540282,
+      "kl": 0.3154296875,
       "learning_rate": 4.981145227759457e-06,
-      "loss": 0.5007,
-      "num_tokens": 21182933.0,
-      "reward": 0.09726563096046448,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.0204,
+      "num_tokens": 20301240.0,
+      "reward": 0.09726562350988388,
+      "reward_std": 0.0020409778226166964,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
       "rewards/format_reward/mean": 0.97265625,
@@ -1023,26 +1023,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 1611.0,
-      "completions/max_terminated_length": 1611.0,
-      "completions/mean_length": 537.484375,
-      "completions/mean_terminated_length": 530.6324462890625,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/clipped_ratio": -6.84375,
+      "completions/max_length": 1759.0,
+      "completions/max_terminated_length": 1759.0,
+      "completions/mean_length": 550.166015625,
+      "completions/mean_terminated_length": 536.2271118164062,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.03313185583165435,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2.4928540218102038,
-      "kl": 0.14483642578125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 7806.320146811803,
+      "kl": 572.0,
       "learning_rate": 4.979215594284924e-06,
-      "loss": 0.0115,
-      "num_tokens": 21763277.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 5.7231,
+      "num_tokens": 20888077.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 37
     },
     {
@@ -1051,26 +1051,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
+      "completions/clipped_ratio": -6.890625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1866.0,
-      "completions/mean_length": 540.787109375,
-      "completions/mean_terminated_length": 532.1146240234375,
-      "completions/min_length": 148.0,
-      "completions/min_terminated_length": 148.0,
+      "completions/max_terminated_length": 1769.0,
+      "completions/mean_length": 570.6875,
+      "completions/mean_terminated_length": 558.4415893554688,
+      "completions/min_length": 100.0,
+      "completions/min_terminated_length": 100.0,
       "epoch": 0.03402731139467204,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 1.980183337733373,
-      "kl": 0.057220458984375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 29.0123039860226,
+      "kl": 2.9072265625,
       "learning_rate": 4.977192427388722e-06,
-      "loss": 0.0236,
-      "num_tokens": 22365280.0,
-      "reward": 0.09785156697034836,
-      "reward_std": 0.00316358613781631,
+      "loss": 0.051,
+      "num_tokens": 21505389.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.978515625,
-      "rewards/format_reward/std": 0.14513419568538666,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 38
     },
     {
@@ -1079,26 +1079,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1825.0,
-      "completions/max_terminated_length": 1825.0,
-      "completions/mean_length": 511.64453125,
-      "completions/mean_terminated_length": 511.64453125,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 1847.0,
+      "completions/max_terminated_length": 1847.0,
+      "completions/mean_length": 538.337890625,
+      "completions/mean_terminated_length": 531.9466552734375,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
       "epoch": 0.03492276695768973,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.018814791012791984,
-      "kl": 0.019622802734375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 1.830223160421892,
+      "kl": 0.1162109375,
       "learning_rate": 4.9750758119588824e-06,
-      "loss": 0.0002,
-      "num_tokens": 22928442.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0029,
+      "num_tokens": 22082218.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 39
     },
     {
@@ -1107,20 +1107,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1528.0,
-      "completions/mean_length": 538.15625,
-      "completions/mean_terminated_length": 533.9725952148438,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -6.84375,
+      "completions/max_length": 1363.0,
+      "completions/max_terminated_length": 1363.0,
+      "completions/mean_length": 560.26953125,
+      "completions/mean_terminated_length": 548.7888793945312,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.03581822252070741,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09930995271707406,
-      "kl": 0.02301025390625,
+      "grad_norm": 5.189694636533056,
+      "kl": 0.1761474609375,
       "learning_rate": 4.972865836804349e-06,
-      "loss": 0.0122,
-      "num_tokens": 23494282.0,
+      "loss": 0.006,
+      "num_tokens": 22659380.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1135,26 +1135,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1301.0,
-      "completions/mean_length": 460.962890625,
-      "completions/mean_terminated_length": 452.0592956542969,
-      "completions/min_length": 130.0,
-      "completions/min_terminated_length": 130.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1279.0,
+      "completions/max_terminated_length": 1279.0,
+      "completions/mean_length": 478.83203125,
+      "completions/mean_terminated_length": 475.21063232421875,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.036713678083725096,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 41.394496617699204,
-      "kl": 6.682464599609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08930606384058017,
+      "kl": 0.115966796875,
       "learning_rate": 4.970562594651254e-06,
-      "loss": 0.0667,
-      "num_tokens": 24003543.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0013,
+      "num_tokens": 23177790.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 41
     },
     {
@@ -1163,26 +1163,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1939.0,
-      "completions/mean_length": 475.712890625,
-      "completions/mean_terminated_length": 466.44598388671875,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 1525.0,
+      "completions/max_terminated_length": 1525.0,
+      "completions/mean_length": 544.818359375,
+      "completions/mean_terminated_length": 535.7534790039062,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.03760913364674278,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 24.391100211271684,
-      "kl": 1.68402099609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.12171028839751209,
+      "kl": 0.1319580078125,
       "learning_rate": 4.968166182139026e-06,
-      "loss": 0.0169,
-      "num_tokens": 24575076.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0031,
+      "num_tokens": 23784705.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 42
     },
     {
@@ -1191,26 +1191,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.796875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1772.0,
-      "completions/mean_length": 434.248046875,
-      "completions/mean_terminated_length": 429.7013854980469,
-      "completions/min_length": 79.0,
-      "completions/min_terminated_length": 79.0,
+      "completions/max_terminated_length": 1370.0,
+      "completions/mean_length": 490.720703125,
+      "completions/mean_terminated_length": 470.34869384765625,
+      "completions/min_length": 85.0,
+      "completions/min_terminated_length": 85.0,
       "epoch": 0.038504589209760465,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.5910587074093245,
-      "kl": 0.1524658203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 7.7636935355164045,
+      "kl": 0.246826171875,
       "learning_rate": 4.9656766998163306e-06,
-      "loss": 0.0015,
-      "num_tokens": 25061187.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0094,
+      "num_tokens": 24299730.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 43
     },
     {
@@ -1219,26 +1219,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
+      "completions/clipped_ratio": -6.921875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2036.0,
-      "completions/mean_length": 470.498046875,
-      "completions/mean_terminated_length": 464.6909484863281,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/max_terminated_length": 1637.0,
+      "completions/mean_length": 527.525390625,
+      "completions/mean_terminated_length": 520.1400756835938,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.03940004477277815,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.14956360476131622,
-      "kl": 0.06591796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 4.974293908259754,
+      "kl": 0.6517333984375,
       "learning_rate": 4.963094252136865e-06,
-      "loss": 0.0007,
-      "num_tokens": 25601474.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0236,
+      "num_tokens": 24869215.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 44
     },
     {
@@ -1247,26 +1247,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.78125,
+      "completions/clipped_ratio": -6.890625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1885.0,
-      "completions/mean_length": 506.697265625,
-      "completions/mean_terminated_length": 471.13250732421875,
-      "completions/min_length": 103.0,
-      "completions/min_terminated_length": 103.0,
+      "completions/max_terminated_length": 1675.0,
+      "completions/mean_length": 517.080078125,
+      "completions/mean_terminated_length": 506.8396301269531,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.040295500335795834,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 16.11234098312743,
-      "kl": 1.211181640625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 5.969358530129748,
+      "kl": 0.1383056640625,
       "learning_rate": 4.960418947454958e-06,
-      "loss": 0.0129,
-      "num_tokens": 26188823.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0292,
+      "num_tokens": 25461880.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 45
     },
     {
@@ -1275,26 +1275,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 1863.0,
-      "completions/max_terminated_length": 1863.0,
-      "completions/mean_length": 490.3203125,
-      "completions/mean_terminated_length": 479.1208190917969,
-      "completions/min_length": 102.0,
-      "completions/min_terminated_length": 102.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1767.0,
+      "completions/max_terminated_length": 1767.0,
+      "completions/mean_length": 547.33203125,
+      "completions/mean_terminated_length": 540.8327026367188,
+      "completions/min_length": 75.0,
+      "completions/min_terminated_length": 75.0,
       "epoch": 0.04119095589881352,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.9840541759000635,
-      "kl": 0.1519775390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 3.964515594688607,
+      "kl": 0.09521484375,
       "learning_rate": 4.957650898021038e-06,
-      "loss": 0.0015,
-      "num_tokens": 26738683.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0095,
+      "num_tokens": 26040930.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 46
     },
     {
@@ -1303,26 +1303,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1553.0,
-      "completions/mean_length": 475.044921875,
-      "completions/mean_terminated_length": 471.96673583984375,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 569.171875,
+      "completions/mean_terminated_length": 545.6984252929688,
+      "completions/min_length": 55.0,
+      "completions/min_terminated_length": 55.0,
       "epoch": 0.04208641146183121,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01990518206423815,
-      "kl": 0.032440185546875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 3.633363104209369,
+      "kl": 0.6241455078125,
       "learning_rate": 4.954790219976915e-06,
-      "loss": 0.0003,
-      "num_tokens": 27293650.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0428,
+      "num_tokens": 26644090.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0026298905722796917,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 47
     },
     {
@@ -1331,26 +1331,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.859375,
+      "completions/clipped_ratio": -6.671875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1737.0,
-      "completions/mean_length": 533.146484375,
-      "completions/mean_terminated_length": 514.7256469726562,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/max_terminated_length": 1566.0,
+      "completions/mean_length": 581.9375,
+      "completions/mean_terminated_length": 558.0386962890625,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.042981867024848894,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 7.314941473605262,
-      "kl": 0.32373046875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 1.3607360038408776,
+      "kl": 0.390625,
       "learning_rate": 4.95183703335091e-06,
-      "loss": 0.0205,
-      "num_tokens": 27921709.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0081,
+      "num_tokens": 27297130.0,
+      "reward": 0.0966796875,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.966796875,
+      "rewards/format_reward/std": 0.17934183776378632,
       "step": 48
     },
     {
@@ -1359,26 +1359,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1629.0,
-      "completions/mean_length": 502.583984375,
-      "completions/mean_terminated_length": 488.8458557128906,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
+      "completions/max_terminated_length": 1945.0,
+      "completions/mean_length": 546.5859375,
+      "completions/mean_terminated_length": 539.001953125,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
       "epoch": 0.04387732258786658,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 4.053172103770152,
-      "kl": 0.665435791015625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.9432769696523784,
+      "kl": 0.1865234375,
       "learning_rate": 4.948791462052819e-06,
-      "loss": 0.0282,
-      "num_tokens": 28457960.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0019,
+      "num_tokens": 27855910.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 49
     },
     {
@@ -1388,25 +1388,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1479.0,
-      "completions/max_terminated_length": 1479.0,
-      "completions/mean_length": 491.30078125,
-      "completions/mean_terminated_length": 487.6062927246094,
-      "completions/min_length": 113.0,
-      "completions/min_terminated_length": 113.0,
+      "completions/max_length": 1071.0,
+      "completions/max_terminated_length": 1071.0,
+      "completions/mean_length": 510.458984375,
+      "completions/mean_terminated_length": 506.47833251953125,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.04477277815088426,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.032153653495619945,
-      "kl": 0.06005859375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08737982579415644,
+      "kl": 0.091796875,
       "learning_rate": 4.945653633868716e-06,
-      "loss": 0.0006,
-      "num_tokens": 29029506.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0053,
+      "num_tokens": 28437265.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 50
     },
     {
@@ -1415,26 +1415,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 1245.0,
-      "completions/max_terminated_length": 1245.0,
-      "completions/mean_length": 523.353515625,
-      "completions/mean_terminated_length": 514.160400390625,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 1840.0,
+      "completions/max_terminated_length": 1840.0,
+      "completions/mean_length": 559.35546875,
+      "completions/mean_terminated_length": 553.75048828125,
+      "completions/min_length": 115.0,
+      "completions/min_terminated_length": 115.0,
       "epoch": 0.04566823371390195,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.04442617311439247,
-      "kl": 0.08599853515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06582492357236211,
+      "kl": 0.0966796875,
       "learning_rate": 4.942423680455584e-06,
-      "loss": 0.0009,
-      "num_tokens": 29606183.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0007,
+      "num_tokens": 29032375.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 51
     },
     {
@@ -1443,20 +1443,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1283.0,
-      "completions/max_terminated_length": 1283.0,
-      "completions/mean_length": 494.517578125,
-      "completions/mean_terminated_length": 493.626220703125,
-      "completions/min_length": 128.0,
-      "completions/min_terminated_length": 128.0,
+      "completions/clipped_ratio": -6.859375,
+      "completions/max_length": 1732.0,
+      "completions/max_terminated_length": 1732.0,
+      "completions/mean_length": 529.138671875,
+      "completions/mean_terminated_length": 513.461181640625,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.04656368927691963,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.057162846752272095,
-      "kl": 0.036529541015625,
+      "grad_norm": 0.24454712354121277,
+      "kl": 0.193115234375,
       "learning_rate": 4.939101737335802e-06,
-      "loss": -0.0034,
-      "num_tokens": 30148768.0,
+      "loss": 0.012,
+      "num_tokens": 29592686.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1471,20 +1471,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1307.0,
-      "completions/max_terminated_length": 1307.0,
-      "completions/mean_length": 488.09765625,
-      "completions/mean_terminated_length": 487.25439453125,
-      "completions/min_length": 104.0,
-      "completions/min_terminated_length": 104.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1830.0,
+      "completions/mean_length": 514.66796875,
+      "completions/mean_terminated_length": 507.16534423828125,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
       "epoch": 0.047459144839937316,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05128260945429217,
-      "kl": 0.037628173828125,
+      "grad_norm": 0.07209160390847483,
+      "kl": 0.0982666015625,
       "learning_rate": 4.935687943891447e-06,
-      "loss": -0.0017,
-      "num_tokens": 30695730.0,
+      "loss": 0.0171,
+      "num_tokens": 30153252.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1499,20 +1499,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2043.0,
-      "completions/max_terminated_length": 2043.0,
-      "completions/mean_length": 547.087890625,
-      "completions/mean_terminated_length": 547.087890625,
-      "completions/min_length": 116.0,
-      "completions/min_terminated_length": 116.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 1958.0,
+      "completions/max_terminated_length": 1958.0,
+      "completions/mean_length": 556.134765625,
+      "completions/mean_terminated_length": 545.2826538085938,
+      "completions/min_length": 74.0,
+      "completions/min_terminated_length": 74.0,
       "epoch": 0.048354600402955,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06726520989605488,
-      "kl": 0.023834228515625,
+      "grad_norm": 1.2182649822769658,
+      "kl": 0.1866455078125,
       "learning_rate": 4.932182443358458e-06,
-      "loss": -0.001,
-      "num_tokens": 31311647.0,
+      "loss": 0.013,
+      "num_tokens": 30773801.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1528,19 +1528,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 1871.0,
-      "completions/max_terminated_length": 1871.0,
-      "completions/mean_length": 508.474609375,
-      "completions/mean_terminated_length": 499.8717956542969,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_length": 1496.0,
+      "completions/max_terminated_length": 1496.0,
+      "completions/mean_length": 520.10546875,
+      "completions/mean_terminated_length": 513.6646728515625,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.04925005596597269,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.031009644101716777,
-      "kl": 0.079254150390625,
+      "grad_norm": 0.06929038708329119,
+      "kl": 0.1387939453125,
       "learning_rate": 4.928585382820616e-06,
-      "loss": 0.0008,
-      "num_tokens": 31831858.0,
+      "loss": 0.0014,
+      "num_tokens": 31299967.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1555,26 +1555,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 1503.0,
-      "completions/max_terminated_length": 1503.0,
-      "completions/mean_length": 577.771484375,
-      "completions/mean_terminated_length": 570.49609375,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1904.0,
+      "completions/max_terminated_length": 1904.0,
+      "completions/mean_length": 542.359375,
+      "completions/mean_terminated_length": 540.060791015625,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.050145511528990376,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 0.17520768141349025,
-      "kl": 0.1038818359375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.15563688859048186,
+      "kl": 0.104736328125,
       "learning_rate": 4.924896913203376e-06,
-      "loss": -0.001,
-      "num_tokens": 32438557.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0031250000465661287,
+      "loss": -0.0022,
+      "num_tokens": 31888535.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 56
     },
     {
@@ -1583,26 +1583,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2004.0,
-      "completions/mean_length": 564.052734375,
-      "completions/mean_terminated_length": 560.36474609375,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1506.0,
+      "completions/max_terminated_length": 1506.0,
+      "completions/mean_length": 516.908203125,
+      "completions/mean_terminated_length": 515.5968627929688,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.05104096709200806,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10349796740366579,
-      "kl": 0.03631591796875,
+      "grad_norm": 0.09565916384806829,
+      "kl": 0.07861328125,
       "learning_rate": 4.921117189267535e-06,
-      "loss": 0.0141,
-      "num_tokens": 33035768.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0044,
+      "num_tokens": 32461608.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 57
     },
     {
@@ -1611,26 +1611,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1589.0,
-      "completions/max_terminated_length": 1589.0,
-      "completions/mean_length": 542.61328125,
-      "completions/mean_terminated_length": 542.61328125,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1916.0,
+      "completions/max_terminated_length": 1916.0,
+      "completions/mean_length": 525.482421875,
+      "completions/mean_terminated_length": 521.7431640625,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.051936422655025745,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08585433577443426,
-      "kl": 0.021392822265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0748699769594785,
+      "kl": 0.1156005859375,
       "learning_rate": 4.917246369602742e-06,
-      "loss": -0.0013,
-      "num_tokens": 33627778.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0002,
+      "num_tokens": 33044847.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 58
     },
     {
@@ -1639,20 +1639,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1705.0,
-      "completions/max_terminated_length": 1705.0,
-      "completions/mean_length": 545.0625,
-      "completions/mean_terminated_length": 539.7598266601562,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2001.0,
+      "completions/max_terminated_length": 2001.0,
+      "completions/mean_length": 514.23046875,
+      "completions/mean_terminated_length": 513.3111572265625,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.05283187821804343,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06357442394169845,
-      "kl": 0.1036376953125,
+      "grad_norm": 0.03709928067002307,
+      "kl": 0.079833984375,
       "learning_rate": 4.9132846166208355e-06,
-      "loss": -0.0002,
-      "num_tokens": 34219218.0,
+      "loss": -0.0006,
+      "num_tokens": 33620501.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1667,26 +1667,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1646.0,
-      "completions/max_terminated_length": 1646.0,
-      "completions/mean_length": 561.75,
-      "completions/mean_terminated_length": 558.19091796875,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1458.0,
+      "completions/mean_length": 511.310546875,
+      "completions/mean_terminated_length": 505.2843322753906,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
       "epoch": 0.053727333781061114,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09726885501749043,
-      "kl": 0.11737060546875,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.13840494930716188,
+      "kl": 0.06787109375,
       "learning_rate": 4.9092320965490365e-06,
-      "loss": 0.0006,
-      "num_tokens": 34830658.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0211,
+      "num_tokens": 34206116.0,
+      "reward": 0.09902343899011612,
+      "reward_std": 0.003411140525713563,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "step": 60
     },
     {
@@ -1695,26 +1695,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2037.0,
-      "completions/max_terminated_length": 2037.0,
-      "completions/mean_length": 563.16015625,
-      "completions/mean_terminated_length": 557.4133911132812,
-      "completions/min_length": 108.0,
-      "completions/min_terminated_length": 108.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1895.0,
+      "completions/max_terminated_length": 1895.0,
+      "completions/mean_length": 518.853515625,
+      "completions/mean_terminated_length": 518.853515625,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.0546227893440788,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 7.603189195829842,
-      "kl": 0.100677490234375,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.14872642942212774,
+      "kl": 0.0692138671875,
       "learning_rate": 4.905088979422971e-06,
       "loss": 0.0128,
-      "num_tokens": 35441012.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "num_tokens": 34793785.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 61
     },
     {
@@ -1723,26 +1723,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1732.0,
-      "completions/max_terminated_length": 1732.0,
-      "completions/mean_length": 518.525390625,
-      "completions/mean_terminated_length": 518.525390625,
-      "completions/min_length": 119.0,
-      "completions/min_terminated_length": 119.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1878.0,
+      "completions/max_terminated_length": 1878.0,
+      "completions/mean_length": 474.60546875,
+      "completions/mean_terminated_length": 473.1917724609375,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.05551824490709648,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10269453437417964,
-      "kl": 0.021270751953125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.12988434784735217,
+      "kl": 0.0841064453125,
       "learning_rate": 4.900855439079536e-06,
-      "loss": 0.0029,
-      "num_tokens": 36009649.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": -0.0012,
+      "num_tokens": 35339935.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 62
     },
     {
@@ -1751,20 +1751,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1435.0,
-      "completions/max_terminated_length": 1272.0,
-      "completions/mean_length": 542.779296875,
-      "completions/mean_terminated_length": 540.3529663085938,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1579.0,
+      "completions/max_terminated_length": 1579.0,
+      "completions/mean_length": 504.658203125,
+      "completions/mean_terminated_length": 504.658203125,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.056413700470114174,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07525577027012759,
-      "kl": 0.019866943359375,
+      "grad_norm": 0.07899678142978885,
+      "kl": 0.06787109375,
       "learning_rate": 4.8965316531496055e-06,
-      "loss": 0.0009,
-      "num_tokens": 36600624.0,
+      "loss": -0.0002,
+      "num_tokens": 35911392.0,
       "reward": 0.099609375,
       "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
@@ -1780,25 +1780,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1666.0,
-      "completions/max_terminated_length": 1666.0,
-      "completions/mean_length": 603.5859375,
-      "completions/mean_terminated_length": 599.5882568359375,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1658.0,
+      "completions/mean_length": 541.5859375,
+      "completions/mean_terminated_length": 535.678466796875,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.05730915603313186,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.7608650680571825,
-      "kl": 0.17962646484375,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 0.1638638882495904,
+      "kl": 0.06365966796875,
       "learning_rate": 4.892117803050578e-06,
-      "loss": 0.0018,
-      "num_tokens": 37192524.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0252,
+      "num_tokens": 36471548.0,
+      "reward": 0.09882812947034836,
+      "reward_std": 0.004192390479147434,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 64
     },
     {
@@ -1807,26 +1807,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1343.0,
-      "completions/mean_length": 553.01953125,
-      "completions/mean_terminated_length": 550.0939331054688,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_terminated_length": 1190.0,
+      "completions/mean_length": 509.482421875,
+      "completions/mean_terminated_length": 505.13140869140625,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
       "epoch": 0.05820461159614954,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04778415655998196,
-      "kl": 0.0208740234375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.12491007191362062,
+      "kl": 0.0806884765625,
       "learning_rate": 4.887614073978761e-06,
-      "loss": 0.0126,
-      "num_tokens": 37774438.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0189,
+      "num_tokens": 37031171.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0026298905722796917,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 65
     },
     {
@@ -1835,26 +1835,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1554.0,
-      "completions/mean_length": 564.185546875,
-      "completions/mean_terminated_length": 561.2817993164062,
-      "completions/min_length": 118.0,
-      "completions/min_terminated_length": 118.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1841.0,
+      "completions/max_terminated_length": 1841.0,
+      "completions/mean_length": 521.2890625,
+      "completions/mean_terminated_length": 521.2890625,
+      "completions/min_length": 98.0,
+      "completions/min_terminated_length": 98.0,
       "epoch": 0.05910006715916723,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09161873092433927,
-      "kl": 0.020843505859375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08381886765190119,
+      "kl": 0.06219482421875,
       "learning_rate": 4.883020654901609e-06,
-      "loss": 0.0103,
-      "num_tokens": 38392933.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0019,
+      "num_tokens": 37627703.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 66
     },
     {
@@ -1864,19 +1864,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1349.0,
-      "completions/max_terminated_length": 1349.0,
-      "completions/mean_length": 525.9296875,
-      "completions/mean_terminated_length": 525.9296875,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_length": 1381.0,
+      "completions/max_terminated_length": 1381.0,
+      "completions/mean_length": 477.830078125,
+      "completions/mean_terminated_length": 477.830078125,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.05999552272218491,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012620561722483294,
-      "kl": 0.022552490234375,
+      "grad_norm": 0.013399015519228851,
+      "kl": 0.0623779296875,
       "learning_rate": 4.878337738549785e-06,
-      "loss": 0.0002,
-      "num_tokens": 38968513.0,
+      "loss": 0.0006,
+      "num_tokens": 38178656.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1892,25 +1892,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1809.0,
-      "completions/mean_length": 588.72265625,
-      "completions/mean_terminated_length": 585.866943359375,
-      "completions/min_length": 218.0,
-      "completions/min_terminated_length": 218.0,
+      "completions/max_length": 1351.0,
+      "completions/max_terminated_length": 1351.0,
+      "completions/mean_length": 559.1171875,
+      "completions/mean_terminated_length": 558.129150390625,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.060890978285202596,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 0.1361201287941182,
-      "kl": 0.018341064453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.02053915419301268,
+      "kl": 0.0797119140625,
       "learning_rate": 4.873565521409082e-06,
-      "loss": 0.0149,
-      "num_tokens": 39585059.0,
-      "reward": 0.09921874850988388,
-      "reward_std": 0.0031250000465661287,
+      "loss": 0.0008,
+      "num_tokens": 38780044.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 68
     },
     {
@@ -1919,20 +1919,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1505.0,
-      "completions/max_terminated_length": 1505.0,
-      "completions/mean_length": 562.810546875,
-      "completions/mean_terminated_length": 561.7357788085938,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1736.0,
+      "completions/max_terminated_length": 1736.0,
+      "completions/mean_length": 528.73046875,
+      "completions/mean_terminated_length": 528.73046875,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.06178643384822028,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.019277916524184716,
-      "kl": 0.035003662109375,
+      "grad_norm": 0.013023668854051146,
+      "kl": 0.06231689453125,
       "learning_rate": 4.868704203712173e-06,
-      "loss": 0.0003,
-      "num_tokens": 40165250.0,
+      "loss": 0.0006,
+      "num_tokens": 39342786.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1947,26 +1947,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1305.0,
-      "completions/max_terminated_length": 1305.0,
-      "completions/mean_length": 560.72265625,
-      "completions/mean_terminated_length": 560.72265625,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1704.0,
+      "completions/mean_length": 539.8671875,
+      "completions/mean_terminated_length": 529.748046875,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.06268188941123796,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009928765161792547,
-      "kl": 0.017822265625,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 7.439993648948339,
+      "kl": 0.08404541015625,
       "learning_rate": 4.86375398943021e-06,
-      "loss": 0.0002,
-      "num_tokens": 40755572.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0415,
+      "num_tokens": 39922430.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 70
     },
     {
@@ -1976,25 +1976,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1758.0,
-      "completions/max_terminated_length": 1758.0,
-      "completions/mean_length": 550.47265625,
-      "completions/mean_terminated_length": 550.47265625,
-      "completions/min_length": 221.0,
-      "completions/min_terminated_length": 221.0,
+      "completions/max_length": 1456.0,
+      "completions/max_terminated_length": 1456.0,
+      "completions/mean_length": 529.509765625,
+      "completions/mean_terminated_length": 529.509765625,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.06357734497425566,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08245008848029971,
-      "kl": 0.017364501953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.012082899535339748,
+      "kl": 0.06158447265625,
       "learning_rate": 4.858715086264274e-06,
-      "loss": 0.0008,
-      "num_tokens": 41345766.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0006,
+      "num_tokens": 40501891.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 71
     },
     {
@@ -2004,25 +2004,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1483.0,
-      "completions/max_terminated_length": 1483.0,
-      "completions/mean_length": 575.744140625,
-      "completions/mean_terminated_length": 575.744140625,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/max_length": 1235.0,
+      "completions/max_terminated_length": 1235.0,
+      "completions/mean_length": 563.224609375,
+      "completions/mean_terminated_length": 563.224609375,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.06447280053727333,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009114395477044235,
-      "kl": 0.01678466796875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.10284201346946772,
+      "kl": 0.05999755859375,
       "learning_rate": 4.853587705636646e-06,
-      "loss": 0.0002,
-      "num_tokens": 41973347.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0043,
+      "num_tokens": 41123062.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 72
     },
     {
@@ -2031,26 +2031,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1467.0,
-      "completions/max_terminated_length": 1467.0,
-      "completions/mean_length": 509.81640625,
-      "completions/mean_terminated_length": 508.53619384765625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1747.0,
+      "completions/max_terminated_length": 1747.0,
+      "completions/mean_length": 533.79296875,
+      "completions/mean_terminated_length": 533.79296875,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.06536825610029103,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01945409715205245,
-      "kl": 0.033905029296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.050213646166851394,
+      "kl": 0.0599365234375,
       "learning_rate": 4.84837206268195e-06,
-      "loss": 0.0003,
-      "num_tokens": 42519461.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0,
+      "num_tokens": 41681452.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 73
     },
     {
@@ -2060,25 +2060,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1322.0,
-      "completions/max_terminated_length": 1322.0,
-      "completions/mean_length": 497.779296875,
-      "completions/mean_terminated_length": 497.779296875,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_length": 1085.0,
+      "completions/max_terminated_length": 1085.0,
+      "completions/mean_length": 509.712890625,
+      "completions/mean_terminated_length": 509.712890625,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.0662637116633087,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012293864921911191,
-      "kl": 0.02044677734375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06633850089342387,
+      "kl": 0.05963134765625,
       "learning_rate": 4.8430683762381195e-06,
-      "loss": 0.0002,
-      "num_tokens": 43051540.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0061,
+      "num_tokens": 42219641.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 74
     },
     {
@@ -2087,26 +2087,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1970.0,
-      "completions/mean_length": 557.455078125,
-      "completions/mean_terminated_length": 551.60986328125,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1555.0,
+      "completions/max_terminated_length": 1555.0,
+      "completions/mean_length": 574.283203125,
+      "completions/mean_terminated_length": 574.283203125,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.0671591672263264,
-      "frac_reward_zero_std": 0.84375,
-      "grad_norm": 0.13630913617793994,
-      "kl": 0.0151824951171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.020666631584363913,
+      "kl": 0.060791015625,
       "learning_rate": 4.837676868837213e-06,
-      "loss": 0.0097,
-      "num_tokens": 43647869.0,
-      "reward": 0.09882812201976776,
-      "reward_std": 0.004192390479147434,
+      "loss": 0.0006,
+      "num_tokens": 42824586.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.98828125,
-      "rewards/format_reward/std": 0.10772226005792618,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 75
     },
     {
@@ -2115,20 +2115,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1829.0,
-      "completions/mean_length": 522.078125,
-      "completions/mean_terminated_length": 519.0919799804688,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1942.0,
+      "completions/max_terminated_length": 1942.0,
+      "completions/mean_length": 521.0546875,
+      "completions/mean_terminated_length": 521.0546875,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.06805462278934409,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06750608349497755,
-      "kl": 0.017974853515625,
+      "grad_norm": 0.06331685772788986,
+      "kl": 0.05889892578125,
       "learning_rate": 4.832197766696085e-06,
-      "loss": 0.0069,
-      "num_tokens": 44203525.0,
+      "loss": -0.0,
+      "num_tokens": 43379718.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2143,26 +2143,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1706.0,
-      "completions/mean_length": 554.322265625,
-      "completions/mean_terminated_length": 548.4647216796875,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1598.0,
+      "completions/max_terminated_length": 1598.0,
+      "completions/mean_length": 568.814453125,
+      "completions/mean_terminated_length": 568.814453125,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.06895007835236176,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.10947781516811841,
-      "kl": 0.015289306640625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07366287417422632,
+      "kl": 0.0562744140625,
       "learning_rate": 4.826631299706887e-06,
-      "loss": 0.0199,
-      "num_tokens": 44794538.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": -0.0016,
+      "num_tokens": 43978151.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 77
     },
     {
@@ -2172,25 +2172,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1439.0,
-      "completions/max_terminated_length": 1439.0,
-      "completions/mean_length": 561.875,
-      "completions/mean_terminated_length": 561.875,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/max_length": 1806.0,
+      "completions/max_terminated_length": 1806.0,
+      "completions/mean_length": 569.267578125,
+      "completions/mean_terminated_length": 569.267578125,
+      "completions/min_length": 220.0,
+      "completions/min_terminated_length": 220.0,
       "epoch": 0.06984553391537945,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012482978592863939,
-      "kl": 0.01763916015625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.11848160599061058,
+      "kl": 0.058349609375,
       "learning_rate": 4.820977701427424e-06,
-      "loss": 0.0002,
-      "num_tokens": 45364666.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0152,
+      "num_tokens": 44552064.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 78
     },
     {
@@ -2200,25 +2200,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1624.0,
-      "completions/max_terminated_length": 1624.0,
-      "completions/mean_length": 549.216796875,
-      "completions/mean_terminated_length": 549.216796875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_length": 1567.0,
+      "completions/max_terminated_length": 1567.0,
+      "completions/mean_length": 557.267578125,
+      "completions/mean_terminated_length": 557.267578125,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.07074098947839713,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05673428927364843,
-      "kl": 0.01702880859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.011448547949088674,
+      "kl": 0.06097412109375,
       "learning_rate": 4.81523720907136e-06,
-      "loss": 0.0002,
-      "num_tokens": 45967289.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 45158809.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 79
     },
     {
@@ -2228,25 +2228,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1561.0,
-      "completions/max_terminated_length": 1561.0,
-      "completions/mean_length": 518.8046875,
-      "completions/mean_terminated_length": 518.8046875,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/max_length": 2009.0,
+      "completions/max_terminated_length": 2009.0,
+      "completions/mean_length": 552.33984375,
+      "completions/mean_terminated_length": 552.33984375,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.07163644504141482,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006457555530544499,
-      "kl": 0.016326904296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.056084177201828476,
+      "kl": 0.0584716796875,
       "learning_rate": 4.809410063498254e-06,
-      "loss": 0.0002,
-      "num_tokens": 46545573.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0002,
+      "num_tokens": 45754263.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 80
     },
     {
@@ -2256,25 +2256,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1591.0,
-      "completions/max_terminated_length": 1591.0,
-      "completions/mean_length": 538.142578125,
-      "completions/mean_terminated_length": 538.142578125,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/max_length": 1401.0,
+      "completions/max_terminated_length": 1401.0,
+      "completions/mean_length": 543.6796875,
+      "completions/mean_terminated_length": 543.6796875,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.0725319006044325,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06228818790381435,
-      "kl": 0.016632080078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10338543772762039,
+      "kl": 0.0606689453125,
       "learning_rate": 4.8034965092034656e-06,
-      "loss": -0.0002,
-      "num_tokens": 47144638.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0038,
+      "num_tokens": 46356163.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 81
     },
     {
@@ -2284,19 +2284,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2037.0,
-      "completions/mean_length": 554.693359375,
-      "completions/mean_terminated_length": 551.7710571289062,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/max_length": 1176.0,
+      "completions/max_terminated_length": 1176.0,
+      "completions/mean_length": 584.32421875,
+      "completions/mean_terminated_length": 583.5518798828125,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
       "epoch": 0.07342735616745019,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.062287259542847764,
-      "kl": 0.0159454345703125,
+      "grad_norm": 0.0722417078888751,
+      "kl": 0.0614013671875,
       "learning_rate": 4.797496794307889e-06,
-      "loss": 0.0149,
-      "num_tokens": 47733585.0,
+      "loss": 0.0038,
+      "num_tokens": 46960281.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2311,26 +2311,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1470.0,
-      "completions/max_terminated_length": 1470.0,
-      "completions/mean_length": 566.072265625,
-      "completions/mean_terminated_length": 566.072265625,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1328.0,
+      "completions/mean_length": 599.443359375,
+      "completions/mean_terminated_length": 596.6085815429688,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.07432281173046787,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006352211521854682,
-      "kl": 0.0157623291015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04419533464180944,
+      "kl": 0.0595703125,
       "learning_rate": 4.791411170547545e-06,
-      "loss": 0.0002,
-      "num_tokens": 48350982.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0122,
+      "num_tokens": 47594764.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 83
     },
     {
@@ -2340,19 +2340,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1757.0,
-      "completions/max_terminated_length": 1757.0,
-      "completions/mean_length": 543.951171875,
-      "completions/mean_terminated_length": 543.951171875,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_length": 1370.0,
+      "completions/max_terminated_length": 1370.0,
+      "completions/mean_length": 567.396484375,
+      "completions/mean_terminated_length": 567.396484375,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.07521826729348556,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007813740834546406,
-      "kl": 0.016326904296875,
+      "grad_norm": 0.008048576448064306,
+      "kl": 0.0582275390625,
       "learning_rate": 4.785239893263017e-06,
-      "loss": 0.0002,
-      "num_tokens": 48958221.0,
+      "loss": 0.0006,
+      "num_tokens": 48214007.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2368,25 +2368,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1216.0,
-      "completions/max_terminated_length": 1216.0,
-      "completions/mean_length": 550.677734375,
-      "completions/mean_terminated_length": 550.677734375,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/max_length": 1268.0,
+      "completions/max_terminated_length": 1268.0,
+      "completions/mean_length": 568.2421875,
+      "completions/mean_terminated_length": 568.2421875,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.07611372285650325,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052051410170601047,
-      "kl": 0.016571044921875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.13977703925477217,
+      "kl": 0.06085205078125,
       "learning_rate": 4.778983221388742e-06,
-      "loss": -0.0018,
-      "num_tokens": 49535336.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0049,
+      "num_tokens": 48800115.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 85
     },
     {
@@ -2395,20 +2395,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1647.0,
-      "completions/max_terminated_length": 1647.0,
-      "completions/mean_length": 565.21484375,
-      "completions/mean_terminated_length": 565.21484375,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1454.0,
+      "completions/mean_length": 582.78515625,
+      "completions/mean_terminated_length": 579.9177856445312,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.07700917841952093,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.062482518099836674,
-      "kl": 0.0159912109375,
+      "grad_norm": 0.07934026261985073,
+      "kl": 0.0589599609375,
       "learning_rate": 4.77264141744214e-06,
-      "loss": 0.0016,
-      "num_tokens": 50145670.0,
+      "loss": 0.0097,
+      "num_tokens": 49419445.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2423,26 +2423,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1536.0,
-      "completions/mean_length": 573.24609375,
-      "completions/mean_terminated_length": 570.3600463867188,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1647.0,
+      "completions/max_terminated_length": 1647.0,
+      "completions/mean_length": 590.314453125,
+      "completions/mean_terminated_length": 590.314453125,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.07790463398253862,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.1095124911093521,
-      "kl": 0.0146026611328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06933720029794796,
+      "kl": 0.0592041015625,
       "learning_rate": 4.766214747512603e-06,
-      "loss": 0.0114,
-      "num_tokens": 50749044.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.001,
+      "num_tokens": 50031558.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 87
     },
     {
@@ -2451,20 +2451,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1443.0,
-      "completions/max_terminated_length": 1443.0,
-      "completions/mean_length": 550.958984375,
-      "completions/mean_terminated_length": 550.958984375,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1328.0,
+      "completions/max_terminated_length": 1328.0,
+      "completions/mean_length": 559.21875,
+      "completions/mean_terminated_length": 558.4931640625,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.0788000895455563,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006622879741401474,
-      "kl": 0.014495849609375,
+      "grad_norm": 0.15169183218007004,
+      "kl": 0.07476806640625,
       "learning_rate": 4.759703481250331e-06,
-      "loss": 0.0001,
-      "num_tokens": 51349935.0,
+      "loss": 0.0007,
+      "num_tokens": 50636678.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2480,19 +2480,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1532.0,
-      "completions/max_terminated_length": 1532.0,
-      "completions/mean_length": 546.904296875,
-      "completions/mean_terminated_length": 546.904296875,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/max_length": 1333.0,
+      "completions/max_terminated_length": 1333.0,
+      "completions/mean_length": 577.947265625,
+      "completions/mean_terminated_length": 577.947265625,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.07969554510857399,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005845451074109807,
-      "kl": 0.0149993896484375,
+      "grad_norm": 0.007228029815201077,
+      "kl": 0.058837890625,
       "learning_rate": 4.753107891855015e-06,
-      "loss": 0.0002,
-      "num_tokens": 51983870.0,
+      "loss": 0.0006,
+      "num_tokens": 51286507.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2508,25 +2508,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1887.0,
-      "completions/max_terminated_length": 1887.0,
-      "completions/mean_length": 548.8828125,
-      "completions/mean_terminated_length": 548.8828125,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/max_length": 1744.0,
+      "completions/max_terminated_length": 1744.0,
+      "completions/mean_length": 552.662109375,
+      "completions/mean_terminated_length": 552.662109375,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.08059100067159167,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006245472310213368,
-      "kl": 0.01531982421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05217256333372253,
+      "kl": 0.06201171875,
       "learning_rate": 4.746428256064375e-06,
-      "loss": 0.0002,
-      "num_tokens": 52583970.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.001,
+      "num_tokens": 51888542.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 90
     },
     {
@@ -2536,25 +2536,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1645.0,
-      "completions/max_terminated_length": 1645.0,
-      "completions/mean_length": 518.572265625,
-      "completions/mean_terminated_length": 518.572265625,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_length": 1383.0,
+      "completions/max_terminated_length": 1383.0,
+      "completions/mean_length": 529.599609375,
+      "completions/mean_terminated_length": 529.599609375,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.08148645623460936,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006150422825146274,
-      "kl": 0.0150299072265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07897485826909563,
+      "kl": 0.0584716796875,
       "learning_rate": 4.7396648541425534e-06,
-      "loss": 0.0002,
-      "num_tokens": 53150455.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0055,
+      "num_tokens": 52460673.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 91
     },
     {
@@ -2564,19 +2564,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1218.0,
-      "completions/max_terminated_length": 1218.0,
-      "completions/mean_length": 510.095703125,
-      "completions/mean_terminated_length": 510.095703125,
+      "completions/max_length": 1405.0,
+      "completions/max_terminated_length": 1405.0,
+      "completions/mean_length": 534.078125,
+      "completions/mean_terminated_length": 534.078125,
       "completions/min_length": 159.0,
       "completions/min_terminated_length": 159.0,
       "epoch": 0.08238191179762704,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0077462527558477936,
-      "kl": 0.0150909423828125,
+      "grad_norm": 0.009069004256641576,
+      "kl": 0.058837890625,
       "learning_rate": 4.732817969868348e-06,
-      "loss": 0.0002,
-      "num_tokens": 53730632.0,
+      "loss": 0.0006,
+      "num_tokens": 53053129.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2592,19 +2592,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1567.0,
-      "completions/max_terminated_length": 1567.0,
-      "completions/mean_length": 583.322265625,
-      "completions/mean_terminated_length": 583.322265625,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_length": 1509.0,
+      "completions/max_terminated_length": 1509.0,
+      "completions/mean_length": 581.345703125,
+      "completions/mean_terminated_length": 581.345703125,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.08327736736064473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005995103203475892,
-      "kl": 0.013946533203125,
+      "grad_norm": 0.008041339436883252,
+      "kl": 0.05926513671875,
       "learning_rate": 4.7258878905233095e-06,
-      "loss": 0.0001,
-      "num_tokens": 54355005.0,
+      "loss": 0.0006,
+      "num_tokens": 53676490.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2620,25 +2620,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1284.0,
-      "completions/max_terminated_length": 1284.0,
-      "completions/mean_length": 562.65625,
-      "completions/mean_terminated_length": 562.65625,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/max_length": 1690.0,
+      "completions/max_terminated_length": 1690.0,
+      "completions/mean_length": 570.1796875,
+      "completions/mean_terminated_length": 570.1796875,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.08417282292366242,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008773903997987694,
-      "kl": 0.0150146484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08004068477244176,
+      "kl": 0.06011962890625,
       "learning_rate": 4.718874906879688e-06,
-      "loss": 0.0001,
-      "num_tokens": 54960861.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0048,
+      "num_tokens": 54286198.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 94
     },
     {
@@ -2648,19 +2648,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1428.0,
-      "completions/max_terminated_length": 1428.0,
-      "completions/mean_length": 510.4375,
-      "completions/mean_terminated_length": 510.4375,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/max_length": 1286.0,
+      "completions/max_terminated_length": 1286.0,
+      "completions/mean_length": 503.283203125,
+      "completions/mean_terminated_length": 503.283203125,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.0850682784866801,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005650435077930954,
-      "kl": 0.014801025390625,
+      "grad_norm": 0.007110972110569579,
+      "kl": 0.05828857421875,
       "learning_rate": 4.711779313188231e-06,
-      "loss": 0.0001,
-      "num_tokens": 55487277.0,
+      "loss": 0.0006,
+      "num_tokens": 54808951.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2676,25 +2676,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1508.0,
-      "completions/max_terminated_length": 1508.0,
-      "completions/mean_length": 600.46875,
-      "completions/mean_terminated_length": 600.46875,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/max_length": 1328.0,
+      "completions/max_terminated_length": 1328.0,
+      "completions/mean_length": 575.7421875,
+      "completions/mean_terminated_length": 575.7421875,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.08596373404969779,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005507101143768758,
-      "kl": 0.012725830078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06146080952581339,
+      "kl": 0.058349609375,
       "learning_rate": 4.70460140716584e-06,
-      "loss": 0.0001,
-      "num_tokens": 56115997.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0038,
+      "num_tokens": 55425011.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 96
     },
     {
@@ -2703,26 +2703,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1394.0,
-      "completions/mean_length": 501.939453125,
-      "completions/mean_terminated_length": 498.91387939453125,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1760.0,
+      "completions/max_terminated_length": 1760.0,
+      "completions/mean_length": 510.607421875,
+      "completions/mean_terminated_length": 510.607421875,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.08685918961271547,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07000735578379153,
-      "kl": 0.0143585205078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.013320757255101928,
+      "kl": 0.05645751953125,
       "learning_rate": 4.697341489983076e-06,
-      "loss": 0.0115,
-      "num_tokens": 56651950.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 55965402.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 97
     },
     {
@@ -2732,19 +2732,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1373.0,
-      "completions/max_terminated_length": 1373.0,
-      "completions/mean_length": 582.009765625,
-      "completions/mean_terminated_length": 582.009765625,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/max_length": 1890.0,
+      "completions/max_terminated_length": 1890.0,
+      "completions/mean_length": 565.958984375,
+      "completions/mean_terminated_length": 565.958984375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.08775464517573316,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005593438153271221,
-      "kl": 0.0135498046875,
+      "grad_norm": 0.007573822836148033,
+      "kl": 0.05853271484375,
       "learning_rate": 4.6899998662515215e-06,
-      "loss": 0.0001,
-      "num_tokens": 57266531.0,
+      "loss": 0.0006,
+      "num_tokens": 56571765.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2760,19 +2760,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1399.0,
-      "completions/max_terminated_length": 1399.0,
-      "completions/mean_length": 511.146484375,
-      "completions/mean_terminated_length": 511.146484375,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/max_length": 1135.0,
+      "completions/max_terminated_length": 1135.0,
+      "completions/mean_length": 503.03125,
+      "completions/mean_terminated_length": 503.03125,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.08865010073875083,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005254315257483802,
-      "kl": 0.014007568359375,
+      "grad_norm": 0.007334410202363631,
+      "kl": 0.05633544921875,
       "learning_rate": 4.682576844011007e-06,
-      "loss": 0.0001,
-      "num_tokens": 57823134.0,
+      "loss": 0.0006,
+      "num_tokens": 57124213.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2788,19 +2788,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1946.0,
-      "completions/max_terminated_length": 1946.0,
-      "completions/mean_length": 561.2421875,
-      "completions/mean_terminated_length": 561.2421875,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/max_length": 1658.0,
+      "completions/max_terminated_length": 1658.0,
+      "completions/mean_length": 536.2421875,
+      "completions/mean_terminated_length": 536.2421875,
+      "completions/min_length": 134.0,
+      "completions/min_terminated_length": 134.0,
       "epoch": 0.08954555630176853,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0050205373550895075,
-      "kl": 0.0132904052734375,
+      "grad_norm": 0.007211066238880028,
+      "kl": 0.05853271484375,
       "learning_rate": 4.675072734716678e-06,
-      "loss": 0.0001,
-      "num_tokens": 58394426.0,
+      "loss": 0.0006,
+      "num_tokens": 57682705.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2815,26 +2815,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1703.0,
-      "completions/mean_length": 631.42578125,
-      "completions/mean_terminated_length": 628.6536254882812,
-      "completions/min_length": 225.0,
-      "completions/min_terminated_length": 225.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1632.0,
+      "completions/max_terminated_length": 1632.0,
+      "completions/mean_length": 554.22265625,
+      "completions/mean_terminated_length": 554.22265625,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.09044101186478622,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0760184868440684,
-      "kl": 0.01263427734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006708160382694399,
+      "kl": 0.05792236328125,
       "learning_rate": 4.667487853225931e-06,
-      "loss": 0.0097,
-      "num_tokens": 59062004.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 58310755.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 101
     },
     {
@@ -2843,20 +2843,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1712.0,
-      "completions/mean_length": 610.859375,
-      "completions/mean_terminated_length": 607.2745361328125,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_terminated_length": 1931.0,
+      "completions/mean_length": 566.173828125,
+      "completions/mean_terminated_length": 563.2739868164062,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.0913364674278039,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05564333998730713,
-      "kl": 0.02105712890625,
+      "grad_norm": 0.07622570043853691,
+      "kl": 0.05584716796875,
       "learning_rate": 4.659822517785203e-06,
-      "loss": 0.0092,
-      "num_tokens": 59662988.0,
+      "loss": 0.0137,
+      "num_tokens": 58888860.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2871,26 +2871,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2037.0,
-      "completions/mean_length": 611.19921875,
-      "completions/mean_terminated_length": 608.387451171875,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1372.0,
+      "completions/max_terminated_length": 1372.0,
+      "completions/mean_length": 553.12109375,
+      "completions/mean_terminated_length": 553.12109375,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.09223192299082159,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10935821324700373,
-      "kl": 0.013153076171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.049130696414956106,
+      "kl": 0.05828857421875,
       "learning_rate": 4.6520770500166165e-06,
-      "loss": 0.0109,
-      "num_tokens": 60289554.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": -0.0001,
+      "num_tokens": 59485690.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 103
     },
     {
@@ -2899,26 +2899,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1446.0,
-      "completions/max_terminated_length": 1446.0,
-      "completions/mean_length": 640.951171875,
-      "completions/mean_terminated_length": 640.951171875,
-      "completions/min_length": 225.0,
-      "completions/min_terminated_length": 225.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1689.0,
+      "completions/max_terminated_length": 1689.0,
+      "completions/mean_length": 592.68359375,
+      "completions/mean_terminated_length": 591.5635986328125,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.09312737855383926,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.060757676548859295,
-      "kl": 0.0118408203125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 4.817209315659024,
+      "kl": 0.07928466796875,
       "learning_rate": 4.644251774904487e-06,
-      "loss": 0.0005,
-      "num_tokens": 60964441.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0076,
+      "num_tokens": 60135864.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 104
     },
     {
@@ -2928,19 +2928,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1451.0,
-      "completions/max_terminated_length": 1451.0,
-      "completions/mean_length": 592.787109375,
-      "completions/mean_terminated_length": 592.787109375,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/max_length": 1234.0,
+      "completions/max_terminated_length": 1234.0,
+      "completions/mean_length": 563.57421875,
+      "completions/mean_terminated_length": 563.57421875,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.09402283411685695,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005244427082623526,
-      "kl": 0.011688232421875,
+      "grad_norm": 0.009712812135983481,
+      "kl": 0.05499267578125,
       "learning_rate": 4.636347020781684e-06,
-      "loss": 0.0001,
-      "num_tokens": 61555548.0,
+      "loss": 0.0006,
+      "num_tokens": 60712014.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2955,20 +2955,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1920.0,
-      "completions/mean_length": 576.23046875,
-      "completions/mean_terminated_length": 573.3502807617188,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1801.0,
+      "completions/max_terminated_length": 1801.0,
+      "completions/mean_length": 551.984375,
+      "completions/mean_terminated_length": 551.984375,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.09491828967987463,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08855757430577488,
-      "kl": 0.0129241943359375,
+      "grad_norm": 0.06067404264171204,
+      "kl": 0.05560302734375,
       "learning_rate": 4.6283631193158605e-06,
-      "loss": 0.0125,
-      "num_tokens": 62129122.0,
+      "loss": 0.0012,
+      "num_tokens": 61273174.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2984,19 +2984,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1757.0,
-      "completions/max_terminated_length": 1757.0,
-      "completions/mean_length": 587.9765625,
-      "completions/mean_terminated_length": 585.6888427734375,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/max_length": 1733.0,
+      "completions/max_terminated_length": 1733.0,
+      "completions/mean_length": 565.373046875,
+      "completions/mean_terminated_length": 564.3267822265625,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.09581374524289232,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.018552764774495144,
-      "kl": 0.0251617431640625,
+      "grad_norm": 19.400452290618507,
+      "kl": 3.197265625,
       "learning_rate": 4.620300405495532e-06,
-      "loss": 0.0003,
-      "num_tokens": 62720198.0,
+      "loss": 0.032,
+      "num_tokens": 61852677.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3011,26 +3011,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1823.0,
-      "completions/max_terminated_length": 1823.0,
-      "completions/mean_length": 590.748046875,
-      "completions/mean_terminated_length": 590.748046875,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1473.0,
+      "completions/mean_length": 554.125,
+      "completions/mean_terminated_length": 551.2015380859375,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.09670920080591,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011091417692320823,
-      "kl": 0.01226806640625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0524692551884981,
+      "kl": 0.05401611328125,
       "learning_rate": 4.612159217616022e-06,
-      "loss": 0.0001,
-      "num_tokens": 63348581.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0146,
+      "num_tokens": 62462309.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 108
     },
     {
@@ -3040,19 +3040,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1711.0,
-      "completions/max_terminated_length": 1711.0,
-      "completions/mean_length": 558.81640625,
-      "completions/mean_terminated_length": 558.81640625,
-      "completions/min_length": 249.0,
-      "completions/min_terminated_length": 249.0,
+      "completions/max_length": 1408.0,
+      "completions/max_terminated_length": 1408.0,
+      "completions/mean_length": 546.9453125,
+      "completions/mean_terminated_length": 546.9453125,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
       "epoch": 0.09760465636892769,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009595552547661724,
-      "kl": 0.0125274658203125,
+      "grad_norm": 0.006616016440406459,
+      "kl": 0.05621337890625,
       "learning_rate": 4.603939897265268e-06,
-      "loss": 0.0001,
-      "num_tokens": 63942599.0,
+      "loss": 0.0006,
+      "num_tokens": 63050249.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3068,19 +3068,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1506.0,
-      "completions/max_terminated_length": 1506.0,
-      "completions/mean_length": 533.255859375,
-      "completions/mean_terminated_length": 533.255859375,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/max_length": 1746.0,
+      "completions/max_terminated_length": 1746.0,
+      "completions/mean_length": 540.046875,
+      "completions/mean_terminated_length": 540.046875,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.09850011193194538,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004424584359067543,
-      "kl": 0.0120849609375,
+      "grad_norm": 0.006058315501498899,
+      "kl": 0.0535888671875,
       "learning_rate": 4.595642789309492e-06,
-      "loss": 0.0001,
-      "num_tokens": 64507882.0,
+      "loss": 0.0005,
+      "num_tokens": 63619009.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3096,19 +3096,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1444.0,
-      "completions/max_terminated_length": 1444.0,
-      "completions/mean_length": 543.14453125,
-      "completions/mean_terminated_length": 543.14453125,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_length": 1615.0,
+      "completions/max_terminated_length": 1615.0,
+      "completions/mean_length": 546.98046875,
+      "completions/mean_terminated_length": 546.98046875,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.09939556749496306,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005542511266984594,
-      "kl": 0.0128021240234375,
+      "grad_norm": 0.006228639344980519,
+      "kl": 0.05523681640625,
       "learning_rate": 4.587268241878724e-06,
-      "loss": 0.0001,
-      "num_tokens": 65052900.0,
+      "loss": 0.0006,
+      "num_tokens": 64165991.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3123,26 +3123,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1646.0,
-      "completions/mean_length": 572.822265625,
-      "completions/mean_terminated_length": 566.6071166992188,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1374.0,
+      "completions/max_terminated_length": 1374.0,
+      "completions/mean_length": 555.0703125,
+      "completions/mean_terminated_length": 554.5205688476562,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
       "epoch": 0.10029102305798075,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10304996756847001,
-      "kl": 0.01776123046875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 1.2290379791712172,
+      "kl": 0.05560302734375,
       "learning_rate": 4.578816606352205e-06,
-      "loss": 0.0254,
-      "num_tokens": 65664441.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0045,
+      "num_tokens": 64768443.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 112
     },
     {
@@ -3152,25 +3152,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1262.0,
-      "completions/max_terminated_length": 1262.0,
-      "completions/mean_length": 570.673828125,
-      "completions/mean_terminated_length": 569.7299194335938,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/max_length": 1258.0,
+      "completions/max_terminated_length": 1258.0,
+      "completions/mean_length": 563.681640625,
+      "completions/mean_terminated_length": 562.8062744140625,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.10118647862099843,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009706307742035009,
-      "kl": 0.0243988037109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 16.255998645519178,
+      "kl": 0.74365234375,
       "learning_rate": 4.570288237343632e-06,
-      "loss": 0.0002,
-      "num_tokens": 66256930.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0048,
+      "num_tokens": 65357352.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 113
     },
     {
@@ -3179,26 +3179,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1857.0,
-      "completions/max_terminated_length": 1857.0,
-      "completions/mean_length": 613.189453125,
-      "completions/mean_terminated_length": 612.127197265625,
-      "completions/min_length": 225.0,
-      "completions/min_terminated_length": 225.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1490.0,
+      "completions/max_terminated_length": 1490.0,
+      "completions/mean_length": 586.935546875,
+      "completions/mean_terminated_length": 586.935546875,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.10208193418401612,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00881535183664981,
-      "kl": 0.02374267578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0774700444571494,
+      "kl": 0.0572509765625,
       "learning_rate": 4.561683492686289e-06,
-      "loss": 0.0002,
-      "num_tokens": 66881555.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0038,
+      "num_tokens": 65968535.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 114
     },
     {
@@ -3208,19 +3208,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1393.0,
-      "completions/max_terminated_length": 1393.0,
-      "completions/mean_length": 577.60546875,
-      "completions/mean_terminated_length": 577.60546875,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/max_length": 1247.0,
+      "completions/max_terminated_length": 1247.0,
+      "completions/mean_length": 541.302734375,
+      "completions/mean_terminated_length": 541.302734375,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.1029773897470338,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004670068300340548,
-      "kl": 0.0115203857421875,
+      "grad_norm": 0.00852963260021217,
+      "kl": 0.0596923828125,
       "learning_rate": 4.5530027334180285e-06,
-      "loss": 0.0001,
-      "num_tokens": 67466761.0,
+      "loss": 0.0006,
+      "num_tokens": 66535154.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3236,25 +3236,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1225.0,
-      "completions/max_terminated_length": 1225.0,
-      "completions/mean_length": 559.638671875,
-      "completions/mean_terminated_length": 559.638671875,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_length": 1153.0,
+      "completions/max_terminated_length": 1153.0,
+      "completions/mean_length": 504.041015625,
+      "completions/mean_terminated_length": 504.041015625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.10387284531005149,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09101415243660085,
-      "kl": 0.0117034912109375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00898169383540532,
+      "kl": 0.060791015625,
       "learning_rate": 4.544246323766122e-06,
-      "loss": 0.0026,
-      "num_tokens": 68064704.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 67104631.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 116
     },
     {
@@ -3264,19 +3264,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1907.0,
-      "completions/max_terminated_length": 1907.0,
-      "completions/mean_length": 582.666015625,
-      "completions/mean_terminated_length": 582.666015625,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 1247.0,
+      "completions/max_terminated_length": 1247.0,
+      "completions/mean_length": 537.59375,
+      "completions/mean_terminated_length": 537.59375,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.10476830087306917,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005123090555119176,
-      "kl": 0.0115203857421875,
+      "grad_norm": 0.009282105832995153,
+      "kl": 0.06085205078125,
       "learning_rate": 4.535414631131983e-06,
-      "loss": 0.0001,
-      "num_tokens": 68651269.0,
+      "loss": 0.0006,
+      "num_tokens": 67668119.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3291,26 +3291,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1942.0,
-      "completions/mean_length": 560.41015625,
-      "completions/mean_terminated_length": 557.4990234375,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1405.0,
+      "completions/max_terminated_length": 1405.0,
+      "completions/mean_length": 516.166015625,
+      "completions/mean_terminated_length": 516.166015625,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.10566375643608686,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08107964407237964,
-      "kl": 0.01141357421875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07276575023389194,
+      "kl": 0.06048583984375,
       "learning_rate": 4.526508026075746e-06,
-      "loss": 0.0113,
-      "num_tokens": 69243847.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0023,
+      "num_tokens": 68238044.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 118
     },
     {
@@ -3319,20 +3319,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1367.0,
-      "completions/max_terminated_length": 1367.0,
-      "completions/mean_length": 581.55078125,
-      "completions/mean_terminated_length": 580.6907958984375,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1659.0,
+      "completions/max_terminated_length": 1659.0,
+      "completions/mean_length": 546.119140625,
+      "completions/mean_terminated_length": 546.119140625,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
       "epoch": 0.10655921199910455,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.08267105634677789,
-      "kl": 0.02154541015625,
+      "grad_norm": 0.007709260903345854,
+      "kl": 0.056884765625,
       "learning_rate": 4.517526882300721e-06,
-      "loss": 0.0002,
-      "num_tokens": 69856081.0,
+      "loss": 0.0006,
+      "num_tokens": 68832137.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3348,25 +3348,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1711.0,
-      "completions/max_terminated_length": 1711.0,
-      "completions/mean_length": 568.03125,
-      "completions/mean_terminated_length": 568.03125,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_length": 1518.0,
+      "completions/max_terminated_length": 1518.0,
+      "completions/mean_length": 510.16796875,
+      "completions/mean_terminated_length": 510.16796875,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.10745466756212223,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010809362339601222,
-      "kl": 0.0114898681640625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.12374567038474613,
+      "kl": 0.05853271484375,
       "learning_rate": 4.508471576637713e-06,
-      "loss": 0.0001,
-      "num_tokens": 70470401.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0052,
+      "num_tokens": 69416831.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 120
     },
     {
@@ -3375,26 +3375,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1704.0,
-      "completions/mean_length": 572.75390625,
-      "completions/mean_terminated_length": 569.866943359375,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1343.0,
+      "completions/max_terminated_length": 1343.0,
+      "completions/mean_length": 517.318359375,
+      "completions/mean_terminated_length": 517.318359375,
       "completions/min_length": 192.0,
       "completions/min_terminated_length": 192.0,
       "epoch": 0.10835012312513992,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09457771857394989,
-      "kl": 0.0112457275390625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007982824080905008,
+      "kl": 0.0570068359375,
       "learning_rate": 4.499342489029211e-06,
-      "loss": 0.0123,
-      "num_tokens": 71065347.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 69983394.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 121
     },
     {
@@ -3404,25 +3404,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1688.0,
-      "completions/max_terminated_length": 1688.0,
-      "completions/mean_length": 578.771484375,
-      "completions/mean_terminated_length": 578.771484375,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/max_length": 1176.0,
+      "completions/max_terminated_length": 1176.0,
+      "completions/mean_length": 503.2109375,
+      "completions/mean_terminated_length": 503.2109375,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
       "epoch": 0.1092455786881576,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009517196854567733,
-      "kl": 0.0113067626953125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05137313735665502,
+      "kl": 0.0572509765625,
       "learning_rate": 4.490140002513449e-06,
-      "loss": 0.0001,
-      "num_tokens": 71674942.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0022,
+      "num_tokens": 70554302.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 122
     },
     {
@@ -3431,26 +3431,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1412.0,
-      "completions/max_terminated_length": 1412.0,
-      "completions/mean_length": 555.53515625,
-      "completions/mean_terminated_length": 555.53515625,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1103.0,
+      "completions/max_terminated_length": 1103.0,
+      "completions/mean_length": 492.255859375,
+      "completions/mean_terminated_length": 491.2739562988281,
+      "completions/min_length": 90.0,
+      "completions/min_terminated_length": 90.0,
       "epoch": 0.11014103425117529,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009205657515159494,
-      "kl": 0.011474609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 6.6575044592016654,
+      "kl": 0.06549072265625,
       "learning_rate": 4.48086450320833e-06,
-      "loss": 0.0001,
-      "num_tokens": 72255696.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0053,
+      "num_tokens": 71102657.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 123
     },
     {
@@ -3460,25 +3460,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1758.0,
-      "completions/max_terminated_length": 1758.0,
-      "completions/mean_length": 553.37890625,
-      "completions/mean_terminated_length": 553.37890625,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/max_length": 1139.0,
+      "completions/max_terminated_length": 1139.0,
+      "completions/mean_length": 501.88671875,
+      "completions/mean_terminated_length": 501.88671875,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
       "epoch": 0.11103648981419297,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00819542614576692,
-      "kl": 0.010772705078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.13448036932392413,
+      "kl": 0.05462646484375,
       "learning_rate": 4.4715163802952266e-06,
-      "loss": 0.0001,
-      "num_tokens": 72841858.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0022,
+      "num_tokens": 71662455.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 124
     },
     {
@@ -3487,20 +3487,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1450.0,
-      "completions/max_terminated_length": 1450.0,
-      "completions/mean_length": 618.697265625,
-      "completions/mean_terminated_length": 617.0704345703125,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1883.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 558.08203125,
+      "completions/mean_terminated_length": 558.08203125,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.11193194537721066,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.013122697785598695,
-      "kl": 0.0197296142578125,
+      "grad_norm": 0.008863906624678173,
+      "kl": 0.05322265625,
       "learning_rate": 4.462096026002655e-06,
-      "loss": 0.0002,
-      "num_tokens": 73446887.0,
+      "loss": 0.0005,
+      "num_tokens": 72236449.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3516,19 +3516,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1454.0,
-      "completions/max_terminated_length": 1454.0,
-      "completions/mean_length": 570.470703125,
-      "completions/mean_terminated_length": 570.470703125,
-      "completions/min_length": 238.0,
-      "completions/min_terminated_length": 238.0,
+      "completions/max_length": 1646.0,
+      "completions/max_terminated_length": 1646.0,
+      "completions/mean_length": 523.720703125,
+      "completions/mean_terminated_length": 523.720703125,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.11282740094022835,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0066434644747071486,
-      "kl": 0.010589599609375,
+      "grad_norm": 0.009601045986280915,
+      "kl": 0.0523681640625,
       "learning_rate": 4.4526038355898144e-06,
-      "loss": 0.0001,
-      "num_tokens": 74051720.0,
+      "loss": 0.0005,
+      "num_tokens": 72817346.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3543,20 +3543,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1642.0,
-      "completions/max_terminated_length": 1642.0,
-      "completions/mean_length": 564.19921875,
-      "completions/mean_terminated_length": 562.0900268554688,
-      "completions/min_length": 222.0,
-      "completions/min_terminated_length": 222.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1414.0,
+      "completions/max_terminated_length": 1414.0,
+      "completions/mean_length": 511.84375,
+      "completions/mean_terminated_length": 511.84375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.11372285650324603,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.022334212444641555,
-      "kl": 0.0200653076171875,
+      "grad_norm": 0.01150702874361092,
+      "kl": 0.05126953125,
       "learning_rate": 4.4430402073300035e-06,
-      "loss": 0.0002,
-      "num_tokens": 74651822.0,
+      "loss": 0.0005,
+      "num_tokens": 73390642.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3571,20 +3571,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1330.0,
-      "completions/max_terminated_length": 1330.0,
-      "completions/mean_length": 573.5546875,
-      "completions/mean_terminated_length": 572.8140869140625,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1298.0,
+      "completions/max_terminated_length": 1298.0,
+      "completions/mean_length": 535.732421875,
+      "completions/mean_terminated_length": 535.732421875,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.11461831206626372,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005480335975480727,
-      "kl": 0.0178680419921875,
+      "grad_norm": 0.9117448141973743,
+      "kl": 0.12908935546875,
       "learning_rate": 4.433405542493909e-06,
-      "loss": 0.0002,
-      "num_tokens": 75240074.0,
+      "loss": 0.0013,
+      "num_tokens": 73959529.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3599,20 +3599,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1416.0,
-      "completions/max_terminated_length": 1416.0,
-      "completions/mean_length": 614.1953125,
-      "completions/mean_terminated_length": 614.1953125,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1485.0,
+      "completions/mean_length": 606.228515625,
+      "completions/mean_terminated_length": 594.8759765625,
+      "completions/min_length": 275.0,
+      "completions/min_terminated_length": 275.0,
       "epoch": 0.1155137676292814,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006019467357919121,
-      "kl": 0.0113525390625,
+      "grad_norm": 0.014323382678470751,
+      "kl": 0.0703125,
       "learning_rate": 4.4237002453327734e-06,
-      "loss": 0.0001,
-      "num_tokens": 75861150.0,
+      "loss": 0.0007,
+      "num_tokens": 74576526.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3627,26 +3627,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1408.0,
-      "completions/max_terminated_length": 1408.0,
-      "completions/mean_length": 561.0,
-      "completions/mean_terminated_length": 559.1353149414062,
-      "completions/min_length": 174.0,
-      "completions/min_terminated_length": 174.0,
+      "completions/clipped_ratio": -6.828125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1993.0,
+      "completions/mean_length": 693.314453125,
+      "completions/mean_terminated_length": 663.5708618164062,
+      "completions/min_length": 261.0,
+      "completions/min_terminated_length": 261.0,
       "epoch": 0.11640922319229909,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.016407802186772742,
-      "kl": 0.0272216796875,
+      "frac_reward_zero_std": 0.8125,
+      "grad_norm": 0.18814500549038302,
+      "kl": 0.106689453125,
       "learning_rate": 4.4139247230614245e-06,
-      "loss": 0.0003,
-      "num_tokens": 76456942.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0036,
+      "num_tokens": 75240063.0,
+      "reward": 0.09882812201976776,
+      "reward_std": 0.004687500186264515,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 130
     },
     {
@@ -3655,20 +3655,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1149.0,
-      "completions/max_terminated_length": 1149.0,
-      "completions/mean_length": 547.41015625,
-      "completions/mean_terminated_length": 547.41015625,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/clipped_ratio": -6.53125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2036.0,
+      "completions/mean_length": 822.12109375,
+      "completions/mean_terminated_length": 745.8215942382812,
+      "completions/min_length": 282.0,
+      "completions/min_terminated_length": 282.0,
       "epoch": 0.11730467875531676,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004386721645387016,
-      "kl": 0.01092529296875,
+      "grad_norm": 0.02343943908233584,
+      "kl": 0.1158447265625,
       "learning_rate": 4.404079385841201e-06,
-      "loss": 0.0001,
-      "num_tokens": 77024864.0,
+      "loss": 0.0012,
+      "num_tokens": 75948637.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3683,26 +3683,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1368.0,
-      "completions/max_terminated_length": 1368.0,
-      "completions/mean_length": 595.248046875,
-      "completions/mean_terminated_length": 595.248046875,
-      "completions/min_length": 132.0,
-      "completions/min_terminated_length": 132.0,
+      "completions/clipped_ratio": -6.015625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2039.0,
+      "completions/mean_length": 994.90625,
+      "completions/mean_terminated_length": 847.144775390625,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.11820013431833445,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0040065240940903215,
-      "kl": 0.0105133056640625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.07306244420161156,
+      "kl": 0.11376953125,
       "learning_rate": 4.394164646762734e-06,
-      "loss": 0.0001,
-      "num_tokens": 77623023.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0054,
+      "num_tokens": 76751421.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 132
     },
     {
@@ -3711,26 +3711,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1660.0,
-      "completions/max_terminated_length": 1660.0,
-      "completions/mean_length": 584.369140625,
-      "completions/mean_terminated_length": 582.9862670898438,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -5.84375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2009.0,
+      "completions/mean_length": 1017.630859375,
+      "completions/mean_terminated_length": 843.5501708984375,
+      "completions/min_length": 270.0,
+      "completions/min_terminated_length": 270.0,
       "epoch": 0.11909558988135213,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01841556230332692,
-      "kl": 0.0206756591796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04774455505526359,
+      "kl": 0.11669921875,
       "learning_rate": 4.384180921828618e-06,
-      "loss": 0.0002,
-      "num_tokens": 78280956.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0014,
+      "num_tokens": 77631184.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 133
     },
     {
@@ -3739,26 +3739,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -5.1875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1502.0,
-      "completions/mean_length": 593.40625,
-      "completions/mean_terminated_length": 590.5596923828125,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "completions/max_terminated_length": 2044.0,
+      "completions/mean_length": 1152.490234375,
+      "completions/mean_terminated_length": 890.169189453125,
+      "completions/min_length": 266.0,
+      "completions/min_terminated_length": 266.0,
       "epoch": 0.11999104544436982,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07155830102688933,
-      "kl": 0.0109710693359375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07160446510060933,
+      "kl": 0.120361328125,
       "learning_rate": 4.374128629935955e-06,
-      "loss": 0.0096,
-      "num_tokens": 78888988.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0006,
+      "num_tokens": 78525467.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 134
     },
     {
@@ -3767,26 +3767,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1534.0,
-      "completions/max_terminated_length": 1534.0,
-      "completions/mean_length": 585.830078125,
-      "completions/mean_terminated_length": 585.830078125,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -4.59375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1966.0,
+      "completions/mean_length": 1246.322265625,
+      "completions/mean_terminated_length": 901.4664306640625,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.12088650100738751,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004840567540768247,
-      "kl": 0.0105743408203125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.05630071606380346,
+      "kl": 0.11083984375,
       "learning_rate": 4.364008192858781e-06,
-      "loss": 0.0001,
-      "num_tokens": 79501093.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.003,
+      "num_tokens": 79475744.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 135
     },
     {
@@ -3795,26 +3795,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1482.0,
-      "completions/max_terminated_length": 1482.0,
-      "completions/mean_length": 592.638671875,
-      "completions/mean_terminated_length": 592.638671875,
-      "completions/min_length": 139.0,
-      "completions/min_terminated_length": 139.0,
+      "completions/clipped_ratio": -4.5625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2027.0,
+      "completions/mean_length": 1288.12890625,
+      "completions/mean_terminated_length": 955.1516723632812,
+      "completions/min_length": 243.0,
+      "completions/min_terminated_length": 243.0,
       "epoch": 0.12178195657040519,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00446197838012533,
-      "kl": 0.0106964111328125,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 795332.7231700029,
+      "kl": 14656.082153320312,
       "learning_rate": 4.353820035230366e-06,
-      "loss": 0.0001,
-      "num_tokens": 80102332.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 145.7659,
+      "num_tokens": 80433074.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 136
     },
     {
@@ -3823,26 +3823,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1632.0,
-      "completions/max_terminated_length": 1632.0,
-      "completions/mean_length": 536.201171875,
-      "completions/mean_terminated_length": 534.630126953125,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -4.265625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2030.0,
+      "completions/mean_length": 1294.751953125,
+      "completions/mean_terminated_length": 903.5994262695312,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.12267741213342288,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014022538127291095,
-      "kl": 0.0203704833984375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.1687795917678753,
+      "kl": 0.126708984375,
       "learning_rate": 4.3435645845254e-06,
-      "loss": 0.0002,
-      "num_tokens": 80674819.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0049,
+      "num_tokens": 81393939.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 137
     },
     {
@@ -3851,26 +3851,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1379.0,
-      "completions/max_terminated_length": 1379.0,
-      "completions/mean_length": 611.9296875,
-      "completions/mean_terminated_length": 611.9296875,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -4.015625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2025.0,
+      "completions/mean_length": 1392.58203125,
+      "completions/mean_terminated_length": 1002.5980834960938,
+      "completions/min_length": 301.0,
+      "completions/min_terminated_length": 301.0,
       "epoch": 0.12357286769644056,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004263453316826543,
-      "kl": 0.010223388671875,
+      "frac_reward_zero_std": 0.78125,
+      "grad_norm": 0.09891611716752852,
+      "kl": 0.1055908203125,
       "learning_rate": 4.333242271042054e-06,
-      "loss": 0.0001,
-      "num_tokens": 81339439.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0155,
+      "num_tokens": 82458253.0,
+      "reward": 0.09843750298023224,
+      "reward_std": 0.005754890851676464,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
       "step": 138
     },
     {
@@ -3879,26 +3879,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1569.0,
-      "completions/max_terminated_length": 1569.0,
-      "completions/mean_length": 565.08203125,
-      "completions/mean_terminated_length": 565.08203125,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -4.3125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2020.0,
+      "completions/mean_length": 1325.6328125,
+      "completions/mean_terminated_length": 960.2000122070312,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.12446832325945825,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05663482825888845,
-      "kl": 0.009979248046875,
+      "frac_reward_zero_std": 0.75,
+      "grad_norm": 0.11212612719912256,
+      "kl": 0.1063232421875,
       "learning_rate": 4.32285352788393e-06,
-      "loss": 0.0015,
-      "num_tokens": 81948409.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.005,
+      "num_tokens": 83456625.0,
+      "reward": 0.09824219346046448,
+      "reward_std": 0.006536140572279692,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.982421875,
+      "rewards/format_reward/std": 0.13154059648513794,
       "step": 139
     },
     {
@@ -3907,26 +3907,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1278.0,
-      "completions/max_terminated_length": 1278.0,
-      "completions/mean_length": 557.494140625,
-      "completions/mean_terminated_length": 557.494140625,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/clipped_ratio": -4.15625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2022.0,
+      "completions/mean_length": 1346.60546875,
+      "completions/mean_terminated_length": 959.7757568359375,
+      "completions/min_length": 272.0,
+      "completions/min_terminated_length": 272.0,
       "epoch": 0.12536377882247593,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032894992448206737,
-      "kl": 0.0096282958984375,
+      "frac_reward_zero_std": 0.78125,
+      "grad_norm": 12694.24016060589,
+      "kl": 618.6192626953125,
       "learning_rate": 4.312398790941882e-06,
-      "loss": 0.0001,
-      "num_tokens": 82548918.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 6.1707,
+      "num_tokens": 84461159.0,
+      "reward": 0.09843750298023224,
+      "reward_std": 0.005754890851676464,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
       "step": 140
     },
     {
@@ -3935,26 +3935,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1240.0,
-      "completions/max_terminated_length": 1240.0,
-      "completions/mean_length": 507.896484375,
-      "completions/mean_terminated_length": 507.896484375,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -4.671875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2011.0,
+      "completions/mean_length": 1241.689453125,
+      "completions/mean_terminated_length": 910.7245483398438,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.1262592343854936,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0036947275566920268,
-      "kl": 0.010162353515625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.059238726705022914,
+      "kl": 0.1033935546875,
       "learning_rate": 4.301878498875735e-06,
-      "loss": 0.0001,
-      "num_tokens": 83095585.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 85383528.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 141
     },
     {
@@ -3963,26 +3963,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1426.0,
-      "completions/max_terminated_length": 1426.0,
-      "completions/mean_length": 519.384765625,
-      "completions/mean_terminated_length": 519.384765625,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/clipped_ratio": -4.421875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2011.0,
+      "completions/mean_length": 1294.9375,
+      "completions/mean_terminated_length": 936.85302734375,
+      "completions/min_length": 306.0,
+      "completions/min_terminated_length": 306.0,
       "epoch": 0.1271546899485113,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006416399119562177,
-      "kl": 0.0099639892578125,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 0.08422896350695738,
+      "kl": 0.1038818359375,
       "learning_rate": 4.291293093095873e-06,
-      "loss": 0.0001,
-      "num_tokens": 83645174.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0012,
+      "num_tokens": 86330200.0,
+      "reward": 0.09882812947034836,
+      "reward_std": 0.004192390479147434,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 142
     },
     {
@@ -3991,26 +3991,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1506.0,
-      "completions/max_terminated_length": 1506.0,
-      "completions/mean_length": 593.76953125,
-      "completions/mean_terminated_length": 593.0410766601562,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -4.203125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2007.0,
+      "completions/mean_length": 1359.435546875,
+      "completions/mean_terminated_length": 989.3063354492188,
+      "completions/min_length": 276.0,
+      "completions/min_terminated_length": 276.0,
       "epoch": 0.128050145511529,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007294106241650529,
-      "kl": 0.015716552734375,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 1.0100953665705108,
+      "kl": 0.123291015625,
       "learning_rate": 4.280643017744723e-06,
-      "loss": 0.0002,
-      "num_tokens": 84292448.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0056,
+      "num_tokens": 87369495.0,
+      "reward": 0.09902344644069672,
+      "reward_std": 0.00390625,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "step": 143
     },
     {
@@ -4019,26 +4019,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1758.0,
-      "completions/max_terminated_length": 1758.0,
-      "completions/mean_length": 615.357421875,
-      "completions/mean_terminated_length": 615.357421875,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/clipped_ratio": -3.859375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2021.0,
+      "completions/mean_length": 1424.6328125,
+      "completions/mean_terminated_length": 1021.7492065429688,
+      "completions/min_length": 348.0,
+      "completions/min_terminated_length": 348.0,
       "epoch": 0.12894560107454667,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006310239000280281,
-      "kl": 0.0094757080078125,
+      "frac_reward_zero_std": 0.6875,
+      "grad_norm": 0.09935539337282194,
+      "kl": 0.0885009765625,
       "learning_rate": 4.269928719678117e-06,
-      "loss": 0.0001,
-      "num_tokens": 84913975.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0017,
+      "num_tokens": 88405371.0,
+      "reward": 0.09765625,
+      "reward_std": 0.008384780958294868,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9765625,
+      "rewards/format_reward/std": 0.15143637359142303,
       "step": 144
     },
     {
@@ -4047,26 +4047,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1225.0,
-      "completions/max_terminated_length": 1225.0,
-      "completions/mean_length": 541.259765625,
-      "completions/mean_terminated_length": 541.259765625,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -4.140625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 1348.701171875,
+      "completions/mean_terminated_length": 959.7294921875,
+      "completions/min_length": 338.0,
+      "completions/min_terminated_length": 338.0,
       "epoch": 0.12984105663756437,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0076739424344186005,
-      "kl": 0.009979248046875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.2595208434649135,
+      "kl": 0.113037109375,
       "learning_rate": 4.2591506484465426e-06,
-      "loss": 0.0001,
-      "num_tokens": 85494732.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0059,
+      "num_tokens": 89399538.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 145
     },
     {
@@ -4075,26 +4075,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -4.203125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1797.0,
-      "completions/mean_length": 564.115234375,
-      "completions/mean_terminated_length": 559.4392700195312,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_terminated_length": 2045.0,
+      "completions/mean_length": 1366.169921875,
+      "completions/mean_terminated_length": 999.6607055664062,
+      "completions/min_length": 247.0,
+      "completions/min_terminated_length": 247.0,
       "epoch": 0.13073651220058205,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 30.466730837647507,
-      "kl": 0.0369110107421875,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.08052216200621251,
+      "kl": 0.1043701171875,
       "learning_rate": 4.248309256276283e-06,
-      "loss": 0.0211,
-      "num_tokens": 86086071.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.006,
+      "num_tokens": 90401529.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 146
     },
     {
@@ -4103,26 +4103,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1374.0,
-      "completions/max_terminated_length": 1374.0,
-      "completions/mean_length": 529.3046875,
-      "completions/mean_terminated_length": 529.3046875,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -4.359375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2045.0,
+      "completions/mean_length": 1320.82421875,
+      "completions/mean_terminated_length": 962.5364990234375,
+      "completions/min_length": 255.0,
+      "completions/min_terminated_length": 255.0,
       "epoch": 0.13163196776359973,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00698639674310776,
-      "kl": 0.00970458984375,
+      "frac_reward_zero_std": 0.78125,
+      "grad_norm": 2.41982629512539,
+      "kl": 0.103759765625,
       "learning_rate": 4.23740499805044e-06,
-      "loss": 0.0001,
-      "num_tokens": 86626435.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0062,
+      "num_tokens": 91347151.0,
+      "reward": 0.0986328125,
+      "reward_std": 0.00546875037252903,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.986328125,
+      "rewards/format_reward/std": 0.1162383034825325,
       "step": 147
     },
     {
@@ -4131,26 +4131,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1790.0,
-      "completions/max_terminated_length": 1790.0,
-      "completions/mean_length": 532.439453125,
-      "completions/mean_terminated_length": 532.439453125,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/clipped_ratio": -4.515625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2046.0,
+      "completions/mean_length": 1281.87109375,
+      "completions/mean_terminated_length": 936.7875366210938,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
       "epoch": 0.1325274233266174,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004340388783933389,
-      "kl": 0.009613037109375,
+      "frac_reward_zero_std": 0.6875,
+      "grad_norm": 1.0229642621220172,
+      "kl": 0.1658935546875,
       "learning_rate": 4.22643833128985e-06,
-      "loss": 0.0001,
-      "num_tokens": 87206116.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0131,
+      "num_tokens": 92310541.0,
+      "reward": 0.09687500447034836,
+      "reward_std": 0.009193411096930504,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.96875,
+      "rewards/format_reward/std": 0.17416280508041382,
       "step": 148
     },
     {
@@ -4159,26 +4159,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1481.0,
-      "completions/max_terminated_length": 1481.0,
-      "completions/mean_length": 561.947265625,
-      "completions/mean_terminated_length": 561.947265625,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -4.21875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 1364.658203125,
+      "completions/mean_terminated_length": 1000.4820556640625,
+      "completions/min_length": 315.0,
+      "completions/min_terminated_length": 315.0,
       "epoch": 0.1334228788896351,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032762673509562613,
-      "kl": 0.00921630859375,
+      "frac_reward_zero_std": 0.75,
+      "grad_norm": 0.11093213327135282,
+      "kl": 0.0858154296875,
       "learning_rate": 4.215409716133885e-06,
-      "loss": 0.0001,
-      "num_tokens": 87796905.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0018,
+      "num_tokens": 93312318.0,
+      "reward": 0.09824219346046448,
+      "reward_std": 0.006536140572279692,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.982421875,
+      "rewards/format_reward/std": 0.13154059648513794,
       "step": 149
     },
     {
@@ -4187,26 +4187,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1430.0,
-      "completions/max_terminated_length": 1430.0,
-      "completions/mean_length": 590.71875,
-      "completions/mean_terminated_length": 590.71875,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -4.5625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2046.0,
+      "completions/mean_length": 1338.37890625,
+      "completions/mean_terminated_length": 1027.42138671875,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.1343183344526528,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07984510566712394,
-      "kl": 0.00885009765625,
+      "frac_reward_zero_std": 0.8125,
+      "grad_norm": 0.3995251198579641,
+      "kl": 0.0902099609375,
       "learning_rate": 4.204319615321151e-06,
-      "loss": 0.0036,
-      "num_tokens": 88415945.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0086,
+      "num_tokens": 94314160.0,
+      "reward": 0.09882812947034836,
+      "reward_std": 0.004687500186264515,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 150
     },
     {
@@ -4215,26 +4215,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1422.0,
-      "completions/max_terminated_length": 1422.0,
-      "completions/mean_length": 583.884765625,
-      "completions/mean_terminated_length": 582.7808227539062,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/clipped_ratio": -4.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2040.0,
+      "completions/mean_length": 1252.2109375,
+      "completions/mean_terminated_length": 984.1775512695312,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.13521379001567047,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0037633651975202153,
-      "kl": 0.014678955078125,
+      "frac_reward_zero_std": 0.75,
+      "grad_norm": 0.11590978634626535,
+      "kl": 0.08642578125,
       "learning_rate": 4.193168494170065e-06,
-      "loss": 0.0001,
-      "num_tokens": 89032094.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0085,
+      "num_tokens": 95272492.0,
+      "reward": 0.09804687649011612,
+      "reward_std": 0.006822281051427126,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98046875,
+      "rewards/format_reward/std": 0.1385180652141571,
       "step": 151
     },
     {
@@ -4243,26 +4243,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1403.0,
-      "completions/max_terminated_length": 1403.0,
-      "completions/mean_length": 586.375,
-      "completions/mean_terminated_length": 586.375,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -5.046875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2045.0,
+      "completions/mean_length": 1240.875,
+      "completions/mean_terminated_length": 980.1757202148438,
+      "completions/min_length": 241.0,
+      "completions/min_terminated_length": 241.0,
       "epoch": 0.13610924557868817,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031194194696154568,
-      "kl": 0.0089874267578125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.06993291127200373,
+      "kl": 0.120849609375,
       "learning_rate": 4.181956820559339e-06,
-      "loss": 0.0001,
-      "num_tokens": 89625566.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0054,
+      "num_tokens": 96201068.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0026298905722796917,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 152
     },
     {
@@ -4271,26 +4271,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -5.578125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1679.0,
-      "completions/mean_length": 545.861328125,
-      "completions/mean_terminated_length": 542.9216918945312,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/max_terminated_length": 2033.0,
+      "completions/mean_length": 1163.3671875,
+      "completions/mean_terminated_length": 972.1520385742188,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
       "epoch": 0.13700470114170585,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.03997537222476599,
-      "kl": 0.0093994140625,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 0.0861319462004464,
+      "kl": 0.0950927734375,
       "learning_rate": 4.170685064908342e-06,
-      "loss": 0.0148,
-      "num_tokens": 90198135.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0035,
+      "num_tokens": 97089800.0,
+      "reward": 0.09902343899011612,
+      "reward_std": 0.00390625,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "step": 153
     },
     {
@@ -4299,26 +4299,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1516.0,
-      "completions/max_terminated_length": 1516.0,
-      "completions/mean_length": 585.26953125,
-      "completions/mean_terminated_length": 584.1995849609375,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -5.3125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 1177.39453125,
+      "completions/mean_terminated_length": 944.6583862304688,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
       "epoch": 0.13790015670472353,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005841763677871051,
-      "kl": 0.0170135498046875,
+      "frac_reward_zero_std": 0.78125,
+      "grad_norm": 0.26491805736565843,
+      "kl": 0.1044921875,
       "learning_rate": 4.159353700157365e-06,
-      "loss": 0.0002,
-      "num_tokens": 90787937.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0087,
+      "num_tokens": 97982770.0,
+      "reward": 0.09843750298023224,
+      "reward_std": 0.005754890851676464,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
       "step": 154
     },
     {
@@ -4327,26 +4327,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1345.0,
-      "completions/max_terminated_length": 1345.0,
-      "completions/mean_length": 526.51171875,
-      "completions/mean_terminated_length": 526.51171875,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -5.796875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2033.0,
+      "completions/mean_length": 1122.61328125,
+      "completions/mean_terminated_length": 958.8092041015625,
+      "completions/min_length": 274.0,
+      "completions/min_terminated_length": 274.0,
       "epoch": 0.1387956122677412,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004156360771568148,
-      "kl": 0.00970458984375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.0671134919909452,
+      "kl": 0.0772705078125,
       "learning_rate": 4.14796320174778e-06,
-      "loss": 0.0001,
-      "num_tokens": 91344391.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0031,
+      "num_tokens": 98844428.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 155
     },
     {
@@ -4355,26 +4355,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1665.0,
-      "completions/max_terminated_length": 1665.0,
-      "completions/mean_length": 545.390625,
-      "completions/mean_terminated_length": 545.390625,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -5.6875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2039.0,
+      "completions/mean_length": 1108.546875,
+      "completions/mean_terminated_length": 924.168212890625,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.1396910678307589,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0033243904947931058,
-      "kl": 0.0087432861328125,
+      "frac_reward_zero_std": 0.8125,
+      "grad_norm": 0.09175374952691938,
+      "kl": 0.098876953125,
       "learning_rate": 4.136514047602087e-06,
-      "loss": 0.0001,
-      "num_tokens": 91938655.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0086,
+      "num_tokens": 99727028.0,
+      "reward": 0.09882812947034836,
+      "reward_std": 0.004687500186264515,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 156
     },
     {
@@ -4383,26 +4383,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1236.0,
-      "completions/max_terminated_length": 1236.0,
-      "completions/mean_length": 548.173828125,
-      "completions/mean_terminated_length": 548.173828125,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/clipped_ratio": -5.84375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2020.0,
+      "completions/mean_length": 1090.578125,
+      "completions/mean_terminated_length": 928.8218994140625,
+      "completions/min_length": 277.0,
+      "completions/min_terminated_length": 277.0,
       "epoch": 0.14058652339377659,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0034809305022384003,
-      "kl": 0.0088653564453125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.057742259396112945,
+      "kl": 0.0733642578125,
       "learning_rate": 4.1250067181038635e-06,
-      "loss": 0.0001,
-      "num_tokens": 92537080.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0023,
+      "num_tokens": 100603164.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 157
     },
     {
@@ -4411,26 +4411,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1552.0,
-      "completions/max_terminated_length": 1552.0,
-      "completions/mean_length": 574.640625,
-      "completions/mean_terminated_length": 574.1820068359375,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -5.640625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2011.0,
+      "completions/mean_length": 1118.96875,
+      "completions/mean_terminated_length": 928.79052734375,
+      "completions/min_length": 245.0,
+      "completions/min_terminated_length": 245.0,
       "epoch": 0.14148197895679426,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.8063755669722333,
-      "kl": 0.0182952880859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009952836592016451,
+      "kl": 0.0738525390625,
       "learning_rate": 4.113441696077608e-06,
-      "loss": 0.0002,
-      "num_tokens": 93164864.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 101509644.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 158
     },
     {
@@ -4439,26 +4439,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.046875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1657.0,
-      "completions/mean_length": 575.2890625,
-      "completions/mean_terminated_length": 572.4070434570312,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/max_terminated_length": 2047.0,
+      "completions/mean_length": 1012.00390625,
+      "completions/mean_terminated_length": 871.8802490234375,
+      "completions/min_length": 216.0,
+      "completions/min_terminated_length": 216.0,
       "epoch": 0.14237743451981194,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.050040387791137446,
-      "kl": 0.00885009765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.14858531404881065,
+      "kl": 0.076904296875,
       "learning_rate": 4.101819466768484e-06,
-      "loss": 0.0164,
-      "num_tokens": 93760132.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0044,
+      "num_tokens": 102328510.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 159
     },
     {
@@ -4467,26 +4467,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1469.0,
-      "completions/max_terminated_length": 1469.0,
-      "completions/mean_length": 585.19921875,
-      "completions/mean_terminated_length": 585.19921875,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -6.40625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1991.0,
+      "completions/mean_length": 921.74609375,
+      "completions/mean_terminated_length": 831.4556274414062,
+      "completions/min_length": 223.0,
+      "completions/min_terminated_length": 223.0,
       "epoch": 0.14327289008282965,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0036055384423648723,
-      "kl": 0.0088653564453125,
+      "frac_reward_zero_std": 0.75,
+      "grad_norm": 0.1361238385560723,
+      "kl": 0.11376953125,
       "learning_rate": 4.0901405178219535e-06,
-      "loss": 0.0001,
-      "num_tokens": 94360650.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0019,
+      "num_tokens": 103101340.0,
+      "reward": 0.09843750298023224,
+      "reward_std": 0.0062500000931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.984375,
+      "rewards/format_reward/std": 0.12414088100194931,
       "step": 160
     },
     {
@@ -4495,26 +4495,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1168.0,
-      "completions/max_terminated_length": 1168.0,
-      "completions/mean_length": 546.8515625,
-      "completions/mean_terminated_length": 546.8515625,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/clipped_ratio": -6.4375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1997.0,
+      "completions/mean_length": 858.59765625,
+      "completions/mean_terminated_length": 768.6428833007812,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.14416834564584732,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004011318075529169,
-      "kl": 0.0095977783203125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.056481508031488666,
+      "kl": 0.07568359375,
       "learning_rate": 4.078405339263326e-06,
-      "loss": 0.0001,
-      "num_tokens": 94930350.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0145,
+      "num_tokens": 103830654.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 161
     },
     {
@@ -4523,26 +4523,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1495.0,
-      "completions/max_terminated_length": 1495.0,
-      "completions/mean_length": 628.458984375,
-      "completions/mean_terminated_length": 628.458984375,
-      "completions/min_length": 248.0,
-      "completions/min_terminated_length": 248.0,
+      "completions/clipped_ratio": -6.4375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1971.0,
+      "completions/mean_length": 945.384765625,
+      "completions/mean_terminated_length": 861.9937133789062,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.145063801208865,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005474106861111618,
-      "kl": 0.00885009765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.05837129013687458,
+      "kl": 0.06982421875,
       "learning_rate": 4.06661442347719e-06,
-      "loss": 0.0001,
-      "num_tokens": 95586297.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0009,
+      "num_tokens": 104648867.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 162
     },
     {
@@ -4551,26 +4551,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1564.0,
-      "completions/max_terminated_length": 1564.0,
-      "completions/mean_length": 605.478515625,
-      "completions/mean_terminated_length": 605.478515625,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -6.734375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2028.0,
+      "completions/mean_length": 852.2265625,
+      "completions/mean_terminated_length": 811.1596069335938,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.1459592567718827,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006117440922761299,
-      "kl": 0.0095062255859375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.079316699909495,
+      "kl": 0.0712890625,
       "learning_rate": 4.054768265186758e-06,
-      "loss": 0.0001,
-      "num_tokens": 96205422.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0102,
+      "num_tokens": 105394327.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 163
     },
     {
@@ -4579,26 +4579,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1669.0,
-      "completions/max_terminated_length": 1669.0,
-      "completions/mean_length": 614.33203125,
-      "completions/mean_terminated_length": 614.33203125,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -6.6875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2008.0,
+      "completions/mean_length": 787.642578125,
+      "completions/mean_terminated_length": 736.4085083007812,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.14685471233490038,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007188095412002734,
-      "kl": 0.010223388671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.034540728831484666,
+      "kl": 0.085693359375,
       "learning_rate": 4.0428673614331036e-06,
-      "loss": 0.0001,
-      "num_tokens": 96808328.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0013,
+      "num_tokens": 106085968.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 164
     },
     {
@@ -4607,26 +4607,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1823.0,
-      "completions/max_terminated_length": 1823.0,
-      "completions/mean_length": 652.791015625,
-      "completions/mean_terminated_length": 652.791015625,
+      "completions/clipped_ratio": -6.59375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2048.0,
+      "completions/mean_length": 830.478515625,
+      "completions/mean_terminated_length": 765.3436279296875,
       "completions/min_length": 210.0,
       "completions/min_terminated_length": 210.0,
       "epoch": 0.14775016789791806,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0068502617917810356,
-      "kl": 0.0096588134765625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.06310920256066563,
+      "kl": 0.074951171875,
       "learning_rate": 4.030912211554316e-06,
-      "loss": 0.0001,
-      "num_tokens": 97478221.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0152,
+      "num_tokens": 106846837.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 165
     },
     {
@@ -4635,26 +4635,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.640625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1845.0,
-      "completions/mean_length": 663.80078125,
-      "completions/mean_terminated_length": 661.0919799804688,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/max_terminated_length": 1986.0,
+      "completions/mean_length": 834.1953125,
+      "completions/mean_terminated_length": 777.104248046875,
+      "completions/min_length": 221.0,
+      "completions/min_terminated_length": 221.0,
       "epoch": 0.14864562346093574,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09300391631800778,
-      "kl": 0.0095672607421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.011733653254552742,
+      "kl": 0.0711669921875,
       "learning_rate": 4.018903317164539e-06,
-      "loss": 0.0123,
-      "num_tokens": 98113095.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0007,
+      "num_tokens": 107568953.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 166
     },
     {
@@ -4663,26 +4663,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.796875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1403.0,
-      "completions/mean_length": 624.248046875,
-      "completions/mean_terminated_length": 621.4618530273438,
-      "completions/min_length": 283.0,
-      "completions/min_terminated_length": 283.0,
+      "completions/max_terminated_length": 1966.0,
+      "completions/mean_length": 774.447265625,
+      "completions/mean_terminated_length": 741.2685546875,
+      "completions/min_length": 256.0,
+      "completions/min_terminated_length": 256.0,
       "epoch": 0.14954107902395344,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.11604895692678942,
-      "kl": 0.009368896484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03409817940123074,
+      "kl": 0.069580078125,
       "learning_rate": 4.006841182132932e-06,
-      "loss": 0.0143,
-      "num_tokens": 98748934.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": -0.0017,
+      "num_tokens": 108281694.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 167
     },
     {
@@ -4691,26 +4691,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1383.0,
-      "completions/max_terminated_length": 1383.0,
-      "completions/mean_length": 629.244140625,
-      "completions/mean_terminated_length": 628.5205688476562,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/clipped_ratio": -6.828125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1971.0,
+      "completions/mean_length": 756.509765625,
+      "completions/mean_terminated_length": 728.1536865234375,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
       "epoch": 0.15043653458697112,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.08576197401573622,
-      "kl": 0.0146331787109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.053464405396001835,
+      "kl": 0.073486328125,
       "learning_rate": 3.9947263125625195e-06,
-      "loss": 0.0001,
-      "num_tokens": 99351603.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0007,
+      "num_tokens": 108949523.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 168
     },
     {
@@ -4719,26 +4719,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1867.0,
-      "completions/max_terminated_length": 1867.0,
-      "completions/mean_length": 656.34375,
-      "completions/mean_terminated_length": 656.34375,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.671875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2010.0,
+      "completions/mean_length": 788.6875,
+      "completions/mean_terminated_length": 734.826904296875,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.1513319901499888,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052099211200067566,
-      "kl": 0.0095672607421875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07949072343589329,
+      "kl": 0.0740966796875,
       "learning_rate": 3.982559216768967e-06,
-      "loss": 0.008,
-      "num_tokens": 100005155.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0034,
+      "num_tokens": 109670835.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 169
     },
     {
@@ -4747,26 +4747,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1222.0,
-      "completions/max_terminated_length": 1222.0,
-      "completions/mean_length": 606.388671875,
-      "completions/mean_terminated_length": 606.388671875,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/clipped_ratio": -6.796875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2000.0,
+      "completions/mean_length": 698.345703125,
+      "completions/mean_terminated_length": 663.1843872070312,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.1522274457130065,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005600844822173869,
-      "kl": 0.0096893310546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.045217268440116885,
+      "kl": 0.0711669921875,
       "learning_rate": 3.970340405259245e-06,
-      "loss": 0.0001,
-      "num_tokens": 100608298.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0016,
+      "num_tokens": 110321060.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 170
     },
     {
@@ -4775,26 +4775,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1278.0,
-      "completions/max_terminated_length": 1278.0,
-      "completions/mean_length": 584.5703125,
-      "completions/mean_terminated_length": 584.5703125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1813.0,
+      "completions/mean_length": 644.583984375,
+      "completions/mean_terminated_length": 625.1307373046875,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.15312290127602418,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004428337264446385,
-      "kl": 0.009765625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05776747187731656,
+      "kl": 0.0753173828125,
       "learning_rate": 3.958070390710214e-06,
-      "loss": 0.0001,
-      "num_tokens": 101192766.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0006,
+      "num_tokens": 110936255.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 171
     },
     {
@@ -4803,26 +4803,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1509.0,
-      "completions/max_terminated_length": 1509.0,
-      "completions/mean_length": 620.2578125,
-      "completions/mean_terminated_length": 620.2578125,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2021.0,
+      "completions/mean_length": 692.998046875,
+      "completions/mean_terminated_length": 682.3287353515625,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
       "epoch": 0.15401835683904186,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004033698260223417,
-      "kl": 0.0086822509765625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.056419783846101156,
+      "kl": 0.072021484375,
       "learning_rate": 3.945749687947109e-06,
-      "loss": 0.0001,
-      "num_tokens": 101809218.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0005,
+      "num_tokens": 111589950.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 172
     },
     {
@@ -4831,26 +4831,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1283.0,
-      "completions/max_terminated_length": 1283.0,
-      "completions/mean_length": 572.96875,
-      "completions/mean_terminated_length": 572.2837524414062,
-      "completions/min_length": 218.0,
-      "completions/min_terminated_length": 218.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1706.0,
+      "completions/mean_length": 645.02734375,
+      "completions/mean_terminated_length": 631.1913452148438,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.15491381240205954,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2.1602393924211474,
-      "kl": 0.0108795166015625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.008793484041353715,
+      "kl": 0.0721435546875,
       "learning_rate": 3.933378813921942e-06,
-      "loss": 0.0032,
-      "num_tokens": 102370978.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 112188604.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 173
     },
     {
@@ -4859,20 +4859,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.859375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1435.0,
-      "completions/mean_length": 636.423828125,
-      "completions/mean_terminated_length": 633.6614379882812,
-      "completions/min_length": 205.0,
-      "completions/min_terminated_length": 205.0,
+      "completions/max_terminated_length": 1917.0,
+      "completions/mean_length": 708.283203125,
+      "completions/mean_terminated_length": 684.3120727539062,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.15580926796507724,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.057483002571070425,
-      "kl": 0.0085906982421875,
+      "grad_norm": 0.041267501649098925,
+      "kl": 0.0745849609375,
       "learning_rate": 3.920958287691811e-06,
-      "loss": 0.0125,
-      "num_tokens": 103003291.0,
+      "loss": -0.0033,
+      "num_tokens": 112857709.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -4886,27 +4886,27 @@
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1548.0,
-      "completions/max_terminated_length": 1548.0,
-      "completions/mean_length": 591.62109375,
-      "completions/mean_terminated_length": 591.62109375,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1992.0,
+      "completions/mean_length": 678.421875,
+      "completions/mean_terminated_length": 664.9152221679688,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.15670472352809492,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0036255201179309767,
-      "kl": 0.0087127685546875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.06445929796633103,
+      "kl": 0.0740966796875,
       "learning_rate": 3.908488630397121e-06,
-      "loss": 0.0001,
-      "num_tokens": 103616073.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0031,
+      "num_tokens": 113514933.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 175
     },
     {
@@ -4915,20 +4915,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.921875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1623.0,
-      "completions/mean_length": 577.505859375,
-      "completions/mean_terminated_length": 572.5725708007812,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_terminated_length": 1982.0,
+      "completions/mean_length": 672.736328125,
+      "completions/mean_terminated_length": 659.173583984375,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.1576001790911126,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 47744.02884793233,
-      "kl": 1600.0066375732422,
+      "grad_norm": 0.05116548843844329,
+      "kl": 0.0716552734375,
       "learning_rate": 3.8959703652397175e-06,
-      "loss": 16.0277,
-      "num_tokens": 104218476.0,
+      "loss": -0.0012,
+      "num_tokens": 114166094.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -4943,26 +4943,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.90625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1608.0,
-      "completions/mean_length": 605.826171875,
-      "completions/mean_terminated_length": 597.326171875,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/max_terminated_length": 2028.0,
+      "completions/mean_length": 710.96875,
+      "completions/mean_terminated_length": 695.1541748046875,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.1584956346541303,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06412456396717171,
-      "kl": 0.0096893310546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006938377171807984,
+      "kl": 0.092529296875,
       "learning_rate": 3.883404017460935e-06,
-      "loss": 0.0128,
-      "num_tokens": 104827891.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0012597277527675033,
+      "loss": 0.0009,
+      "num_tokens": 114829342.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 177
     },
     {
@@ -4971,26 +4971,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1319.0,
-      "completions/max_terminated_length": 1319.0,
-      "completions/mean_length": 580.29296875,
-      "completions/mean_terminated_length": 580.29296875,
-      "completions/min_length": 263.0,
-      "completions/min_terminated_length": 263.0,
+      "completions/clipped_ratio": -6.875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1971.0,
+      "completions/mean_length": 691.232421875,
+      "completions/mean_terminated_length": 669.6964721679688,
+      "completions/min_length": 251.0,
+      "completions/min_terminated_length": 251.0,
       "epoch": 0.15939109021714798,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004861320279381554,
-      "kl": 0.00921630859375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.0818158362296435,
+      "kl": 0.0672607421875,
       "learning_rate": 3.870790114319559e-06,
-      "loss": 0.0001,
-      "num_tokens": 105415273.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0031,
+      "num_tokens": 115473525.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 178
     },
     {
@@ -4999,26 +4999,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1593.0,
-      "completions/max_terminated_length": 1593.0,
-      "completions/mean_length": 550.037109375,
-      "completions/mean_terminated_length": 550.037109375,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1721.0,
+      "completions/mean_length": 639.69140625,
+      "completions/mean_terminated_length": 620.1702880859375,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.16028654578016566,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.061738880104298316,
-      "kl": 0.009918212890625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007890322312255954,
+      "kl": 0.07080078125,
       "learning_rate": 3.858129185069701e-06,
-      "loss": -0.0,
-      "num_tokens": 105979836.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 116083991.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 179
     },
     {
@@ -5027,26 +5027,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1259.0,
-      "completions/max_terminated_length": 1259.0,
-      "completions/mean_length": 570.638671875,
-      "completions/mean_terminated_length": 570.638671875,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1891.0,
+      "completions/mean_length": 689.5546875,
+      "completions/mean_terminated_length": 679.472412109375,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.16118200134318333,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004637633663432918,
-      "kl": 0.009307861328125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07882687764220778,
+      "kl": 0.11083984375,
       "learning_rate": 3.845421760938597e-06,
-      "loss": 0.0001,
-      "num_tokens": 106596611.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0029,
+      "num_tokens": 116761651.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 180
     },
     {
@@ -5055,20 +5055,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1052.0,
-      "completions/mean_length": 532.470703125,
-      "completions/mean_terminated_length": 529.5048828125,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/max_terminated_length": 2033.0,
+      "completions/mean_length": 635.724609375,
+      "completions/mean_terminated_length": 627.4008178710938,
+      "completions/min_length": 233.0,
+      "completions/min_terminated_length": 233.0,
       "epoch": 0.16207745690620104,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08330239155652147,
-      "kl": 0.0096435546875,
+      "grad_norm": 0.0725677041475304,
+      "kl": 0.070556640625,
       "learning_rate": 3.832668375104312e-06,
-      "loss": 0.016,
-      "num_tokens": 107165668.0,
+      "loss": -0.0004,
+      "num_tokens": 117383574.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -5083,20 +5083,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1999.0,
-      "completions/max_terminated_length": 1999.0,
-      "completions/mean_length": 569.216796875,
-      "completions/mean_terminated_length": 569.216796875,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2003.0,
+      "completions/mean_length": 713.12109375,
+      "completions/mean_terminated_length": 699.9566040039062,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
       "epoch": 0.16297291246921872,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005295883162438518,
-      "kl": 0.0092315673828125,
+      "grad_norm": 0.014271174770978515,
+      "kl": 0.0697021484375,
       "learning_rate": 3.8198695626733725e-06,
-      "loss": 0.0001,
-      "num_tokens": 107772163.0,
+      "loss": 0.0007,
+      "num_tokens": 118063748.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5111,26 +5111,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1655.0,
-      "completions/max_terminated_length": 1655.0,
-      "completions/mean_length": 589.267578125,
-      "completions/mean_terminated_length": 589.267578125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -6.84375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2040.0,
+      "completions/mean_length": 713.150390625,
+      "completions/mean_terminated_length": 686.5597534179688,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
       "epoch": 0.1638683680322364,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005391282000726491,
-      "kl": 0.0091400146484375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.08507378122083202,
+      "kl": 0.105224609375,
       "learning_rate": 3.8070258606583156e-06,
-      "loss": 0.0001,
-      "num_tokens": 108372076.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0037,
+      "num_tokens": 118727089.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 183
     },
     {
@@ -5139,20 +5139,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1423.0,
-      "completions/max_terminated_length": 1423.0,
-      "completions/mean_length": 595.279296875,
-      "completions/mean_terminated_length": 595.279296875,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1989.0,
+      "completions/mean_length": 722.576171875,
+      "completions/mean_terminated_length": 712.1397705078125,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.16476382359525407,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0952696472933219,
-      "kl": 0.009857177734375,
+      "grad_norm": 0.07094156347361062,
+      "kl": 0.0694580078125,
       "learning_rate": 3.7941378079551544e-06,
-      "loss": 0.0028,
-      "num_tokens": 108992747.0,
+      "loss": 0.0022,
+      "num_tokens": 119412936.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -5167,26 +5167,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1404.0,
-      "completions/max_terminated_length": 1404.0,
-      "completions/mean_length": 567.849609375,
-      "completions/mean_terminated_length": 567.849609375,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 696.935546875,
+      "completions/mean_terminated_length": 680.9150390625,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.16565927915827178,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.058420288604048326,
-      "kl": 0.00994873046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007349496040761767,
+      "kl": 0.0682373046875,
       "learning_rate": 3.7812059453207677e-06,
-      "loss": 0.0013,
-      "num_tokens": 109598686.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 120084967.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 185
     },
     {
@@ -5195,26 +5195,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1777.0,
-      "completions/max_terminated_length": 1777.0,
-      "completions/mean_length": 547.923828125,
-      "completions/mean_terminated_length": 547.923828125,
-      "completions/min_length": 152.0,
-      "completions/min_terminated_length": 152.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1895.0,
+      "completions/mean_length": 676.48828125,
+      "completions/mean_terminated_length": 665.68896484375,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
       "epoch": 0.16655473472128945,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005346867400011268,
-      "kl": 0.009521484375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07706499183432042,
+      "kl": 0.067626953125,
       "learning_rate": 3.768230815350213e-06,
-      "loss": 0.0001,
-      "num_tokens": 110194007.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0007,
+      "num_tokens": 120746113.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 186
     },
     {
@@ -5223,26 +5223,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1268.0,
-      "completions/max_terminated_length": 1268.0,
-      "completions/mean_length": 544.93359375,
-      "completions/mean_terminated_length": 544.93359375,
-      "completions/min_length": 106.0,
-      "completions/min_terminated_length": 106.0,
+      "completions/clipped_ratio": -6.875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1893.0,
+      "completions/mean_length": 636.73046875,
+      "completions/mean_terminated_length": 616.3154907226562,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.16745019028430713,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008310871893682886,
-      "kl": 0.0106201171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04911030768903344,
+      "kl": 0.099853515625,
       "learning_rate": 3.7552129624539557e-06,
-      "loss": 0.0001,
-      "num_tokens": 110773573.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.002,
+      "num_tokens": 121372679.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 187
     },
     {
@@ -5251,26 +5251,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1137.0,
-      "completions/max_terminated_length": 1137.0,
-      "completions/mean_length": 530.16796875,
-      "completions/mean_terminated_length": 529.2857055664062,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1942.0,
+      "completions/mean_length": 626.787109375,
+      "completions/mean_terminated_length": 615.596435546875,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.16834564584732484,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 158.14659789913986,
-      "kl": 22.0072021484375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.06941023633883373,
+      "kl": 0.0716552734375,
       "learning_rate": 3.7421529328350316e-06,
-      "loss": 0.2275,
-      "num_tokens": 111347771.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0033,
+      "num_tokens": 121996346.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 188
     },
     {
@@ -5279,26 +5279,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1653.0,
-      "completions/max_terminated_length": 1653.0,
-      "completions/mean_length": 594.1875,
-      "completions/mean_terminated_length": 594.1875,
-      "completions/min_length": 155.0,
-      "completions/min_terminated_length": 155.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1995.0,
+      "completions/mean_length": 690.220703125,
+      "completions/mean_terminated_length": 676.8303833007812,
+      "completions/min_length": 127.0,
+      "completions/min_terminated_length": 127.0,
       "epoch": 0.16924110141034251,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008808132081770864,
-      "kl": 0.0102081298828125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07568288588640923,
+      "kl": 0.066162109375,
       "learning_rate": 3.7290512744661274e-06,
-      "loss": 0.0001,
-      "num_tokens": 111973515.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0084,
+      "num_tokens": 122671259.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 189
     },
     {
@@ -5307,26 +5307,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1485.0,
-      "completions/max_terminated_length": 1485.0,
-      "completions/mean_length": 583.603515625,
-      "completions/mean_terminated_length": 583.603515625,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1983.0,
+      "completions/mean_length": 697.283203125,
+      "completions/mean_terminated_length": 681.266845703125,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.1701365569733602,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010967235624627656,
-      "kl": 0.01129150390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.049509900781209426,
+      "kl": 0.0650634765625,
       "learning_rate": 3.715908537066589e-06,
-      "loss": 0.0001,
-      "num_tokens": 112570704.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.001,
+      "num_tokens": 123326652.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 190
     },
     {
@@ -5335,20 +5335,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1795.0,
-      "completions/max_terminated_length": 1795.0,
-      "completions/mean_length": 570.9375,
-      "completions/mean_terminated_length": 570.9375,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1783.0,
+      "completions/mean_length": 626.283203125,
+      "completions/mean_terminated_length": 615.0885620117188,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.17103201253637787,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.013565646568363069,
-      "kl": 0.0124664306640625,
+      "grad_norm": 0.007656796040386732,
+      "kl": 0.066650390625,
       "learning_rate": 3.7027252720793538e-06,
-      "loss": 0.0001,
-      "num_tokens": 113166720.0,
+      "loss": 0.0007,
+      "num_tokens": 123951005.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5363,26 +5363,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1883.0,
-      "completions/max_terminated_length": 1883.0,
-      "completions/mean_length": 581.537109375,
-      "completions/mean_terminated_length": 581.537109375,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 665.228515625,
+      "completions/mean_terminated_length": 648.83203125,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.17192746809939558,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014691973713636838,
-      "kl": 0.013153076171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05792298135309963,
+      "kl": 0.0677490234375,
       "learning_rate": 3.689502032647817e-06,
-      "loss": 0.0001,
-      "num_tokens": 113769571.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0015,
+      "num_tokens": 124596706.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 192
     },
     {
@@ -5391,26 +5391,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1793.0,
-      "completions/max_terminated_length": 1793.0,
-      "completions/mean_length": 550.32421875,
-      "completions/mean_terminated_length": 550.32421875,
-      "completions/min_length": 104.0,
-      "completions/min_terminated_length": 104.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1688.0,
+      "completions/mean_length": 620.7578125,
+      "completions/mean_terminated_length": 609.5196533203125,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.17282292366241325,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06606192521106205,
-      "kl": 0.0179595947265625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007164808257495399,
+      "kl": 0.06610107421875,
       "learning_rate": 3.6762393735926245e-06,
-      "loss": 0.0033,
-      "num_tokens": 114331817.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 125195014.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 193
     },
     {
@@ -5419,26 +5419,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2043.0,
-      "completions/mean_length": 587.3203125,
-      "completions/mean_terminated_length": 584.4618530273438,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/max_terminated_length": 1790.0,
+      "completions/mean_length": 638.5703125,
+      "completions/mean_terminated_length": 627.472412109375,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.17371837922543093,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0182360788370408,
-      "kl": 0.0151214599609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.060704628699271526,
+      "kl": 0.06787109375,
       "learning_rate": 3.6629378513883852e-06,
-      "loss": 0.0002,
-      "num_tokens": 114940445.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0009,
+      "num_tokens": 125829882.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 194
     },
     {
@@ -5447,20 +5447,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1629.0,
-      "completions/max_terminated_length": 1629.0,
-      "completions/mean_length": 559.173828125,
-      "completions/mean_terminated_length": 559.173828125,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1977.0,
+      "completions/mean_length": 624.65625,
+      "completions/mean_terminated_length": 619.0745239257812,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.17461383478844864,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.015121651645506792,
-      "kl": 0.01385498046875,
+      "grad_norm": 0.007082540003861009,
+      "kl": 0.066162109375,
       "learning_rate": 3.6495980241403307e-06,
-      "loss": 0.0001,
-      "num_tokens": 115504902.0,
+      "loss": 0.0007,
+      "num_tokens": 126427866.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5477,24 +5477,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1650.0,
-      "completions/mean_length": 644.818359375,
-      "completions/mean_terminated_length": 642.0723876953125,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/max_terminated_length": 2033.0,
+      "completions/mean_length": 701.626953125,
+      "completions/mean_terminated_length": 698.9921875,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.1755092903514663,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10159970928736292,
-      "kl": 0.012451171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006717488479532777,
+      "kl": 0.0650634765625,
       "learning_rate": 3.636220451560896e-06,
-      "loss": 0.0108,
-      "num_tokens": 116142665.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0007,
+      "num_tokens": 127094715.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 196
     },
     {
@@ -5503,20 +5503,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1878.0,
-      "completions/mean_length": 619.048828125,
-      "completions/mean_terminated_length": 613.4451293945312,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/max_terminated_length": 1944.0,
+      "completions/mean_length": 662.830078125,
+      "completions/mean_terminated_length": 651.9232177734375,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
       "epoch": 0.176404745914484,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.0956666312311249,
-      "kl": 0.0129852294921875,
+      "grad_norm": 0.08069331728310686,
+      "kl": 0.0665283203125,
       "learning_rate": 3.622805694946235e-06,
-      "loss": 0.011,
-      "num_tokens": 116783586.0,
+      "loss": -0.0005,
+      "num_tokens": 127758052.0,
       "reward": 0.099609375,
       "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
@@ -5531,26 +5531,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1488.0,
-      "completions/max_terminated_length": 1488.0,
-      "completions/mean_length": 592.234375,
-      "completions/mean_terminated_length": 592.234375,
-      "completions/min_length": 284.0,
-      "completions/min_terminated_length": 284.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1890.0,
+      "completions/mean_length": 676.98828125,
+      "completions/mean_terminated_length": 671.61181640625,
+      "completions/min_length": 250.0,
+      "completions/min_terminated_length": 250.0,
       "epoch": 0.17730020147750167,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011625600847511287,
-      "kl": 0.0123138427734375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07236099664779054,
+      "kl": 0.068359375,
       "learning_rate": 3.609354317152667e-06,
-      "loss": 0.0001,
-      "num_tokens": 117388506.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0019,
+      "num_tokens": 128406366.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 198
     },
     {
@@ -5559,26 +5559,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1462.0,
-      "completions/max_terminated_length": 1462.0,
-      "completions/mean_length": 586.54296875,
-      "completions/mean_terminated_length": 586.54296875,
-      "completions/min_length": 231.0,
-      "completions/min_terminated_length": 231.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1925.0,
+      "completions/mean_length": 668.119140625,
+      "completions/mean_terminated_length": 662.7078857421875,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.17819565704051937,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010127412065854468,
-      "kl": 0.011474609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.040604973329934714,
+      "kl": 0.0650634765625,
       "learning_rate": 3.595866882573063e-06,
-      "loss": 0.0001,
-      "num_tokens": 117986144.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0003,
+      "num_tokens": 129045771.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 199
     },
     {
@@ -5587,26 +5587,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1648.0,
-      "completions/max_terminated_length": 1648.0,
-      "completions/mean_length": 549.142578125,
-      "completions/mean_terminated_length": 549.142578125,
-      "completions/min_length": 118.0,
-      "completions/min_terminated_length": 118.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1728.0,
+      "completions/mean_length": 622.59765625,
+      "completions/mean_terminated_length": 602.839599609375,
+      "completions/min_length": 122.0,
+      "completions/min_terminated_length": 122.0,
       "epoch": 0.17909111260353705,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009383976587188618,
-      "kl": 0.0116729736328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03790785409368292,
+      "kl": 0.078369140625,
       "learning_rate": 3.5823439571131675e-06,
-      "loss": 0.0001,
-      "num_tokens": 118578153.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0017,
+      "num_tokens": 129675389.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 200
     },
     {
@@ -5615,20 +5615,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1422.0,
-      "completions/max_terminated_length": 1422.0,
-      "completions/mean_length": 582.240234375,
-      "completions/mean_terminated_length": 582.240234375,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1954.0,
+      "completions/mean_length": 647.595703125,
+      "completions/mean_terminated_length": 642.1039428710938,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.17998656816655473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005973744897078448,
-      "kl": 0.0115814208984375,
+      "grad_norm": 0.009018080934348364,
+      "kl": 0.067138671875,
       "learning_rate": 3.5687861081678477e-06,
-      "loss": 0.0001,
-      "num_tokens": 119173828.0,
+      "loss": 0.0007,
+      "num_tokens": 130304526.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5643,26 +5643,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1376.0,
-      "completions/max_terminated_length": 1376.0,
-      "completions/mean_length": 579.224609375,
-      "completions/mean_terminated_length": 579.224609375,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1711.0,
+      "completions/mean_length": 616.826171875,
+      "completions/mean_terminated_length": 608.3909912109375,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.18088202372957243,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004596046170425412,
-      "kl": 0.010162353515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04772213814585424,
+      "kl": 0.068115234375,
       "learning_rate": 3.555193904597291e-06,
-      "loss": 0.0001,
-      "num_tokens": 119778087.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0004,
+      "num_tokens": 130928037.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 202
     },
     {
@@ -5671,20 +5671,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1391.0,
-      "completions/max_terminated_length": 1391.0,
-      "completions/mean_length": 574.671875,
-      "completions/mean_terminated_length": 574.671875,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
+      "completions/clipped_ratio": -6.875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1809.0,
+      "completions/mean_length": 642.734375,
+      "completions/mean_terminated_length": 620.4285888671875,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.1817774792925901,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004380723205360899,
-      "kl": 0.009918212890625,
+      "grad_norm": 0.0082751790561264,
+      "kl": 0.065185546875,
       "learning_rate": 3.541567916703138e-06,
-      "loss": 0.0001,
-      "num_tokens": 120354047.0,
+      "loss": 0.0007,
+      "num_tokens": 131538845.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5699,20 +5699,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1886.0,
-      "completions/max_terminated_length": 1886.0,
-      "completions/mean_length": 626.44140625,
-      "completions/mean_terminated_length": 625.624267578125,
-      "completions/min_length": 121.0,
-      "completions/min_terminated_length": 121.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1929.0,
+      "completions/mean_length": 691.69921875,
+      "completions/mean_terminated_length": 678.323486328125,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.1826729348556078,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 46.46835422177109,
-      "kl": 1.1946563720703125,
+      "grad_norm": 0.06259185264923466,
+      "kl": 0.06427001953125,
       "learning_rate": 3.5279087162045517e-06,
-      "loss": 0.019,
-      "num_tokens": 120988257.0,
+      "loss": 0.009,
+      "num_tokens": 132206467.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -5727,26 +5727,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1310.0,
-      "completions/max_terminated_length": 1310.0,
-      "completions/mean_length": 588.21484375,
-      "completions/mean_terminated_length": 588.21484375,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2009.0,
+      "completions/mean_length": 672.505859375,
+      "completions/mean_terminated_length": 664.3988647460938,
+      "completions/min_length": 245.0,
+      "completions/min_terminated_length": 245.0,
       "epoch": 0.18356839041862547,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006442269227273499,
-      "kl": 0.009613037109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07151737136940692,
+      "kl": 0.0640869140625,
       "learning_rate": 3.5142168762142265e-06,
-      "loss": 0.0001,
-      "num_tokens": 121626655.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0023,
+      "num_tokens": 132888022.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 205
     },
     {
@@ -5755,26 +5755,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1938.0,
-      "completions/max_terminated_length": 1938.0,
-      "completions/mean_length": 617.9921875,
-      "completions/mean_terminated_length": 617.9921875,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/clipped_ratio": -6.859375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1765.0,
+      "completions/mean_length": 700.201171875,
+      "completions/mean_terminated_length": 676.08544921875,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.18446384598164317,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007299388832908803,
-      "kl": 0.0101470947265625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.08013199443660099,
+      "kl": 0.067626953125,
       "learning_rate": 3.500492971214347e-06,
       "loss": 0.0001,
-      "num_tokens": 122254827.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "num_tokens": 133558285.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 206
     },
     {
@@ -5783,26 +5783,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1213.0,
-      "completions/mean_length": 549.857421875,
-      "completions/mean_terminated_length": 546.9256591796875,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1677.0,
+      "completions/max_terminated_length": 1677.0,
+      "completions/mean_length": 591.958984375,
+      "completions/mean_terminated_length": 591.958984375,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.18535930154466085,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007711548389239973,
-      "kl": 0.0094757080078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07996600764674648,
+      "kl": 0.06640625,
       "learning_rate": 3.48673757703248e-06,
-      "loss": 0.0001,
-      "num_tokens": 122837138.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0019,
+      "num_tokens": 134162152.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 207
     },
     {
@@ -5811,20 +5811,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1328.0,
-      "completions/max_terminated_length": 1328.0,
-      "completions/mean_length": 559.216796875,
-      "completions/mean_terminated_length": 559.216796875,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1960.0,
+      "completions/mean_length": 624.052734375,
+      "completions/mean_terminated_length": 618.4686889648438,
+      "completions/min_length": 250.0,
+      "completions/min_terminated_length": 250.0,
       "epoch": 0.18625475710767853,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007709733879977267,
-      "kl": 0.0093841552734375,
+      "grad_norm": 0.010916793033113123,
+      "kl": 0.065185546875,
       "learning_rate": 3.472951270817418e-06,
-      "loss": 0.0001,
-      "num_tokens": 123437713.0,
+      "loss": 0.0007,
+      "num_tokens": 134795923.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5839,26 +5839,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1805.0,
-      "completions/mean_length": 551.783203125,
-      "completions/mean_terminated_length": 548.8551635742188,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/max_terminated_length": 2023.0,
+      "completions/mean_length": 617.498046875,
+      "completions/mean_terminated_length": 606.2342529296875,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.1871502126706962,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006901104494651673,
-      "kl": 0.0103912353515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05456360479658491,
+      "kl": 0.0648193359375,
       "learning_rate": 3.4591346310149578e-06,
-      "loss": 0.0001,
-      "num_tokens": 124022594.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0019,
+      "num_tokens": 135414450.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 209
     },
     {
@@ -5867,26 +5867,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1738.0,
-      "completions/mean_length": 577.857421875,
-      "completions/mean_terminated_length": 574.9804077148438,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_terminated_length": 2038.0,
+      "completions/mean_length": 662.5078125,
+      "completions/mean_terminated_length": 654.3418579101562,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.1880456682337139,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005911630324218355,
-      "kl": 0.0094451904296875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.10789941880075726,
+      "kl": 0.06365966796875,
       "learning_rate": 3.445288237343632e-06,
-      "loss": 0.0001,
-      "num_tokens": 124651737.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0034,
+      "num_tokens": 136086934.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 210
     },
     {
@@ -5895,26 +5895,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1783.0,
-      "completions/mean_length": 631.33203125,
-      "completions/mean_terminated_length": 628.5596923828125,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_terminated_length": 2008.0,
+      "completions/mean_length": 717.3984375,
+      "completions/mean_terminated_length": 709.5560302734375,
+      "completions/min_length": 265.0,
+      "completions/min_terminated_length": 265.0,
       "epoch": 0.18894112379673159,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08768246283609199,
-      "kl": 0.009124755859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0062856581992966785,
+      "kl": 0.06195068359375,
       "learning_rate": 3.4314126707703895e-06,
-      "loss": 0.0086,
-      "num_tokens": 125308083.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 136787346.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 211
     },
     {
@@ -5923,26 +5923,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1173.0,
-      "completions/max_terminated_length": 1173.0,
-      "completions/mean_length": 552.49609375,
-      "completions/mean_terminated_length": 552.49609375,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2039.0,
+      "completions/mean_length": 625.490234375,
+      "completions/mean_terminated_length": 619.9118041992188,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.18983657935974926,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00454190795624133,
-      "kl": 0.0095672607421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04953062525787495,
+      "kl": 0.0645751953125,
       "learning_rate": 3.4175085134862128e-06,
-      "loss": 0.0001,
-      "num_tokens": 125907969.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0006,
+      "num_tokens": 137424605.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 212
     },
     {
@@ -5951,26 +5951,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1566.0,
-      "completions/max_terminated_length": 1566.0,
-      "completions/mean_length": 611.28125,
-      "completions/mean_terminated_length": 611.28125,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2006.0,
+      "completions/mean_length": 654.0859375,
+      "completions/mean_terminated_length": 646.8939208984375,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.19073203492276697,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003633879019486841,
-      "kl": 0.008270263671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0655278626205979,
+      "kl": 0.10150146484375,
       "learning_rate": 3.4035763488816953e-06,
-      "loss": 0.0001,
-      "num_tokens": 126542337.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0028,
+      "num_tokens": 138080889.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 213
     },
     {
@@ -5979,20 +5979,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1747.0,
-      "completions/max_terminated_length": 1747.0,
-      "completions/mean_length": 565.1015625,
-      "completions/mean_terminated_length": 565.1015625,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1689.0,
+      "completions/mean_length": 611.712890625,
+      "completions/mean_terminated_length": 608.9021606445312,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.19162749048578465,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003575289557022859,
-      "kl": 0.0095367431640625,
+      "grad_norm": 0.006893797084836839,
+      "kl": 0.06103515625,
       "learning_rate": 3.3896167615225594e-06,
-      "loss": 0.0001,
-      "num_tokens": 127152405.0,
+      "loss": 0.0006,
+      "num_tokens": 138714822.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6007,26 +6007,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1061.0,
-      "completions/max_terminated_length": 1061.0,
-      "completions/mean_length": 512.28515625,
-      "completions/mean_terminated_length": 512.28515625,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1959.0,
+      "completions/mean_length": 565.845703125,
+      "completions/mean_terminated_length": 562.9451904296875,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.19252294604880232,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0037462711645953797,
-      "kl": 0.0099639892578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05962691311018318,
+      "kl": 0.06121826171875,
       "learning_rate": 3.375630337125133e-06,
-      "loss": 0.0001,
-      "num_tokens": 127691607.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0004,
+      "num_tokens": 139281447.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 215
     },
     {
@@ -6036,25 +6036,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1562.0,
-      "completions/max_terminated_length": 1562.0,
-      "completions/mean_length": 579.341796875,
-      "completions/mean_terminated_length": 579.341796875,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/max_length": 1919.0,
+      "completions/max_terminated_length": 1919.0,
+      "completions/mean_length": 620.06640625,
+      "completions/mean_terminated_length": 620.06640625,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.19341840161182,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.037437587504057374,
-      "kl": 0.0086669921875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007079063914301492,
+      "kl": 0.06048583984375,
       "learning_rate": 3.361617662531772e-06,
-      "loss": 0.0002,
-      "num_tokens": 128268070.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 139878761.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 216
     },
     {
@@ -6065,24 +6065,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1805.0,
-      "completions/mean_length": 589.671875,
-      "completions/mean_terminated_length": 583.9530029296875,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/max_terminated_length": 1923.0,
+      "completions/mean_length": 627.935546875,
+      "completions/mean_terminated_length": 622.36669921875,
+      "completions/min_length": 98.0,
+      "completions/min_terminated_length": 98.0,
       "epoch": 0.1943138571748377,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.055260322848602106,
-      "kl": 0.009307861328125,
+      "grad_norm": 0.05517179205307519,
+      "kl": 0.05999755859375,
       "learning_rate": 3.347579325686237e-06,
-      "loss": 0.0096,
-      "num_tokens": 128873150.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.0037,
+      "num_tokens": 140503432.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 217
     },
     {
@@ -6091,26 +6091,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1452.0,
-      "completions/max_terminated_length": 1452.0,
-      "completions/mean_length": 550.720703125,
-      "completions/mean_terminated_length": 550.720703125,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1972.0,
+      "completions/mean_length": 614.9609375,
+      "completions/mean_terminated_length": 609.3411865234375,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.19520931273785538,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004407824970276491,
-      "kl": 0.0103302001953125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06462999787985825,
+      "kl": 0.06182861328125,
       "learning_rate": 3.333515915609027e-06,
-      "loss": 0.0001,
-      "num_tokens": 129445007.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0039,
+      "num_tokens": 141108180.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 218
     },
     {
@@ -6119,20 +6119,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1731.0,
-      "completions/mean_length": 581.884765625,
-      "completions/mean_terminated_length": 579.015625,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1883.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 640.31640625,
+      "completions/mean_terminated_length": 640.31640625,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
       "epoch": 0.19610476830087306,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006748768231765914,
-      "kl": 0.0100860595703125,
+      "grad_norm": 0.00663515657731032,
+      "kl": 0.05987548828125,
       "learning_rate": 3.3194280223726616e-06,
-      "loss": 0.0001,
-      "num_tokens": 130086116.0,
+      "loss": 0.0006,
+      "num_tokens": 141779206.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6147,26 +6147,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1772.0,
-      "completions/max_terminated_length": 1772.0,
-      "completions/mean_length": 589.65625,
-      "completions/mean_terminated_length": 589.65625,
-      "completions/min_length": 86.0,
-      "completions/min_terminated_length": 86.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2013.0,
+      "completions/mean_length": 654.919921875,
+      "completions/mean_terminated_length": 649.4569091796875,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.19700022386389077,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003539717009736075,
-      "kl": 0.0095062255859375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.09335614343429051,
+      "kl": 0.0611572265625,
       "learning_rate": 3.305316237076927e-06,
-      "loss": 0.0001,
-      "num_tokens": 130722708.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0058,
+      "num_tokens": 142449213.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0026298905722796917,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 220
     },
     {
@@ -6175,26 +6175,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1886.0,
-      "completions/max_terminated_length": 1886.0,
-      "completions/mean_length": 540.95703125,
-      "completions/mean_terminated_length": 540.2034912109375,
-      "completions/min_length": 139.0,
-      "completions/min_terminated_length": 139.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1987.0,
+      "completions/mean_length": 599.158203125,
+      "completions/mean_terminated_length": 588.3779296875,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.19789567942690844,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00650263702708007,
-      "kl": 0.0131988525390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03817769249302888,
+      "kl": 0.09942626953125,
       "learning_rate": 3.291181151824071e-06,
-      "loss": 0.0001,
-      "num_tokens": 131274366.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0014,
+      "num_tokens": 143030670.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 221
     },
     {
@@ -6203,20 +6203,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1143.0,
-      "completions/max_terminated_length": 1143.0,
-      "completions/mean_length": 545.39453125,
-      "completions/mean_terminated_length": 545.39453125,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1530.0,
+      "completions/mean_length": 607.646484375,
+      "completions/mean_terminated_length": 599.1571655273438,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.19879113498992612,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0033779834664063595,
-      "kl": 0.00946044921875,
+      "grad_norm": 0.007676197980008601,
+      "kl": 0.06060791015625,
       "learning_rate": 3.27702335969396e-06,
-      "loss": 0.0001,
-      "num_tokens": 131837272.0,
+      "loss": 0.0006,
+      "num_tokens": 143625449.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6231,26 +6231,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1666.0,
-      "completions/max_terminated_length": 1666.0,
-      "completions/mean_length": 565.56640625,
-      "completions/mean_terminated_length": 565.56640625,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/clipped_ratio": -6.875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1841.0,
+      "completions/mean_length": 659.720703125,
+      "completions/mean_terminated_length": 637.6845703125,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.1996865905529438,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.055179447231091354,
-      "kl": 0.00946044921875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.0787303783620633,
+      "kl": 0.09423828125,
       "learning_rate": 3.2628434547191985e-06,
-      "loss": 0.0,
-      "num_tokens": 132422986.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0004,
+      "num_tokens": 144259370.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 223
     },
     {
@@ -6259,20 +6259,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1929.0,
-      "completions/max_terminated_length": 1929.0,
-      "completions/mean_length": 565.845703125,
-      "completions/mean_terminated_length": 565.2308959960938,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1851.0,
+      "completions/mean_length": 631.271484375,
+      "completions/mean_terminated_length": 622.9214477539062,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.2005820461159615,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.21010312450686547,
-      "kl": 0.0242919921875,
+      "grad_norm": 0.005402341436416679,
+      "kl": 0.0595703125,
       "learning_rate": 3.2486420318601973e-06,
-      "loss": 0.0002,
-      "num_tokens": 133012875.0,
+      "loss": 0.0006,
+      "num_tokens": 144882757.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6287,20 +6287,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.890625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1093.0,
-      "completions/mean_length": 578.96484375,
-      "completions/mean_terminated_length": 576.0900268554688,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/max_terminated_length": 1912.0,
+      "completions/mean_length": 676.494140625,
+      "completions/mean_terminated_length": 657.4832153320312,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.20147750167897918,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.013242127221665219,
-      "kl": 0.009429931640625,
+      "grad_norm": 0.005625148130404837,
+      "kl": 0.05645751953125,
       "learning_rate": 3.2344196869802187e-06,
-      "loss": 0.0001,
-      "num_tokens": 133635897.0,
+      "loss": 0.0006,
+      "num_tokens": 145555714.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6315,20 +6315,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2034.0,
-      "completions/max_terminated_length": 2034.0,
-      "completions/mean_length": 566.775390625,
-      "completions/mean_terminated_length": 566.775390625,
-      "completions/min_length": 259.0,
-      "completions/min_terminated_length": 259.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1901.0,
+      "completions/mean_length": 654.791015625,
+      "completions/mean_terminated_length": 643.8208618164062,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.20237295724199686,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003305902510981695,
-      "kl": 0.008819580078125,
+      "grad_norm": 0.006171692676610506,
+      "kl": 0.06085205078125,
       "learning_rate": 3.2201770168203694e-06,
-      "loss": 0.0001,
-      "num_tokens": 134191606.0,
+      "loss": 0.0006,
+      "num_tokens": 146156487.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6343,20 +6343,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1829.0,
-      "completions/mean_length": 614.041015625,
-      "completions/mean_terminated_length": 611.2348022460938,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_terminated_length": 1982.0,
+      "completions/mean_length": 709.08203125,
+      "completions/mean_terminated_length": 698.5393676757812,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.20326841280501456,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003785127098280972,
-      "kl": 0.008941650390625,
+      "grad_norm": 0.007539541625438744,
+      "kl": 0.0589599609375,
       "learning_rate": 3.205914618974563e-06,
-      "loss": 0.0001,
-      "num_tokens": 134825211.0,
+      "loss": 0.0006,
+      "num_tokens": 146838753.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6372,25 +6372,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1418.0,
-      "completions/max_terminated_length": 1418.0,
-      "completions/mean_length": 573.529296875,
-      "completions/mean_terminated_length": 573.529296875,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/max_length": 1851.0,
+      "completions/max_terminated_length": 1851.0,
+      "completions/mean_length": 648.322265625,
+      "completions/mean_terminated_length": 648.322265625,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.20416386836803224,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003247968244119725,
-      "kl": 0.0087890625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.09389806559017558,
+      "kl": 0.05657958984375,
       "learning_rate": 3.1916330918644496e-06,
-      "loss": 0.0001,
-      "num_tokens": 135419226.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0014,
+      "num_tokens": 147471062.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 228
     },
     {
@@ -6399,26 +6399,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1325.0,
-      "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 560.087890625,
-      "completions/mean_terminated_length": 560.087890625,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1809.0,
+      "completions/mean_length": 631.474609375,
+      "completions/mean_terminated_length": 628.7025146484375,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.20505932393104992,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032627582108987214,
-      "kl": 0.0091094970703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.045648453246142924,
+      "kl": 0.0579833984375,
       "learning_rate": 3.177333034714303e-06,
-      "loss": 0.0001,
-      "num_tokens": 135999527.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0005,
+      "num_tokens": 148087913.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 229
     },
     {
@@ -6427,20 +6427,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.921875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1949.0,
-      "completions/mean_length": 586.2265625,
-      "completions/mean_terminated_length": 583.365966796875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_terminated_length": 2032.0,
+      "completions/mean_length": 649.294921875,
+      "completions/mean_terminated_length": 635.5009765625,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
       "epoch": 0.2059547794940676,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09080945802381934,
-      "kl": 0.0089111328125,
+      "grad_norm": 0.05962054355895875,
+      "kl": 0.058349609375,
       "learning_rate": 3.1630150475258813e-06,
-      "loss": 0.0121,
-      "num_tokens": 136620779.0,
+      "loss": 0.0154,
+      "num_tokens": 148741456.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -6456,25 +6456,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1531.0,
-      "completions/max_terminated_length": 1531.0,
-      "completions/mean_length": 538.849609375,
-      "completions/mean_terminated_length": 538.849609375,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/max_length": 1859.0,
+      "completions/max_terminated_length": 1859.0,
+      "completions/mean_length": 608.193359375,
+      "completions/mean_terminated_length": 608.193359375,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.2068502350570853,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003138487381120811,
-      "kl": 0.008453369140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.049023036722908975,
+      "kl": 0.05743408203125,
       "learning_rate": 3.148679731053252e-06,
-      "loss": 0.0001,
-      "num_tokens": 137161918.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0008,
+      "num_tokens": 149318099.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 231
     },
     {
@@ -6483,26 +6483,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1240.0,
-      "completions/max_terminated_length": 1240.0,
-      "completions/mean_length": 542.169921875,
-      "completions/mean_terminated_length": 542.169921875,
-      "completions/min_length": 102.0,
-      "completions/min_terminated_length": 102.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1938.0,
+      "completions/mean_length": 625.474609375,
+      "completions/mean_terminated_length": 617.0903930664062,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.20774569062010298,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009515406157466605,
-      "kl": 0.008758544921875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07269872169199251,
+      "kl": 0.05938720703125,
       "learning_rate": 3.1343276867775805e-06,
-      "loss": 0.0001,
-      "num_tokens": 137728021.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0021,
+      "num_tokens": 149926854.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 232
     },
     {
@@ -6511,26 +6511,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2033.0,
-      "completions/max_terminated_length": 2033.0,
-      "completions/mean_length": 542.384765625,
-      "completions/mean_terminated_length": 542.384765625,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1980.0,
+      "completions/mean_length": 613.642578125,
+      "completions/mean_terminated_length": 608.929443359375,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.20864114618312066,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0803866907591395,
-      "kl": 0.0088348388671875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.6311607973790101,
+      "kl": 0.10137939453125,
       "learning_rate": 3.1199595168819043e-06,
-      "loss": -0.0006,
-      "num_tokens": 138320154.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0164,
+      "num_tokens": 150555471.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 233
     },
     {
@@ -6539,26 +6539,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1453.0,
-      "completions/max_terminated_length": 1453.0,
-      "completions/mean_length": 555.943359375,
-      "completions/mean_terminated_length": 555.943359375,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1926.0,
+      "completions/mean_length": 645.9453125,
+      "completions/mean_terminated_length": 626.5109252929688,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.20953660174613833,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004165672775291565,
-      "kl": 0.009033203125,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.6851753963574233,
+      "kl": 0.09783935546875,
       "learning_rate": 3.105575824225852e-06,
-      "loss": 0.0001,
-      "num_tokens": 138918381.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0114,
+      "num_tokens": 151199779.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 234
     },
     {
@@ -6567,20 +6567,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1522.0,
-      "completions/mean_length": 602.44921875,
-      "completions/mean_terminated_length": 596.7804565429688,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/max_terminated_length": 2006.0,
+      "completions/mean_length": 680.703125,
+      "completions/mean_terminated_length": 669.93701171875,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.21043205730915604,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003358640291167207,
-      "kl": 0.0087738037109375,
+      "grad_norm": 0.008950782233121669,
+      "kl": 0.05804443359375,
       "learning_rate": 3.091177212320363e-06,
-      "loss": 0.0001,
-      "num_tokens": 139548099.0,
+      "loss": 0.0006,
+      "num_tokens": 151869563.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6595,26 +6595,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1849.0,
-      "completions/max_terminated_length": 1849.0,
-      "completions/mean_length": 524.318359375,
-      "completions/mean_terminated_length": 524.318359375,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1773.0,
+      "completions/mean_length": 600.86328125,
+      "completions/mean_terminated_length": 598.0313110351562,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.21132751287217372,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003609540305049046,
-      "kl": 0.0086212158203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06847164647361609,
+      "kl": 0.05633544921875,
       "learning_rate": 3.0767642853023538e-06,
       "loss": 0.0001,
-      "num_tokens": 140092470.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "num_tokens": 152453125.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 236
     },
     {
@@ -6623,26 +6623,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1873.0,
-      "completions/max_terminated_length": 1873.0,
-      "completions/mean_length": 556.666015625,
-      "completions/mean_terminated_length": 554.7005615234375,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1943.0,
+      "completions/mean_length": 621.181640625,
+      "completions/mean_terminated_length": 612.7720947265625,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.2122229684351914,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 316.8174705618427,
-      "kl": 65.50691223144531,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.055953444725904807,
+      "kl": 0.05670166015625,
       "learning_rate": 3.062337647909376e-06,
-      "loss": 0.6582,
-      "num_tokens": 140666603.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 153060290.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 237
     },
     {
@@ -6653,24 +6653,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1543.0,
-      "completions/mean_length": 533.71875,
-      "completions/mean_terminated_length": 530.75537109375,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/max_terminated_length": 1898.0,
+      "completions/mean_length": 596.9453125,
+      "completions/mean_terminated_length": 594.1056518554688,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.2131184239982091,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004126053408821417,
-      "kl": 0.0089111328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05271028890825897,
+      "kl": 0.0595703125,
       "learning_rate": 3.04789790545424e-06,
-      "loss": 0.0001,
-      "num_tokens": 141228219.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0018,
+      "num_tokens": 153654278.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 238
     },
     {
@@ -6679,20 +6679,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1307.0,
-      "completions/mean_length": 571.94140625,
-      "completions/mean_terminated_length": 569.0528564453125,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_terminated_length": 1867.0,
+      "completions/mean_length": 626.3359375,
+      "completions/mean_terminated_length": 620.7608032226562,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.21401387956122678,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031841456494166574,
-      "kl": 0.00885009765625,
+      "grad_norm": 0.00539813134920801,
+      "kl": 0.056640625,
       "learning_rate": 3.033445663799621e-06,
-      "loss": 0.0001,
-      "num_tokens": 141820893.0,
+      "loss": 0.0006,
+      "num_tokens": 154274802.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6707,20 +6707,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1528.0,
-      "completions/max_terminated_length": 1528.0,
-      "completions/mean_length": 556.794921875,
-      "completions/mean_terminated_length": 556.794921875,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1754.0,
+      "completions/mean_length": 632.125,
+      "completions/mean_terminated_length": 626.5725708007812,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.21490933512424445,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005177262843237681,
-      "kl": 0.009063720703125,
+      "grad_norm": 0.0055914288499768205,
+      "kl": 0.05413818359375,
       "learning_rate": 3.018981529332633e-06,
-      "loss": 0.0001,
-      "num_tokens": 142393172.0,
+      "loss": 0.0005,
+      "num_tokens": 154885650.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6735,26 +6735,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1442.0,
-      "completions/max_terminated_length": 1442.0,
-      "completions/mean_length": 572.935546875,
-      "completions/mean_terminated_length": 572.935546875,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1966.0,
+      "completions/mean_length": 654.580078125,
+      "completions/mean_terminated_length": 644.25390625,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.21580479068726213,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0034165096991114357,
-      "kl": 0.0080718994140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06858938507620503,
+      "kl": 0.08441162109375,
       "learning_rate": 3.00450610893939e-06,
-      "loss": 0.0001,
-      "num_tokens": 143008323.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0039,
+      "num_tokens": 155542603.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 241
     },
     {
@@ -6763,26 +6763,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1551.0,
-      "completions/mean_length": 556.876953125,
-      "completions/mean_terminated_length": 553.9589233398438,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/max_terminated_length": 1936.0,
+      "completions/mean_length": 604.103515625,
+      "completions/mean_terminated_length": 598.4412231445312,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.21670024625027984,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029792538892433463,
-      "kl": 0.0089874267578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08157612254136043,
+      "kl": 0.05743408203125,
       "learning_rate": 2.9900200099795396e-06,
-      "loss": 0.0001,
-      "num_tokens": 143602356.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0117,
+      "num_tokens": 156160816.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 242
     },
     {
@@ -6791,26 +6791,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1276.0,
-      "completions/max_terminated_length": 1276.0,
-      "completions/mean_length": 508.087890625,
-      "completions/mean_terminated_length": 508.087890625,
-      "completions/min_length": 162.0,
-      "completions/min_terminated_length": 162.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2035.0,
+      "completions/mean_length": 573.150390625,
+      "completions/mean_terminated_length": 570.26416015625,
+      "completions/min_length": 109.0,
+      "completions/min_terminated_length": 109.0,
       "epoch": 0.21759570181329752,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032021746534605607,
-      "kl": 0.008880615234375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06343258576945847,
+      "kl": 0.05889892578125,
       "learning_rate": 2.9755238402607826e-06,
-      "loss": 0.0001,
-      "num_tokens": 144170369.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0,
+      "num_tokens": 156762141.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 243
     },
     {
@@ -6819,20 +6819,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1538.0,
-      "completions/max_terminated_length": 1538.0,
-      "completions/mean_length": 586.673828125,
-      "completions/mean_terminated_length": 586.673828125,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1798.0,
+      "completions/mean_length": 644.193359375,
+      "completions/mean_terminated_length": 641.4461669921875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.2184911573763152,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04216403891799333,
-      "kl": 0.0085296630859375,
+      "grad_norm": 0.06649190527686924,
+      "kl": 0.056884765625,
       "learning_rate": 2.961018208013367e-06,
-      "loss": -0.0005,
-      "num_tokens": 144809082.0,
+      "loss": 0.0051,
+      "num_tokens": 157430304.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -6847,20 +6847,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1335.0,
-      "completions/mean_length": 601.25,
-      "completions/mean_terminated_length": 598.4187622070312,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/max_terminated_length": 1929.0,
+      "completions/mean_length": 661.330078125,
+      "completions/mean_terminated_length": 655.8922119140625,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.2193866129393329,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004208175970862356,
-      "kl": 0.0084991455078125,
+      "grad_norm": 0.0058189104851992055,
+      "kl": 0.056396484375,
       "learning_rate": 2.9465037218645694e-06,
-      "loss": 0.0001,
-      "num_tokens": 145455786.0,
+      "loss": 0.0006,
+      "num_tokens": 158107769.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6876,19 +6876,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1122.0,
-      "completions/max_terminated_length": 1122.0,
-      "completions/mean_length": 569.181640625,
-      "completions/mean_terminated_length": 569.181640625,
-      "completions/min_length": 134.0,
-      "completions/min_terminated_length": 134.0,
+      "completions/max_length": 1780.0,
+      "completions/max_terminated_length": 1780.0,
+      "completions/mean_length": 600.646484375,
+      "completions/mean_terminated_length": 600.646484375,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.22028206850235058,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00319025018151349,
-      "kl": 0.0084075927734375,
+      "grad_norm": 0.006009210073300697,
+      "kl": 0.060791015625,
       "learning_rate": 2.9319809908131604e-06,
-      "loss": 0.0001,
-      "num_tokens": 146037719.0,
+      "loss": 0.0006,
+      "num_tokens": 158705812.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6903,26 +6903,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1360.0,
-      "completions/mean_length": 593.521484375,
-      "completions/mean_terminated_length": 584.3070678710938,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/max_terminated_length": 1928.0,
+      "completions/mean_length": 621.87890625,
+      "completions/mean_terminated_length": 613.4735107421875,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.22117752406536825,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.12997437289473063,
-      "kl": 0.015045166015625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005694321164073396,
+      "kl": 0.05706787109375,
       "learning_rate": 2.917450624203847e-06,
-      "loss": 0.0128,
-      "num_tokens": 146651010.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 159333622.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 247
     },
     {
@@ -6931,20 +6931,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1196.0,
-      "completions/max_terminated_length": 1196.0,
-      "completions/mean_length": 568.05859375,
-      "completions/mean_terminated_length": 568.05859375,
-      "completions/min_length": 224.0,
-      "completions/min_terminated_length": 224.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1845.0,
+      "completions/mean_length": 599.318359375,
+      "completions/mean_terminated_length": 593.6372680664062,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.22207297962838593,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002920667478560272,
-      "kl": 0.0084991455078125,
+      "grad_norm": 0.005349239849816427,
+      "kl": 0.05615234375,
       "learning_rate": 2.9029132317017118e-06,
-      "loss": 0.0001,
-      "num_tokens": 147252144.0,
+      "loss": 0.0006,
+      "num_tokens": 159950761.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6960,25 +6960,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1365.0,
-      "completions/max_terminated_length": 1365.0,
-      "completions/mean_length": 551.31640625,
-      "completions/mean_terminated_length": 551.31640625,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/max_length": 1974.0,
+      "completions/max_terminated_length": 1974.0,
+      "completions/mean_length": 542.564453125,
+      "completions/mean_terminated_length": 542.564453125,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.22296843519140364,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027478217190436602,
-      "kl": 0.00798797607421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06951417104945298,
+      "kl": 0.05462646484375,
       "learning_rate": 2.888369423266629e-06,
-      "loss": 0.0001,
-      "num_tokens": 147817042.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.001,
+      "num_tokens": 160511178.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 249
     },
     {
@@ -6987,20 +6987,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2025.0,
-      "completions/max_terminated_length": 2025.0,
-      "completions/mean_length": 561.966796875,
-      "completions/mean_terminated_length": 561.966796875,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1705.0,
+      "completions/mean_length": 573.044921875,
+      "completions/mean_terminated_length": 570.1585083007812,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.2238638907544213,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005290429562560041,
-      "kl": 0.008819580078125,
+      "grad_norm": 0.006103601784565662,
+      "kl": 0.05767822265625,
       "learning_rate": 2.8738198091276712e-06,
-      "loss": 0.0001,
-      "num_tokens": 148416193.0,
+      "loss": 0.0006,
+      "num_tokens": 161116001.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7015,20 +7015,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1354.0,
-      "completions/max_terminated_length": 1354.0,
-      "completions/mean_length": 545.177734375,
-      "completions/mean_terminated_length": 545.177734375,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1891.0,
+      "completions/mean_length": 566.23828125,
+      "completions/mean_terminated_length": 563.3385620117188,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.224759346317439,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00344700409889045,
-      "kl": 0.008514404296875,
+      "grad_norm": 0.005820946741876905,
+      "kl": 0.05841064453125,
       "learning_rate": 2.859264999757509e-06,
-      "loss": 0.0001,
-      "num_tokens": 148960860.0,
+      "loss": 0.0006,
+      "num_tokens": 161671451.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7043,26 +7043,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1325.0,
-      "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 575.69921875,
-      "completions/mean_terminated_length": 574.74755859375,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1879.0,
+      "completions/mean_length": 609.8671875,
+      "completions/mean_terminated_length": 604.1729125976562,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.2256548018804567,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09620477170433032,
-      "kl": 0.010467529296875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.9391411364603389,
+      "kl": 0.6685791015625,
       "learning_rate": 2.8447056058467928e-06,
-      "loss": 0.0059,
-      "num_tokens": 149582338.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0145,
+      "num_tokens": 162310423.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 252
     },
     {
@@ -7072,25 +7072,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1466.0,
-      "completions/max_terminated_length": 1466.0,
-      "completions/mean_length": 609.81640625,
-      "completions/mean_terminated_length": 609.81640625,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/max_length": 1870.0,
+      "completions/max_terminated_length": 1870.0,
+      "completions/mean_length": 606.626953125,
+      "completions/mean_terminated_length": 606.626953125,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.22655025744347437,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05824071665078531,
-      "kl": 0.0098114013671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006495409905893996,
+      "kl": 0.05908203125,
       "learning_rate": 2.830142238278531e-06,
-      "loss": -0.0021,
-      "num_tokens": 150232980.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 162959432.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 253
     },
     {
@@ -7099,20 +7099,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1494.0,
-      "completions/mean_length": 614.978515625,
-      "completions/mean_terminated_length": 612.1741333007812,
-      "completions/min_length": 255.0,
-      "completions/min_terminated_length": 255.0,
+      "completions/max_terminated_length": 2048.0,
+      "completions/mean_length": 617.728515625,
+      "completions/mean_terminated_length": 612.11962890625,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.22744571300649205,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002749795848165891,
-      "kl": 0.008148193359375,
+      "grad_norm": 0.0073227247170333624,
+      "kl": 0.05572509765625,
       "learning_rate": 2.81557550810246e-06,
-      "loss": 0.0001,
-      "num_tokens": 150836057.0,
+      "loss": 0.0006,
+      "num_tokens": 163563917.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7127,26 +7127,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1787.0,
-      "completions/mean_length": 583.310546875,
-      "completions/mean_terminated_length": 580.4442138671875,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2001.0,
+      "completions/max_terminated_length": 2001.0,
+      "completions/mean_length": 586.6796875,
+      "completions/mean_terminated_length": 586.6796875,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.22834116856950973,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05832611040542345,
-      "kl": 0.0085906982421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00564245091070103,
+      "kl": 0.05596923828125,
       "learning_rate": 2.8010060265094026e-06,
-      "loss": 0.0012,
-      "num_tokens": 151427464.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 164157049.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 255
     },
     {
@@ -7155,26 +7155,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1373.0,
-      "completions/max_terminated_length": 1373.0,
-      "completions/mean_length": 559.775390625,
-      "completions/mean_terminated_length": 559.775390625,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2027.0,
+      "completions/mean_length": 594.05078125,
+      "completions/mean_terminated_length": 589.6470947265625,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.22923662413252743,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0025223599801953577,
-      "kl": 0.00812530517578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05172954991761498,
+      "kl": 0.056396484375,
       "learning_rate": 2.786434404805629e-06,
-      "loss": 0.0001,
-      "num_tokens": 152019029.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0022,
+      "num_tokens": 164766163.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 256
     },
     {
@@ -7184,19 +7184,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1523.0,
-      "completions/mean_length": 614.580078125,
-      "completions/mean_terminated_length": 611.7749633789062,
-      "completions/min_length": 242.0,
-      "completions/min_terminated_length": 242.0,
+      "completions/max_length": 1935.0,
+      "completions/max_terminated_length": 1935.0,
+      "completions/mean_length": 606.1328125,
+      "completions/mean_terminated_length": 604.4226684570312,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.2301320796955451,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030536557032659947,
-      "kl": 0.0078582763671875,
+      "grad_norm": 0.03439221585128269,
+      "kl": 0.06646728515625,
       "learning_rate": 2.771861254387199e-06,
-      "loss": 0.0001,
-      "num_tokens": 152678078.0,
+      "loss": 0.0007,
+      "num_tokens": 165420887.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7211,20 +7211,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1291.0,
-      "completions/max_terminated_length": 1291.0,
-      "completions/mean_length": 615.45703125,
-      "completions/mean_terminated_length": 615.45703125,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1828.0,
+      "completions/mean_length": 619.390625,
+      "completions/mean_terminated_length": 613.7882690429688,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.2310275352585628,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003043134978341531,
-      "kl": 0.0074005126953125,
+      "grad_norm": 0.005817974948231771,
+      "kl": 0.05487060546875,
       "learning_rate": 2.7572871867143204e-06,
-      "loss": 0.0001,
-      "num_tokens": 153292440.0,
+      "loss": 0.0005,
+      "num_tokens": 166037263.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7240,19 +7240,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1704.0,
-      "completions/max_terminated_length": 1704.0,
-      "completions/mean_length": 576.3359375,
-      "completions/mean_terminated_length": 576.3359375,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/max_length": 2017.0,
+      "completions/max_terminated_length": 2017.0,
+      "completions/mean_length": 578.689453125,
+      "completions/mean_terminated_length": 578.689453125,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.23192299082158047,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0028982682805773425,
-      "kl": 0.00792694091796875,
+      "grad_norm": 0.006405651142742982,
+      "kl": 0.05810546875,
       "learning_rate": 2.742712813285681e-06,
-      "loss": 0.0001,
-      "num_tokens": 153914180.0,
+      "loss": 0.0006,
+      "num_tokens": 166660208.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7267,26 +7267,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1893.0,
-      "completions/max_terminated_length": 1893.0,
-      "completions/mean_length": 612.654296875,
-      "completions/mean_terminated_length": 611.6085815429688,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1865.0,
+      "completions/mean_length": 621.69921875,
+      "completions/mean_terminated_length": 613.292724609375,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.23281844638459817,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00965268256928715,
-      "kl": 0.0164031982421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05751604346871723,
+      "kl": 0.05535888671875,
       "learning_rate": 2.7281387456128017e-06,
-      "loss": 0.0002,
-      "num_tokens": 154577187.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0092,
+      "num_tokens": 167327846.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 260
     },
     {
@@ -7297,18 +7297,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1499.0,
-      "completions/mean_length": 597.28515625,
-      "completions/mean_terminated_length": 594.4461669921875,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/max_terminated_length": 1699.0,
+      "completions/mean_length": 627.65234375,
+      "completions/mean_terminated_length": 624.872802734375,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.23371390194761585,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004089560262463787,
-      "kl": 0.0096588134765625,
+      "grad_norm": 0.005698567996800333,
+      "kl": 0.05517578125,
       "learning_rate": 2.7135655951943716e-06,
-      "loss": 0.0001,
-      "num_tokens": 155223269.0,
+      "loss": 0.0006,
+      "num_tokens": 167989476.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7323,26 +7323,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1608.0,
-      "completions/mean_length": 564.794921875,
-      "completions/mean_terminated_length": 561.892333984375,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1988.0,
+      "completions/max_terminated_length": 1988.0,
+      "completions/mean_length": 594.044921875,
+      "completions/mean_terminated_length": 594.044921875,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.23460935751063353,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10228660877556683,
-      "kl": 0.0084991455078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005400704242655025,
+      "kl": 0.0574951171875,
       "learning_rate": 2.698993973490598e-06,
-      "loss": 0.0131,
-      "num_tokens": 155853820.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 168635003.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 262
     },
     {
@@ -7351,26 +7351,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1835.0,
-      "completions/max_terminated_length": 1835.0,
-      "completions/mean_length": 549.693359375,
-      "completions/mean_terminated_length": 548.9823608398438,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1762.0,
+      "completions/mean_length": 555.1796875,
+      "completions/mean_terminated_length": 549.3255004882812,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.23550481307365123,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01584764131757789,
-      "kl": 0.0160675048828125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08504139270976999,
+      "kl": 0.05609130859375,
       "learning_rate": 2.6844244918975416e-06,
-      "loss": 0.0002,
-      "num_tokens": 156415647.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0133,
+      "num_tokens": 169199639.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 263
     },
     {
@@ -7380,19 +7380,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1293.0,
-      "completions/max_terminated_length": 1293.0,
-      "completions/mean_length": 568.513671875,
-      "completions/mean_terminated_length": 567.7240600585938,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1924.0,
+      "completions/mean_length": 585.38671875,
+      "completions/mean_terminated_length": 582.5244750976562,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.2364002686366689,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09690673980335059,
-      "kl": 0.022705078125,
+      "grad_norm": 0.03248029309967152,
+      "kl": 0.0560302734375,
       "learning_rate": 2.66985776172147e-06,
-      "loss": 0.0004,
-      "num_tokens": 156993862.0,
+      "loss": 0.0153,
+      "num_tokens": 169786493.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -7407,20 +7407,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1643.0,
-      "completions/max_terminated_length": 1643.0,
-      "completions/mean_length": 576.32421875,
-      "completions/mean_terminated_length": 576.32421875,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1704.0,
+      "completions/mean_length": 587.125,
+      "completions/mean_terminated_length": 584.26611328125,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.2372957241996866,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004439510741313418,
-      "kl": 0.008209228515625,
+      "grad_norm": 0.0054535619814076244,
+      "kl": 0.0555419921875,
       "learning_rate": 2.6552943941532088e-06,
-      "loss": 0.0001,
-      "num_tokens": 157604844.0,
+      "loss": 0.0006,
+      "num_tokens": 170403005.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7435,20 +7435,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1738.0,
-      "completions/mean_length": 580.025390625,
-      "completions/mean_terminated_length": 577.1526489257812,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2000.0,
+      "completions/max_terminated_length": 2000.0,
+      "completions/mean_length": 578.591796875,
+      "completions/mean_terminated_length": 578.591796875,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.23819117976270426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005051127249402352,
-      "kl": 0.00849151611328125,
+      "grad_norm": 0.005800541107133192,
+      "kl": 0.0567626953125,
       "learning_rate": 2.6407350002424927e-06,
-      "loss": 0.0001,
-      "num_tokens": 158225225.0,
+      "loss": 0.0006,
+      "num_tokens": 171022652.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7465,18 +7465,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1434.0,
-      "completions/mean_length": 570.8515625,
-      "completions/mean_terminated_length": 567.9608764648438,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_terminated_length": 1917.0,
+      "completions/mean_length": 598.119140625,
+      "completions/mean_terminated_length": 595.2817993164062,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
       "epoch": 0.23908663532572197,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009859382079514532,
-      "kl": 0.0097503662109375,
+      "grad_norm": 0.005198138170323681,
+      "kl": 0.05340576171875,
       "learning_rate": 2.626180190872329e-06,
-      "loss": 0.0001,
-      "num_tokens": 158802237.0,
+      "loss": 0.0005,
+      "num_tokens": 171613625.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7492,19 +7492,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1336.0,
-      "completions/max_terminated_length": 1336.0,
-      "completions/mean_length": 566.79296875,
-      "completions/mean_terminated_length": 566.79296875,
-      "completions/min_length": 231.0,
-      "completions/min_terminated_length": 231.0,
+      "completions/max_length": 1365.0,
+      "completions/max_terminated_length": 1365.0,
+      "completions/mean_length": 574.216796875,
+      "completions/mean_terminated_length": 574.216796875,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
       "epoch": 0.23998209088873965,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004277627243588492,
-      "kl": 0.00750732421875,
+      "grad_norm": 0.00526669304671622,
+      "kl": 0.05548095703125,
       "learning_rate": 2.611630576733372e-06,
-      "loss": 0.0001,
-      "num_tokens": 159404563.0,
+      "loss": 0.0006,
+      "num_tokens": 172219752.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7519,20 +7519,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1528.0,
-      "completions/max_terminated_length": 1528.0,
-      "completions/mean_length": 564.849609375,
-      "completions/mean_terminated_length": 564.849609375,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1925.0,
+      "completions/mean_length": 590.83984375,
+      "completions/mean_terminated_length": 587.98828125,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.24087754645175732,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004866099932917146,
-      "kl": 0.0077972412109375,
+      "grad_norm": 0.005186941679267824,
+      "kl": 0.0560302734375,
       "learning_rate": 2.5970867682982885e-06,
-      "loss": 0.0001,
-      "num_tokens": 159992854.0,
+      "loss": 0.0006,
+      "num_tokens": 172821350.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7547,26 +7547,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1802.0,
-      "completions/mean_length": 570.490234375,
-      "completions/mean_terminated_length": 567.5988159179688,
-      "completions/min_length": 84.0,
-      "completions/min_terminated_length": 84.0,
+      "completions/max_terminated_length": 2007.0,
+      "completions/mean_length": 596.86328125,
+      "completions/mean_terminated_length": 591.172607421875,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.24177300201477503,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05839995382013476,
-      "kl": 0.008270263671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005093847327967122,
+      "kl": 0.05621337890625,
       "learning_rate": 2.582549375796154e-06,
-      "loss": 0.0129,
-      "num_tokens": 160622785.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 173464784.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 270
     },
     {
@@ -7575,26 +7575,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1279.0,
-      "completions/mean_length": 582.490234375,
-      "completions/mean_terminated_length": 579.622314453125,
-      "completions/min_length": 139.0,
-      "completions/min_terminated_length": 139.0,
+      "completions/max_terminated_length": 1905.0,
+      "completions/mean_length": 593.822265625,
+      "completions/mean_terminated_length": 588.11962890625,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.2426684575777927,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0035678064526539688,
-      "kl": 0.008392333984375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.060575861215051076,
+      "kl": 0.05645751953125,
       "learning_rate": 2.568019009186841e-06,
-      "loss": 0.0001,
-      "num_tokens": 161182924.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0023,
+      "num_tokens": 174030725.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 271
     },
     {
@@ -7604,25 +7604,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1361.0,
-      "completions/max_terminated_length": 1361.0,
-      "completions/mean_length": 525.4921875,
-      "completions/mean_terminated_length": 525.4921875,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/max_length": 1777.0,
+      "completions/max_terminated_length": 1777.0,
+      "completions/mean_length": 542.765625,
+      "completions/mean_terminated_length": 542.765625,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.24356391314081038,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031233149161186808,
-      "kl": 0.008209228515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04746305553350059,
+      "kl": 0.0526123046875,
       "learning_rate": 2.5534962781354317e-06,
-      "loss": 0.0001,
-      "num_tokens": 161755176.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0031,
+      "num_tokens": 174611821.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 272
     },
     {
@@ -7631,20 +7631,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1826.0,
-      "completions/max_terminated_length": 1826.0,
-      "completions/mean_length": 568.16796875,
-      "completions/mean_terminated_length": 568.16796875,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1549.0,
+      "completions/mean_length": 583.73046875,
+      "completions/mean_terminated_length": 577.98828125,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.24445936870382806,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0034764954757281943,
-      "kl": 0.0081634521484375,
+      "grad_norm": 0.0045173693793721685,
+      "kl": 0.0533447265625,
       "learning_rate": 2.538981791986634e-06,
-      "loss": 0.0001,
-      "num_tokens": 162342302.0,
+      "loss": 0.0005,
+      "num_tokens": 175206915.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7660,25 +7660,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1273.0,
-      "completions/max_terminated_length": 1273.0,
-      "completions/mean_length": 580.52734375,
-      "completions/mean_terminated_length": 580.52734375,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
+      "completions/max_length": 2019.0,
+      "completions/max_terminated_length": 2019.0,
+      "completions/mean_length": 567.197265625,
+      "completions/mean_terminated_length": 567.197265625,
+      "completions/min_length": 106.0,
+      "completions/min_terminated_length": 106.0,
       "epoch": 0.24535482426684577,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0028758465684256365,
-      "kl": 0.008209228515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06113247025112339,
+      "kl": 0.05450439453125,
       "learning_rate": 2.524476159739218e-06,
-      "loss": 0.0001,
-      "num_tokens": 162964364.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0005,
+      "num_tokens": 175822152.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 274
     },
     {
@@ -7688,19 +7688,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1399.0,
-      "completions/max_terminated_length": 1399.0,
-      "completions/mean_length": 575.3125,
-      "completions/mean_terminated_length": 575.3125,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/max_length": 1987.0,
+      "completions/max_terminated_length": 1987.0,
+      "completions/mean_length": 585.8984375,
+      "completions/mean_terminated_length": 585.8984375,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.24625027982986344,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0039951419743716335,
-      "kl": 0.0081787109375,
+      "grad_norm": 0.00573175071522722,
+      "kl": 0.0528564453125,
       "learning_rate": 2.5099799900204607e-06,
-      "loss": 0.0001,
-      "num_tokens": 163563148.0,
+      "loss": 0.0005,
+      "num_tokens": 176426356.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7715,20 +7715,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1598.0,
-      "completions/max_terminated_length": 1598.0,
-      "completions/mean_length": 605.78125,
-      "completions/mean_terminated_length": 605.78125,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1939.0,
+      "completions/max_terminated_length": 1939.0,
+      "completions/mean_length": 591.904296875,
+      "completions/mean_terminated_length": 589.8297119140625,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.24714573539288112,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027137761723457256,
-      "kl": 0.00742340087890625,
+      "grad_norm": 0.006265698499194678,
+      "kl": 0.0513916015625,
       "learning_rate": 2.4954938910606108e-06,
-      "loss": 0.0001,
-      "num_tokens": 164148028.0,
+      "loss": 0.0005,
+      "num_tokens": 177004131.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7743,26 +7743,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1356.0,
-      "completions/max_terminated_length": 1356.0,
-      "completions/mean_length": 540.462890625,
-      "completions/mean_terminated_length": 540.462890625,
-      "completions/min_length": 103.0,
-      "completions/min_terminated_length": 103.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1694.0,
+      "completions/mean_length": 543.6328125,
+      "completions/mean_terminated_length": 540.6888427734375,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.24804119095589883,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0036590336323787287,
-      "kl": 0.008209228515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06559073582245414,
+      "kl": 0.05224609375,
       "learning_rate": 2.481018470667368e-06,
-      "loss": 0.0001,
-      "num_tokens": 164707705.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0005,
+      "num_tokens": 177565431.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 277
     },
     {
@@ -7771,26 +7771,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1390.0,
-      "completions/max_terminated_length": 1390.0,
-      "completions/mean_length": 572.0390625,
-      "completions/mean_terminated_length": 572.0390625,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1680.0,
+      "completions/mean_length": 568.951171875,
+      "completions/mean_terminated_length": 563.87255859375,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.2489366465189165,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002865287351534278,
-      "kl": 0.00798797607421875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 13.405018764621035,
+      "kl": 0.37408447265625,
       "learning_rate": 2.4665543362003802e-06,
-      "loss": 0.0001,
-      "num_tokens": 165284237.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0184,
+      "num_tokens": 178140382.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 278
     },
     {
@@ -7799,26 +7799,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1581.0,
-      "completions/max_terminated_length": 1581.0,
-      "completions/mean_length": 576.732421875,
-      "completions/mean_terminated_length": 576.732421875,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1781.0,
+      "completions/mean_length": 582.24609375,
+      "completions/mean_terminated_length": 579.377685546875,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.24983210208193418,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026953753539202627,
-      "kl": 0.0083465576171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07793443113322279,
+      "kl": 0.05450439453125,
       "learning_rate": 2.4521020945457615e-06,
-      "loss": 0.0001,
-      "num_tokens": 165892180.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0025,
+      "num_tokens": 178751148.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 279
     },
     {
@@ -7828,25 +7828,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1806.0,
-      "completions/max_terminated_length": 1806.0,
-      "completions/mean_length": 572.642578125,
-      "completions/mean_terminated_length": 572.642578125,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1761.0,
+      "completions/max_terminated_length": 1761.0,
+      "completions/mean_length": 555.86328125,
+      "completions/mean_terminated_length": 555.86328125,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.25072755764495186,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06153457584936372,
-      "kl": 0.007965087890625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005138666007838519,
+      "kl": 0.05181884765625,
       "learning_rate": 2.4376623520906255e-06,
-      "loss": -0.0001,
-      "num_tokens": 166502269.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 179352646.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 280
     },
     {
@@ -7856,19 +7856,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1791.0,
-      "completions/max_terminated_length": 1791.0,
-      "completions/mean_length": 618.21875,
-      "completions/mean_terminated_length": 618.21875,
-      "completions/min_length": 155.0,
-      "completions/min_terminated_length": 155.0,
+      "completions/max_length": 1485.0,
+      "completions/max_terminated_length": 1485.0,
+      "completions/mean_length": 591.35546875,
+      "completions/mean_terminated_length": 591.35546875,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.25162301320796954,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024592397394463637,
-      "kl": 0.0079193115234375,
+      "grad_norm": 0.005950683891807377,
+      "kl": 0.05291748046875,
       "learning_rate": 2.4232357146976478e-06,
-      "loss": 0.0001,
-      "num_tokens": 167133965.0,
+      "loss": 0.0005,
+      "num_tokens": 179970588.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7883,20 +7883,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1651.0,
-      "completions/max_terminated_length": 1651.0,
-      "completions/mean_length": 600.8828125,
-      "completions/mean_terminated_length": 600.8828125,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1767.0,
+      "completions/max_terminated_length": 1767.0,
+      "completions/mean_length": 615.041015625,
+      "completions/mean_terminated_length": 613.628173828125,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
       "epoch": 0.2525184687709872,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003365096936430135,
-      "kl": 0.0074310302734375,
+      "grad_norm": 24.033499681250188,
+      "kl": 2.96002197265625,
       "learning_rate": 2.408822787679637e-06,
-      "loss": 0.0001,
-      "num_tokens": 167729729.0,
+      "loss": 0.0295,
+      "num_tokens": 180573601.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7912,19 +7912,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1282.0,
-      "completions/max_terminated_length": 1282.0,
-      "completions/mean_length": 551.373046875,
-      "completions/mean_terminated_length": 551.373046875,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/max_length": 1644.0,
+      "completions/max_terminated_length": 1644.0,
+      "completions/mean_length": 565.318359375,
+      "completions/mean_terminated_length": 565.318359375,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.25341392433400495,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031736404722537124,
-      "kl": 0.0082550048828125,
+      "grad_norm": 0.0051856058148775286,
+      "kl": 0.0518798828125,
       "learning_rate": 2.3944241757741475e-06,
-      "loss": 0.0001,
-      "num_tokens": 168309168.0,
+      "loss": 0.0005,
+      "num_tokens": 181160180.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7939,26 +7939,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1599.0,
-      "completions/max_terminated_length": 1599.0,
-      "completions/mean_length": 590.669921875,
-      "completions/mean_terminated_length": 590.669921875,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1993.0,
+      "completions/mean_length": 617.09375,
+      "completions/mean_terminated_length": 614.2935180664062,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.2543093798970226,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0567955043559705,
-      "kl": 0.0079345703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0068416300076160775,
+      "kl": 0.05120849609375,
       "learning_rate": 2.380040483118097e-06,
-      "loss": 0.0004,
-      "num_tokens": 168891207.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 181755748.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 284
     },
     {
@@ -7968,25 +7968,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1773.0,
-      "completions/max_terminated_length": 1773.0,
-      "completions/mean_length": 569.77734375,
-      "completions/mean_terminated_length": 569.77734375,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
+      "completions/max_length": 1894.0,
+      "completions/max_terminated_length": 1894.0,
+      "completions/mean_length": 583.033203125,
+      "completions/mean_terminated_length": 583.033203125,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.2552048354600403,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030194281648096032,
-      "kl": 0.00792694091796875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07301203946773449,
+      "kl": 0.051513671875,
       "learning_rate": 2.365672313222419e-06,
-      "loss": 0.0001,
-      "num_tokens": 169487141.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0029,
+      "num_tokens": 182358469.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 285
     },
     {
@@ -7995,20 +7995,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1905.0,
-      "completions/max_terminated_length": 1905.0,
-      "completions/mean_length": 615.369140625,
-      "completions/mean_terminated_length": 615.369140625,
-      "completions/min_length": 240.0,
-      "completions/min_terminated_length": 240.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1728.0,
+      "completions/mean_length": 628.9453125,
+      "completions/mean_terminated_length": 623.3804321289062,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.256100291023058,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003087403099165956,
-      "kl": 0.008270263671875,
+      "grad_norm": 0.0054564530522672825,
+      "kl": 0.05206298828125,
       "learning_rate": 2.351320268946749e-06,
-      "loss": 0.0001,
-      "num_tokens": 170137170.0,
+      "loss": 0.0005,
+      "num_tokens": 183015449.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8023,20 +8023,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1735.0,
-      "completions/max_terminated_length": 1735.0,
-      "completions/mean_length": 505.8828125,
-      "completions/mean_terminated_length": 505.8828125,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1781.0,
+      "completions/mean_length": 538.4453125,
+      "completions/mean_terminated_length": 535.4912109375,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.25699574658607566,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00622764953467058,
-      "kl": 0.009307861328125,
+      "grad_norm": 0.005346092389195165,
+      "kl": 0.0521240234375,
       "learning_rate": 2.336984952474119e-06,
-      "loss": 0.0001,
-      "num_tokens": 170663558.0,
+      "loss": 0.0005,
+      "num_tokens": 183558509.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8053,18 +8053,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1442.0,
-      "completions/mean_length": 529.994140625,
-      "completions/mean_terminated_length": 527.0234985351562,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/max_terminated_length": 1950.0,
+      "completions/mean_length": 547.73046875,
+      "completions/mean_terminated_length": 544.7944946289062,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.25789120214909333,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029162163258119552,
-      "kl": 0.00823974609375,
+      "grad_norm": 0.005394893401423969,
+      "kl": 0.0499267578125,
       "learning_rate": 2.322666965285697e-06,
-      "loss": 0.0001,
-      "num_tokens": 171200787.0,
+      "loss": 0.0005,
+      "num_tokens": 184104819.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8079,20 +8079,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1782.0,
-      "completions/max_terminated_length": 1782.0,
-      "completions/mean_length": 551.056640625,
-      "completions/mean_terminated_length": 551.056640625,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1585.0,
+      "completions/mean_length": 588.498046875,
+      "completions/mean_terminated_length": 582.7745361328125,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.258786657712111,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002784819528636807,
-      "kl": 0.008056640625,
+      "grad_norm": 0.004969546618879694,
+      "kl": 0.05023193359375,
       "learning_rate": 2.3083669081355507e-06,
-      "loss": 0.0001,
-      "num_tokens": 171790752.0,
+      "loss": 0.0005,
+      "num_tokens": 184713954.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8107,20 +8107,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1225.0,
-      "completions/mean_length": 544.69140625,
-      "completions/mean_terminated_length": 541.74951171875,
-      "completions/min_length": 139.0,
-      "completions/min_terminated_length": 139.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1976.0,
+      "completions/max_terminated_length": 1976.0,
+      "completions/mean_length": 552.771484375,
+      "completions/mean_terminated_length": 552.771484375,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.25968211327512875,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0033671102355632226,
-      "kl": 0.0086822509765625,
+      "grad_norm": 0.008575669078842711,
+      "kl": 0.0533447265625,
       "learning_rate": 2.2940853810254377e-06,
-      "loss": 0.0001,
-      "num_tokens": 172367042.0,
+      "loss": 0.0005,
+      "num_tokens": 185294381.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8135,20 +8135,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1565.0,
-      "completions/mean_length": 566.361328125,
-      "completions/mean_terminated_length": 563.4618530273438,
-      "completions/min_length": 121.0,
-      "completions/min_terminated_length": 121.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1260.0,
+      "completions/max_terminated_length": 1260.0,
+      "completions/mean_length": 579.490234375,
+      "completions/mean_terminated_length": 579.490234375,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.2605775688381464,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026251951857379574,
-      "kl": 0.0079498291015625,
+      "grad_norm": 0.0052073739337190885,
+      "kl": 0.05035400390625,
       "learning_rate": 2.2798229831796313e-06,
-      "loss": 0.0001,
-      "num_tokens": 172917003.0,
+      "loss": 0.0005,
+      "num_tokens": 185851064.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8163,26 +8163,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2025.0,
-      "completions/max_terminated_length": 2025.0,
-      "completions/mean_length": 613.81640625,
-      "completions/mean_terminated_length": 613.81640625,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1751.0,
+      "completions/mean_length": 630.0,
+      "completions/mean_terminated_length": 627.2250366210938,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.2614730244011641,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027979065409168315,
-      "kl": 0.00760650634765625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.053543315977731425,
+      "kl": 0.0494384765625,
       "learning_rate": 2.2655803130197816e-06,
-      "loss": 0.0001,
-      "num_tokens": 173525501.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0003,
+      "num_tokens": 186467848.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 292
     },
     {
@@ -8193,18 +8193,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1287.0,
-      "completions/mean_length": 543.486328125,
-      "completions/mean_terminated_length": 540.5420532226562,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/max_terminated_length": 1816.0,
+      "completions/mean_length": 568.5859375,
+      "completions/mean_terminated_length": 565.6907958984375,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.2623684799641818,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002850882049670849,
-      "kl": 0.00788116455078125,
+      "grad_norm": 0.004634463449738635,
+      "kl": 0.05108642578125,
       "learning_rate": 2.2513579681398034e-06,
-      "loss": 0.0001,
-      "num_tokens": 174093798.0,
+      "loss": 0.0005,
+      "num_tokens": 187048996.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8219,20 +8219,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1779.0,
-      "completions/mean_length": 519.80859375,
-      "completions/mean_terminated_length": 516.8179931640625,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/max_terminated_length": 1966.0,
+      "completions/mean_length": 554.080078125,
+      "completions/mean_terminated_length": 548.2216186523438,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.26326393552719946,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002895038223420061,
-      "kl": 0.00860595703125,
+      "grad_norm": 0.004680081807285644,
+      "kl": 0.0513916015625,
       "learning_rate": 2.237156545280803e-06,
-      "loss": 0.0001,
-      "num_tokens": 174623156.0,
+      "loss": 0.0005,
+      "num_tokens": 187595901.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8248,19 +8248,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1346.0,
-      "completions/max_terminated_length": 1346.0,
-      "completions/mean_length": 556.666015625,
-      "completions/mean_terminated_length": 556.666015625,
-      "completions/min_length": 174.0,
-      "completions/min_terminated_length": 174.0,
+      "completions/max_length": 1857.0,
+      "completions/max_terminated_length": 1857.0,
+      "completions/mean_length": 592.625,
+      "completions/mean_terminated_length": 592.625,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.26415939109021713,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031545665495167205,
-      "kl": 0.0075531005859375,
+      "grad_norm": 0.004679039787165404,
+      "kl": 0.052001953125,
       "learning_rate": 2.2229766403060403e-06,
-      "loss": 0.0001,
-      "num_tokens": 175196185.0,
+      "loss": 0.0005,
+      "num_tokens": 188187341.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8275,26 +8275,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1574.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 580.091796875,
-      "completions/mean_terminated_length": 580.091796875,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1792.0,
+      "completions/mean_length": 614.19140625,
+      "completions/mean_terminated_length": 610.5235595703125,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.2650548466532348,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0025604974218733523,
-      "kl": 0.00716400146484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.052447372839830086,
+      "kl": 0.05328369140625,
       "learning_rate": 2.2088188481759305e-06,
-      "loss": 0.0001,
-      "num_tokens": 175782888.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0126,
+      "num_tokens": 188791503.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 296
     },
     {
@@ -8304,25 +8304,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1464.0,
-      "completions/mean_length": 581.8515625,
-      "completions/mean_terminated_length": 578.9823608398438,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/max_length": 1818.0,
+      "completions/max_terminated_length": 1818.0,
+      "completions/mean_length": 627.5078125,
+      "completions/mean_terminated_length": 626.383544921875,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
       "epoch": 0.26595030221625254,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002912348412355826,
-      "kl": 0.0078277587890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04957490670885095,
+      "kl": 0.05078125,
       "learning_rate": 2.194683762923073e-06,
-      "loss": 0.0001,
-      "num_tokens": 176396764.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0005,
+      "num_tokens": 189428755.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 297
     },
     {
@@ -8331,26 +8331,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1329.0,
-      "completions/mean_length": 543.87109375,
-      "completions/mean_terminated_length": 540.9276123046875,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1568.0,
+      "completions/max_terminated_length": 1568.0,
+      "completions/mean_length": 558.3046875,
+      "completions/mean_terminated_length": 558.3046875,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.2668457577792702,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07647397750323286,
-      "kl": 0.0075836181640625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07521000733044311,
+      "kl": 0.05029296875,
       "learning_rate": 2.1805719776273387e-06,
-      "loss": 0.0137,
-      "num_tokens": 176957418.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0034,
+      "num_tokens": 189996799.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 298
     },
     {
@@ -8359,26 +8359,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1864.0,
-      "completions/mean_length": 584.349609375,
-      "completions/mean_terminated_length": 581.4852905273438,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/max_terminated_length": 1702.0,
+      "completions/mean_length": 595.80078125,
+      "completions/mean_terminated_length": 590.1058959960938,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.2677412133422879,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0832525767775582,
-      "kl": 0.0071868896484375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 52.29210967783974,
+      "kl": 2.88189697265625,
       "learning_rate": 2.166484084390974e-06,
-      "loss": 0.0082,
-      "num_tokens": 177551853.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0381,
+      "num_tokens": 190597097.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 299
     },
     {
@@ -8388,25 +8388,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1175.0,
-      "completions/max_terminated_length": 1175.0,
-      "completions/mean_length": 572.251953125,
-      "completions/mean_terminated_length": 572.251953125,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_length": 1919.0,
+      "completions/max_terminated_length": 1919.0,
+      "completions/mean_length": 575.779296875,
+      "completions/mean_terminated_length": 575.779296875,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.2686366689053056,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024824890095177894,
-      "kl": 0.0075225830078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05666066912224791,
+      "kl": 0.0506591796875,
       "learning_rate": 2.1524206743137636e-06,
-      "loss": 0.0001,
-      "num_tokens": 178135006.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0024,
+      "num_tokens": 191182056.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 300
     },
     {
@@ -8415,20 +8415,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1303.0,
-      "completions/max_terminated_length": 1303.0,
-      "completions/mean_length": 590.498046875,
-      "completions/mean_terminated_length": 590.498046875,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1711.0,
+      "completions/mean_length": 602.306640625,
+      "completions/mean_terminated_length": 599.4774780273438,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.26953212446832325,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06156528604399248,
-      "kl": 0.00756072998046875,
+      "grad_norm": 0.04902480949298609,
+      "kl": 0.0518798828125,
       "learning_rate": 2.1383823374682287e-06,
-      "loss": -0.0006,
-      "num_tokens": 178751997.0,
+      "loss": -0.0009,
+      "num_tokens": 191805093.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -8441,22 +8441,22 @@
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1633.0,
-      "completions/max_terminated_length": 1633.0,
-      "completions/mean_length": 574.0390625,
-      "completions/mean_terminated_length": 574.0390625,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1448.0,
+      "completions/mean_length": 600.71484375,
+      "completions/mean_terminated_length": 595.0392456054688,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.27042758003134093,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009728130461388736,
-      "kl": 0.00785064697265625,
+      "grad_norm": 0.004860988243453146,
+      "kl": 0.05169677734375,
       "learning_rate": 2.124369662874868e-06,
-      "loss": 0.0001,
-      "num_tokens": 179350993.0,
+      "loss": 0.0005,
+      "num_tokens": 192417747.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8471,26 +8471,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1401.0,
-      "completions/max_terminated_length": 1401.0,
-      "completions/mean_length": 558.1328125,
-      "completions/mean_terminated_length": 558.1328125,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1668.0,
+      "completions/mean_length": 565.7265625,
+      "completions/mean_terminated_length": 562.8258056640625,
+      "completions/min_length": 111.0,
+      "completions/min_terminated_length": 111.0,
       "epoch": 0.2713230355943586,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00216036159896381,
-      "kl": 0.00812530517578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.054741263922224406,
+      "kl": 0.05029296875,
       "learning_rate": 2.110383238477441e-06,
-      "loss": 0.0001,
-      "num_tokens": 179902389.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0018,
+      "num_tokens": 192973031.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 303
     },
     {
@@ -8499,26 +8499,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1786.0,
-      "completions/max_terminated_length": 1786.0,
-      "completions/mean_length": 616.208984375,
-      "completions/mean_terminated_length": 616.208984375,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1812.0,
+      "completions/max_terminated_length": 1812.0,
+      "completions/mean_length": 627.341796875,
+      "completions/mean_terminated_length": 625.0234985351562,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.27221849115737634,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030452568956129706,
-      "kl": 0.0074005126953125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 1.1739951576785839,
+      "kl": 0.18341064453125,
       "learning_rate": 2.096423651118305e-06,
-      "loss": 0.0001,
-      "num_tokens": 180534224.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0074,
+      "num_tokens": 193610566.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 304
     },
     {
@@ -8527,20 +8527,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1252.0,
-      "completions/max_terminated_length": 1252.0,
-      "completions/mean_length": 563.119140625,
-      "completions/mean_terminated_length": 563.119140625,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1740.0,
+      "completions/mean_length": 594.041015625,
+      "completions/mean_terminated_length": 591.1956787109375,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
       "epoch": 0.273113946720394,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003442268008121624,
-      "kl": 0.00763702392578125,
+      "grad_norm": 0.005936683963889181,
+      "kl": 0.0511474609375,
       "learning_rate": 2.082491486513788e-06,
-      "loss": 0.0001,
-      "num_tokens": 181105917.0,
+      "loss": 0.0005,
+      "num_tokens": 194198091.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8555,26 +8555,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1777.0,
-      "completions/max_terminated_length": 1777.0,
-      "completions/mean_length": 569.068359375,
-      "completions/mean_terminated_length": 569.068359375,
-      "completions/min_length": 236.0,
-      "completions/min_terminated_length": 236.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1528.0,
+      "completions/mean_length": 582.513671875,
+      "completions/mean_terminated_length": 579.6458129882812,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.2740094022834117,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05941448759219727,
-      "kl": 0.008209228515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006122848959639277,
+      "kl": 0.05328369140625,
       "learning_rate": 2.0685873292296116e-06,
-      "loss": -0.0006,
-      "num_tokens": 181673888.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 194772946.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 306
     },
     {
@@ -8583,26 +8583,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1471.0,
-      "completions/mean_length": 573.65625,
-      "completions/mean_terminated_length": 570.7710571289062,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1602.0,
+      "completions/max_terminated_length": 1602.0,
+      "completions/mean_length": 591.427734375,
+      "completions/mean_terminated_length": 591.427734375,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.2749048578464294,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005439544093753847,
-      "kl": 0.00799560546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05947760191798018,
+      "kl": 0.05084228515625,
       "learning_rate": 2.054711762656369e-06,
-      "loss": 0.0001,
-      "num_tokens": 182278032.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0041,
+      "num_tokens": 195386189.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 307
     },
     {
@@ -8613,24 +8613,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1634.0,
-      "completions/mean_length": 588.927734375,
-      "completions/mean_terminated_length": 586.0723876953125,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/max_terminated_length": 1716.0,
+      "completions/mean_length": 598.970703125,
+      "completions/mean_terminated_length": 596.135009765625,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
       "epoch": 0.27580031340944705,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07732112935482617,
-      "kl": 0.0078582763671875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10573090050455233,
+      "kl": 0.0560302734375,
       "learning_rate": 2.040865368985044e-06,
-      "loss": 0.0149,
-      "num_tokens": 182863451.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0109,
+      "num_tokens": 195976750.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 308
     },
     {
@@ -8639,20 +8639,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1832.0,
-      "completions/max_terminated_length": 1832.0,
-      "completions/mean_length": 638.77734375,
-      "completions/mean_terminated_length": 638.77734375,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1736.0,
+      "completions/mean_length": 646.505859375,
+      "completions/mean_terminated_length": 635.470458984375,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.27669576897246473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027031381335453617,
-      "kl": 0.00719451904296875,
+      "grad_norm": 0.005228529652921396,
+      "kl": 0.05120849609375,
       "learning_rate": 2.027048729182583e-06,
-      "loss": 0.0001,
-      "num_tokens": 183501497.0,
+      "loss": 0.0005,
+      "num_tokens": 196618753.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8668,19 +8668,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1680.0,
-      "completions/max_terminated_length": 1680.0,
-      "completions/mean_length": 607.361328125,
-      "completions/mean_terminated_length": 607.361328125,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/max_length": 1819.0,
+      "completions/max_terminated_length": 1819.0,
+      "completions/mean_length": 606.69921875,
+      "completions/mean_terminated_length": 606.69921875,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.2775912245354824,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00369268777417457,
-      "kl": 0.0084075927734375,
+      "grad_norm": 0.0051559766107908914,
+      "kl": 0.05517578125,
       "learning_rate": 2.0132624229675205e-06,
-      "loss": 0.0001,
-      "num_tokens": 184149154.0,
+      "loss": 0.0006,
+      "num_tokens": 197266071.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8695,20 +8695,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1241.0,
-      "completions/max_terminated_length": 1241.0,
-      "completions/mean_length": 571.37109375,
-      "completions/mean_terminated_length": 571.37109375,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1437.0,
+      "completions/mean_length": 596.806640625,
+      "completions/mean_terminated_length": 593.9667358398438,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.27848668009850014,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08066837486158972,
-      "kl": 0.0079193115234375,
+      "grad_norm": 0.0479412945784682,
+      "kl": 0.05206298828125,
       "learning_rate": 1.9995070287856546e-06,
-      "loss": 0.0064,
-      "num_tokens": 184729056.0,
+      "loss": 0.0012,
+      "num_tokens": 197858996.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -8723,26 +8723,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1238.0,
-      "completions/max_terminated_length": 1238.0,
-      "completions/mean_length": 571.208984375,
-      "completions/mean_terminated_length": 571.208984375,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1617.0,
+      "completions/mean_length": 578.19140625,
+      "completions/mean_terminated_length": 575.3150634765625,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.2793821356615178,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002942664647734796,
-      "kl": 0.0076904296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05124512126982719,
+      "kl": 0.05169677734375,
       "learning_rate": 1.985783123785774e-06,
-      "loss": 0.0001,
-      "num_tokens": 185305307.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0183,
+      "num_tokens": 198438822.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 312
     },
     {
@@ -8752,25 +8752,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1427.0,
-      "completions/max_terminated_length": 1427.0,
-      "completions/mean_length": 561.5,
-      "completions/mean_terminated_length": 560.5322875976562,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1749.0,
+      "completions/mean_length": 568.921875,
+      "completions/mean_terminated_length": 566.0274047851562,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.2802775912245355,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008108481289585232,
-      "kl": 0.0139617919921875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05819859258148276,
+      "kl": 0.0533447265625,
       "learning_rate": 1.9720912837954486e-06,
-      "loss": 0.0001,
-      "num_tokens": 185893611.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0,
+      "num_tokens": 199030926.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 313
     },
     {
@@ -8780,25 +8780,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1551.0,
-      "completions/max_terminated_length": 1551.0,
-      "completions/mean_length": 560.962890625,
-      "completions/mean_terminated_length": 560.962890625,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/max_length": 1491.0,
+      "completions/max_terminated_length": 1491.0,
+      "completions/mean_length": 554.88671875,
+      "completions/mean_terminated_length": 554.88671875,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.28117304678755317,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005140104045070806,
-      "kl": 0.00801849365234375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09616545890196271,
+      "kl": 0.0570068359375,
       "learning_rate": 1.958432083296862e-06,
-      "loss": 0.0001,
-      "num_tokens": 186474808.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0083,
+      "num_tokens": 199609012.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 314
     },
     {
@@ -8808,19 +8808,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1372.0,
-      "completions/max_terminated_length": 1372.0,
-      "completions/mean_length": 572.884765625,
-      "completions/mean_terminated_length": 572.131103515625,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1908.0,
+      "completions/mean_length": 562.427734375,
+      "completions/mean_terminated_length": 559.5205688476562,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.28206850235057085,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.1897760386031404,
-      "kl": 0.13326263427734375,
+      "grad_norm": 0.006128274293792745,
+      "kl": 0.052490234375,
       "learning_rate": 1.9448060954027093e-06,
-      "loss": 0.0013,
-      "num_tokens": 187041453.0,
+      "loss": 0.0005,
+      "num_tokens": 200170303.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8836,25 +8836,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1512.0,
-      "completions/max_terminated_length": 1512.0,
-      "completions/mean_length": 567.240234375,
-      "completions/mean_terminated_length": 567.240234375,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1877.0,
+      "completions/max_terminated_length": 1877.0,
+      "completions/mean_length": 551.5390625,
+      "completions/mean_terminated_length": 551.5390625,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
       "epoch": 0.2829639579135885,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004133586004795131,
-      "kl": 0.008209228515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06205881843423876,
+      "kl": 0.0543212890625,
       "learning_rate": 1.931213891832153e-06,
-      "loss": 0.0001,
-      "num_tokens": 187631800.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0005,
+      "num_tokens": 200752611.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 316
     },
     {
@@ -8865,24 +8865,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1612.0,
-      "completions/mean_length": 584.296875,
-      "completions/mean_terminated_length": 578.556884765625,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_terminated_length": 1282.0,
+      "completions/mean_length": 569.83984375,
+      "completions/mean_terminated_length": 564.0431518554688,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.2838594134766062,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003874494807536967,
-      "kl": 0.007781982421875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07968967539050426,
+      "kl": 0.0521240234375,
       "learning_rate": 1.9176560428868336e-06,
-      "loss": 0.0001,
-      "num_tokens": 188221264.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0085,
+      "num_tokens": 201334673.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 317
     },
     {
@@ -8892,19 +8892,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1663.0,
-      "completions/max_terminated_length": 1663.0,
-      "completions/mean_length": 614.064453125,
-      "completions/mean_terminated_length": 614.064453125,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_length": 1782.0,
+      "completions/max_terminated_length": 1782.0,
+      "completions/mean_length": 618.275390625,
+      "completions/mean_terminated_length": 618.275390625,
+      "completions/min_length": 222.0,
+      "completions/min_terminated_length": 222.0,
       "epoch": 0.2847548690396239,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0040355129943725355,
-      "kl": 0.007965087890625,
+      "grad_norm": 0.005425161560628047,
+      "kl": 0.054443359375,
       "learning_rate": 1.9041331174269373e-06,
-      "loss": 0.0001,
-      "num_tokens": 188876785.0,
+      "loss": 0.0005,
+      "num_tokens": 201992350.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8919,20 +8919,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1833.0,
-      "completions/max_terminated_length": 1833.0,
-      "completions/mean_length": 561.494140625,
-      "completions/mean_terminated_length": 561.494140625,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1876.0,
+      "completions/mean_length": 564.7734375,
+      "completions/mean_terminated_length": 561.870849609375,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
       "epoch": 0.2856503246026416,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029625994636393927,
-      "kl": 0.00824737548828125,
+      "grad_norm": 0.005142824320868143,
+      "kl": 0.0537109375,
       "learning_rate": 1.8906456828473341e-06,
-      "loss": 0.0001,
-      "num_tokens": 189480766.0,
+      "loss": 0.0005,
+      "num_tokens": 202598010.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8946,21 +8946,21 @@
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1279.0,
-      "completions/max_terminated_length": 1279.0,
-      "completions/mean_length": 589.814453125,
-      "completions/mean_terminated_length": 589.814453125,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1676.0,
+      "completions/mean_length": 582.58984375,
+      "completions/mean_terminated_length": 576.8432006835938,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.2865457801656593,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029549040436407316,
-      "kl": 0.0076751708984375,
+      "grad_norm": 0.009530691407977925,
+      "kl": 0.0511474609375,
       "learning_rate": 1.8771943050537656e-06,
-      "loss": 0.0001,
-      "num_tokens": 190072879.0,
+      "loss": 0.0005,
+      "num_tokens": 203186424.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8976,19 +8976,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1840.0,
-      "completions/max_terminated_length": 1840.0,
-      "completions/mean_length": 582.072265625,
-      "completions/mean_terminated_length": 582.072265625,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/max_length": 1453.0,
+      "completions/max_terminated_length": 1453.0,
+      "completions/mean_length": 580.2578125,
+      "completions/mean_terminated_length": 580.2578125,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.28744123572867697,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004613611713105247,
-      "kl": 0.00787353515625,
+      "grad_norm": 0.005597161597226462,
+      "kl": 0.05303955078125,
       "learning_rate": 1.8637795484391046e-06,
-      "loss": 0.0001,
-      "num_tokens": 190695076.0,
+      "loss": 0.0005,
+      "num_tokens": 203807692.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9003,26 +9003,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1186.0,
-      "completions/mean_length": 563.12890625,
-      "completions/mean_terminated_length": 560.2230834960938,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/max_terminated_length": 1518.0,
+      "completions/mean_length": 560.55859375,
+      "completions/mean_terminated_length": 556.5628051757812,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.28833669129169465,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04918106982067727,
-      "kl": 0.007476806640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 10.190425237517521,
+      "kl": 2.46160888671875,
       "learning_rate": 1.8504019758596698e-06,
-      "loss": 0.0137,
-      "num_tokens": 191306966.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0244,
+      "num_tokens": 204418266.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 322
     },
     {
@@ -9031,20 +9031,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1462.0,
-      "completions/max_terminated_length": 1462.0,
-      "completions/mean_length": 568.73828125,
-      "completions/mean_terminated_length": 567.2211303710938,
-      "completions/min_length": 120.0,
-      "completions/min_terminated_length": 120.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1940.0,
+      "completions/max_terminated_length": 1940.0,
+      "completions/mean_length": 554.6328125,
+      "completions/mean_terminated_length": 554.6328125,
+      "completions/min_length": 104.0,
+      "completions/min_terminated_length": 104.0,
       "epoch": 0.2892321468547123,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005913204495923806,
-      "kl": 0.0136260986328125,
+      "grad_norm": 0.007024406654251954,
+      "kl": 0.0526123046875,
       "learning_rate": 1.8370621486116163e-06,
-      "loss": 0.0001,
-      "num_tokens": 191898832.0,
+      "loss": 0.0005,
+      "num_tokens": 205002910.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9059,20 +9059,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1475.0,
-      "completions/mean_length": 604.5859375,
-      "completions/mean_terminated_length": 601.76123046875,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1875.0,
+      "completions/max_terminated_length": 1875.0,
+      "completions/mean_length": 582.15625,
+      "completions/mean_terminated_length": 582.15625,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.29012760241773,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.029507018255369777,
-      "kl": 0.0104827880859375,
+      "grad_norm": 0.007452427305034759,
+      "kl": 0.0511474609375,
       "learning_rate": 1.823760626407377e-06,
-      "loss": 0.0001,
-      "num_tokens": 192547196.0,
+      "loss": 0.0005,
+      "num_tokens": 205639790.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9087,20 +9087,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1531.0,
-      "completions/mean_length": 630.859375,
-      "completions/mean_terminated_length": 625.302001953125,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1902.0,
+      "completions/max_terminated_length": 1902.0,
+      "completions/mean_length": 606.71484375,
+      "completions/mean_terminated_length": 606.71484375,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.2910230579807477,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024468077068406937,
-      "kl": 0.00716400146484375,
+      "grad_norm": 0.005388949503012748,
+      "kl": 0.0504150390625,
       "learning_rate": 1.8104979673521838e-06,
-      "loss": 0.0001,
-      "num_tokens": 193197412.0,
+      "loss": 0.0005,
+      "num_tokens": 206277644.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9116,25 +9116,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1583.0,
-      "completions/max_terminated_length": 1583.0,
-      "completions/mean_length": 577.087890625,
-      "completions/mean_terminated_length": 577.087890625,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
+      "completions/max_length": 1746.0,
+      "completions/max_terminated_length": 1746.0,
+      "completions/mean_length": 566.4375,
+      "completions/mean_terminated_length": 566.4375,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.2919185135437654,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003442353077357445,
-      "kl": 0.0084228515625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06290666297265979,
+      "kl": 0.05303955078125,
       "learning_rate": 1.7972747279206482e-06,
-      "loss": 0.0001,
-      "num_tokens": 193793681.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0001,
+      "num_tokens": 206868460.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 326
     },
     {
@@ -9144,19 +9144,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1713.0,
-      "completions/max_terminated_length": 1713.0,
-      "completions/mean_length": 577.369140625,
-      "completions/mean_terminated_length": 577.369140625,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_length": 1924.0,
+      "completions/max_terminated_length": 1924.0,
+      "completions/mean_length": 587.607421875,
+      "completions/mean_terminated_length": 587.607421875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.2928139691067831,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032150616476302094,
-      "kl": 0.00772857666015625,
+      "grad_norm": 0.005627099408001751,
+      "kl": 0.05084228515625,
       "learning_rate": 1.7840914629334122e-06,
-      "loss": 0.0001,
-      "num_tokens": 194412430.0,
+      "loss": 0.0005,
+      "num_tokens": 207492451.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9171,20 +9171,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1558.0,
-      "completions/max_terminated_length": 1558.0,
-      "completions/mean_length": 582.974609375,
-      "completions/mean_terminated_length": 582.974609375,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1831.0,
+      "completions/mean_length": 575.02734375,
+      "completions/mean_terminated_length": 572.1448364257812,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.29370942466980077,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031326415879365894,
-      "kl": 0.0074615478515625,
+      "grad_norm": 0.006024866616452142,
+      "kl": 0.05047607421875,
       "learning_rate": 1.7709487255338731e-06,
-      "loss": 0.0001,
-      "num_tokens": 195039553.0,
+      "loss": 0.0005,
+      "num_tokens": 208115505.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9201,24 +9201,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1566.0,
-      "completions/mean_length": 612.119140625,
-      "completions/mean_terminated_length": 606.48828125,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/max_terminated_length": 1685.0,
+      "completions/mean_length": 610.88671875,
+      "completions/mean_terminated_length": 605.2510375976562,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.29460488023281844,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0986403330371718,
-      "kl": 0.00791168212890625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00783507654646286,
+      "kl": 0.0528564453125,
       "learning_rate": 1.7578470671649684e-06,
-      "loss": 0.0137,
-      "num_tokens": 195696782.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 208772103.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 329
     },
     {
@@ -9228,25 +9228,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1522.0,
-      "completions/max_terminated_length": 1522.0,
-      "completions/mean_length": 565.794921875,
-      "completions/mean_terminated_length": 565.794921875,
-      "completions/min_length": 115.0,
-      "completions/min_terminated_length": 115.0,
+      "completions/max_length": 1555.0,
+      "completions/max_terminated_length": 1555.0,
+      "completions/mean_length": 544.369140625,
+      "completions/mean_terminated_length": 544.369140625,
+      "completions/min_length": 148.0,
+      "completions/min_terminated_length": 148.0,
       "epoch": 0.2955003357958361,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0034720570770899726,
-      "kl": 0.00804901123046875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08261649735534114,
+      "kl": 0.0511474609375,
       "learning_rate": 1.744787037546045e-06,
-      "loss": 0.0001,
-      "num_tokens": 196311365.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0142,
+      "num_tokens": 209375716.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 330
     },
     {
@@ -9255,26 +9255,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1588.0,
-      "completions/mean_length": 608.689453125,
-      "completions/mean_terminated_length": 605.872802734375,
-      "completions/min_length": 236.0,
-      "completions/min_terminated_length": 236.0,
+      "completions/max_terminated_length": 1732.0,
+      "completions/mean_length": 595.642578125,
+      "completions/mean_terminated_length": 589.9470825195312,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.2963957913588538,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007254504464050369,
-      "kl": 0.00806427001953125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05439911125987834,
+      "kl": 0.049560546875,
       "learning_rate": 1.731769184649788e-06,
-      "loss": 0.0001,
-      "num_tokens": 196968886.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0093,
+      "num_tokens": 210026557.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 331
     },
     {
@@ -9283,26 +9283,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1772.0,
-      "completions/max_terminated_length": 1772.0,
-      "completions/mean_length": 571.40234375,
-      "completions/mean_terminated_length": 570.3307495117188,
-      "completions/min_length": 137.0,
-      "completions/min_terminated_length": 137.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1854.0,
+      "completions/max_terminated_length": 1854.0,
+      "completions/mean_length": 576.931640625,
+      "completions/mean_terminated_length": 576.931640625,
+      "completions/min_length": 108.0,
+      "completions/min_terminated_length": 108.0,
       "epoch": 0.2972912469218715,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01230074303508439,
-      "kl": 0.0143280029296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03824221393460093,
+      "kl": 0.0489501953125,
       "learning_rate": 1.7187940546792325e-06,
-      "loss": 0.0001,
-      "num_tokens": 197554612.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0039,
+      "num_tokens": 210615114.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 332
     },
     {
@@ -9311,20 +9311,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1974.0,
-      "completions/max_terminated_length": 1974.0,
-      "completions/mean_length": 594.96875,
-      "completions/mean_terminated_length": 594.96875,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1659.0,
+      "completions/mean_length": 583.22265625,
+      "completions/mean_terminated_length": 577.4784545898438,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.2981867024848892,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027166854804102003,
-      "kl": 0.0079498291015625,
+      "grad_norm": 0.00502000048821662,
+      "kl": 0.051513671875,
       "learning_rate": 1.7058621920448465e-06,
-      "loss": 0.0001,
-      "num_tokens": 198166340.0,
+      "loss": 0.0005,
+      "num_tokens": 211220828.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9339,20 +9339,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1840.0,
-      "completions/max_terminated_length": 1840.0,
-      "completions/mean_length": 597.056640625,
-      "completions/mean_terminated_length": 597.056640625,
-      "completions/min_length": 230.0,
-      "completions/min_terminated_length": 230.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1862.0,
+      "completions/mean_length": 604.205078125,
+      "completions/mean_terminated_length": 601.379638671875,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.2990821580479069,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0025843510081473833,
-      "kl": 0.00785064697265625,
+      "grad_norm": 0.009980348313441741,
+      "kl": 0.0511474609375,
       "learning_rate": 1.6929741393416855e-06,
-      "loss": 0.0001,
-      "num_tokens": 198784609.0,
+      "loss": 0.0005,
+      "num_tokens": 211842757.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9367,20 +9367,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1316.0,
-      "completions/max_terminated_length": 1316.0,
-      "completions/mean_length": 555.533203125,
-      "completions/mean_terminated_length": 555.533203125,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1599.0,
+      "completions/mean_length": 558.705078125,
+      "completions/mean_terminated_length": 555.7905883789062,
+      "completions/min_length": 132.0,
+      "completions/min_terminated_length": 132.0,
       "epoch": 0.29997761361092457,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002875308277545933,
-      "kl": 0.0073699951171875,
+      "grad_norm": 0.005398017340368072,
+      "kl": 0.05047607421875,
       "learning_rate": 1.6801304373266286e-06,
-      "loss": 0.0001,
-      "num_tokens": 199349346.0,
+      "loss": 0.0005,
+      "num_tokens": 212409118.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9395,26 +9395,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1431.0,
-      "completions/max_terminated_length": 1431.0,
-      "completions/mean_length": 555.279296875,
-      "completions/mean_terminated_length": 555.279296875,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1582.0,
+      "completions/max_terminated_length": 1582.0,
+      "completions/mean_length": 577.64453125,
+      "completions/mean_terminated_length": 575.9510498046875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.30087306917394224,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026065604510884808,
-      "kl": 0.00782012939453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0400953710083368,
+      "kl": 0.0546875,
       "learning_rate": 1.667331624895689e-06,
-      "loss": 0.0001,
-      "num_tokens": 199946721.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0065,
+      "num_tokens": 213017944.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 336
     },
     {
@@ -9424,25 +9424,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1670.0,
-      "completions/max_terminated_length": 1622.0,
-      "completions/mean_length": 597.9375,
-      "completions/mean_terminated_length": 595.8395385742188,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1681.0,
+      "completions/mean_length": 585.810546875,
+      "completions/mean_terminated_length": 582.9490966796875,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
       "epoch": 0.3017685247369599,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10141160919841502,
-      "kl": 0.009429931640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005855989237846307,
+      "kl": 0.05377197265625,
       "learning_rate": 1.6545782390614037e-06,
-      "loss": 0.0086,
-      "num_tokens": 200548721.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 213613735.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 337
     },
     {
@@ -9451,20 +9451,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1428.0,
-      "completions/max_terminated_length": 1428.0,
-      "completions/mean_length": 546.87109375,
-      "completions/mean_terminated_length": 546.87109375,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1459.0,
+      "completions/max_terminated_length": 1459.0,
+      "completions/mean_length": 552.6015625,
+      "completions/mean_terminated_length": 551.0430297851562,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.3026639802999776,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024956441500498133,
-      "kl": 0.0073699951171875,
+      "grad_norm": 0.021342755030715328,
+      "kl": 0.0576171875,
       "learning_rate": 1.6418708149302992e-06,
-      "loss": 0.0001,
-      "num_tokens": 201095887.0,
+      "loss": 0.0006,
+      "num_tokens": 214163835.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9479,20 +9479,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1574.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 557.408203125,
-      "completions/mean_terminated_length": 557.408203125,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1754.0,
+      "completions/mean_length": 568.935546875,
+      "completions/mean_terminated_length": 566.0410766601562,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.3035594358629953,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002926922407824277,
-      "kl": 0.0082550048828125,
+      "grad_norm": 0.004817435169613062,
+      "kl": 0.05023193359375,
       "learning_rate": 1.6292098856804423e-06,
-      "loss": 0.0001,
-      "num_tokens": 201678320.0,
+      "loss": 0.0005,
+      "num_tokens": 214752170.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9507,26 +9507,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1695.0,
-      "completions/max_terminated_length": 1695.0,
-      "completions/mean_length": 647.646484375,
-      "completions/mean_terminated_length": 647.646484375,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1948.0,
+      "completions/mean_length": 632.53125,
+      "completions/mean_terminated_length": 624.1885986328125,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.304454891426013,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026743176977703474,
-      "kl": 0.00763702392578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.042312900898714305,
+      "kl": 0.05169677734375,
       "learning_rate": 1.6165959825390661e-06,
-      "loss": 0.0001,
-      "num_tokens": 202390603.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0005,
+      "num_tokens": 215456714.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 340
     },
     {
@@ -9535,20 +9535,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2007.0,
-      "completions/mean_length": 597.0546875,
-      "completions/mean_terminated_length": 594.2152709960938,
-      "completions/min_length": 162.0,
-      "completions/min_terminated_length": 162.0,
+      "completions/max_terminated_length": 1992.0,
+      "completions/mean_length": 611.076171875,
+      "completions/mean_terminated_length": 605.4412231445312,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.3053503469890307,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0664857311197945,
-      "kl": 0.00785064697265625,
+      "grad_norm": 0.06514871663082826,
+      "kl": 0.0489501953125,
       "learning_rate": 1.604029634760284e-06,
-      "loss": 0.0131,
-      "num_tokens": 203001159.0,
+      "loss": 0.0068,
+      "num_tokens": 216074449.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -9563,20 +9563,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1380.0,
-      "completions/max_terminated_length": 1380.0,
-      "completions/mean_length": 582.21875,
-      "completions/mean_terminated_length": 582.21875,
-      "completions/min_length": 115.0,
-      "completions/min_terminated_length": 115.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1917.0,
+      "completions/mean_length": 607.349609375,
+      "completions/mean_terminated_length": 604.5303344726562,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.30624580255204836,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0022938907257129934,
-      "kl": 0.00775146484375,
+      "grad_norm": 0.004697869051960354,
+      "kl": 0.04931640625,
       "learning_rate": 1.59151136960288e-06,
-      "loss": 0.0001,
-      "num_tokens": 203628743.0,
+      "loss": 0.0005,
+      "num_tokens": 216714900.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9591,26 +9591,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1719.0,
-      "completions/max_terminated_length": 1719.0,
-      "completions/mean_length": 580.033203125,
-      "completions/mean_terminated_length": 580.033203125,
-      "completions/min_length": 133.0,
-      "completions/min_terminated_length": 133.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1952.0,
+      "completions/mean_length": 585.927734375,
+      "completions/mean_terminated_length": 583.0665283203125,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.30714125811506604,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002789323837930816,
-      "kl": 0.0079498291015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.041363506025014346,
+      "kl": 0.049560546875,
       "learning_rate": 1.5790417123081903e-06,
-      "loss": 0.0001,
-      "num_tokens": 204204616.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0014,
+      "num_tokens": 217293791.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 343
     },
     {
@@ -9619,26 +9619,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1919.0,
-      "completions/mean_length": 554.44140625,
-      "completions/mean_terminated_length": 548.8372802734375,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1563.0,
+      "completions/max_terminated_length": 1563.0,
+      "completions/mean_length": 552.130859375,
+      "completions/mean_terminated_length": 550.9921875,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.3080367136780837,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 20.33983786834859,
-      "kl": 1.365081787109375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.024577605367231227,
+      "kl": 0.05859375,
       "learning_rate": 1.5666211860780583e-06,
-      "loss": 0.027,
-      "num_tokens": 204760426.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 217848418.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 344
     },
     {
@@ -9648,19 +9648,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1682.0,
-      "completions/max_terminated_length": 1682.0,
-      "completions/mean_length": 594.248046875,
-      "completions/mean_terminated_length": 594.248046875,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/max_length": 1901.0,
+      "completions/max_terminated_length": 1901.0,
+      "completions/mean_length": 602.701171875,
+      "completions/mean_terminated_length": 602.701171875,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.3089321692411014,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026992151848093727,
-      "kl": 0.0073699951171875,
+      "grad_norm": 0.004385742088872913,
+      "kl": 0.04888916015625,
       "learning_rate": 1.5542503120528918e-06,
-      "loss": 0.0001,
-      "num_tokens": 205378505.0,
+      "loss": 0.0005,
+      "num_tokens": 218470825.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9675,20 +9675,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1530.0,
-      "completions/max_terminated_length": 1530.0,
-      "completions/mean_length": 588.36328125,
-      "completions/mean_terminated_length": 588.36328125,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1693.0,
+      "completions/mean_length": 612.8984375,
+      "completions/mean_terminated_length": 607.2706298828125,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.3098276248041191,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002201032636526166,
-      "kl": 0.00725555419921875,
+      "grad_norm": 0.004942302367708638,
+      "kl": 0.04833984375,
       "learning_rate": 1.5419296092897866e-06,
-      "loss": 0.0001,
-      "num_tokens": 205952819.0,
+      "loss": 0.0005,
+      "num_tokens": 219057701.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9703,20 +9703,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1891.0,
-      "completions/max_terminated_length": 1891.0,
-      "completions/mean_length": 532.150390625,
-      "completions/mean_terminated_length": 532.150390625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1459.0,
+      "completions/mean_length": 554.208984375,
+      "completions/mean_terminated_length": 551.2857055664062,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.3107230803671368,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002508504389666509,
-      "kl": 0.00791168212890625,
+      "grad_norm": 0.004833275143594728,
+      "kl": 0.0494384765625,
       "learning_rate": 1.529659594740755e-06,
-      "loss": 0.0001,
-      "num_tokens": 206502000.0,
+      "loss": 0.0005,
+      "num_tokens": 219618176.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9731,20 +9731,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1734.0,
-      "completions/max_terminated_length": 1583.0,
-      "completions/mean_length": 599.701171875,
-      "completions/mean_terminated_length": 593.0157470703125,
-      "completions/min_length": 129.0,
-      "completions/min_terminated_length": 129.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1726.0,
+      "completions/max_terminated_length": 1726.0,
+      "completions/mean_length": 615.357421875,
+      "completions/mean_terminated_length": 615.357421875,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.3116185359301545,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09161593323878525,
-      "kl": 0.00812530517578125,
+      "grad_norm": 0.0541376938239617,
+      "kl": 0.0521240234375,
       "learning_rate": 1.5174407832310338e-06,
-      "loss": 0.0083,
-      "num_tokens": 207139527.0,
+      "loss": -0.0004,
+      "num_tokens": 220263719.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -9759,20 +9759,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1606.0,
-      "completions/mean_length": 601.6015625,
-      "completions/mean_terminated_length": 598.7710571289062,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1533.0,
+      "completions/max_terminated_length": 1533.0,
+      "completions/mean_length": 606.06640625,
+      "completions/mean_terminated_length": 606.06640625,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.31251399149317216,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004103618000259221,
-      "kl": 0.0073089599609375,
+      "grad_norm": 0.007838170301132085,
+      "kl": 0.04876708984375,
       "learning_rate": 1.5052736874374815e-06,
-      "loss": 0.0001,
-      "num_tokens": 207764155.0,
+      "loss": 0.0005,
+      "num_tokens": 220890633.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9787,20 +9787,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1784.0,
-      "completions/max_terminated_length": 1784.0,
-      "completions/mean_length": 548.10546875,
-      "completions/mean_terminated_length": 548.10546875,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1796.0,
+      "completions/mean_length": 563.53125,
+      "completions/mean_terminated_length": 557.7098388671875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.31340944705618984,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00228408374001061,
-      "kl": 0.0070343017578125,
+      "grad_norm": 0.0041665151315056406,
+      "kl": 0.04742431640625,
       "learning_rate": 1.4931588178670695e-06,
-      "loss": 0.0001,
-      "num_tokens": 208330257.0,
+      "loss": 0.0005,
+      "num_tokens": 221464633.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9816,19 +9816,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1369.0,
-      "completions/max_terminated_length": 1369.0,
-      "completions/mean_length": 571.197265625,
-      "completions/mean_terminated_length": 571.197265625,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/max_length": 1522.0,
+      "completions/max_terminated_length": 1522.0,
+      "completions/mean_length": 596.26953125,
+      "completions/mean_terminated_length": 596.26953125,
+      "completions/min_length": 218.0,
+      "completions/min_terminated_length": 218.0,
       "epoch": 0.3143049026192075,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0020964969256138085,
-      "kl": 0.00688934326171875,
+      "grad_norm": 0.004236470661639679,
+      "kl": 0.0465087890625,
       "learning_rate": 1.4810966828354605e-06,
-      "loss": 0.0001,
-      "num_tokens": 208976598.0,
+      "loss": 0.0005,
+      "num_tokens": 222123811.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9844,19 +9844,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1443.0,
-      "completions/max_terminated_length": 1443.0,
-      "completions/mean_length": 561.779296875,
-      "completions/mean_terminated_length": 561.779296875,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/max_length": 1732.0,
+      "completions/max_terminated_length": 1732.0,
+      "completions/mean_length": 579.677734375,
+      "completions/mean_terminated_length": 579.677734375,
+      "completions/min_length": 107.0,
+      "completions/min_terminated_length": 107.0,
       "epoch": 0.3152003581822252,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003302003902452213,
-      "kl": 0.0074615478515625,
+      "grad_norm": 0.004236180586681027,
+      "kl": 0.0506591796875,
       "learning_rate": 1.469087788445684e-06,
-      "loss": 0.0001,
-      "num_tokens": 209568197.0,
+      "loss": 0.0005,
+      "num_tokens": 222724574.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9872,19 +9872,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1252.0,
-      "completions/max_terminated_length": 1252.0,
-      "completions/mean_length": 548.845703125,
-      "completions/mean_terminated_length": 548.845703125,
+      "completions/max_length": 1603.0,
+      "completions/max_terminated_length": 1603.0,
+      "completions/mean_length": 563.109375,
+      "completions/mean_terminated_length": 563.109375,
       "completions/min_length": 191.0,
       "completions/min_terminated_length": 191.0,
       "epoch": 0.31609581374524287,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002776716601889407,
-      "kl": 0.00762176513671875,
+      "grad_norm": 0.004163722780438454,
+      "kl": 0.04901123046875,
       "learning_rate": 1.4571326385668965e-06,
-      "loss": 0.0001,
-      "num_tokens": 210177830.0,
+      "loss": 0.0005,
+      "num_tokens": 223341510.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9900,25 +9900,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1176.0,
-      "completions/max_terminated_length": 1176.0,
-      "completions/mean_length": 563.796875,
-      "completions/mean_terminated_length": 563.796875,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/max_length": 1976.0,
+      "completions/max_terminated_length": 1976.0,
+      "completions/mean_length": 594.8515625,
+      "completions/mean_terminated_length": 594.8515625,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.3169912693082606,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0022599035175075058,
-      "kl": 0.007415771484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05815166195072682,
+      "kl": 0.04833984375,
       "learning_rate": 1.4452317348132434e-06,
-      "loss": 0.0001,
-      "num_tokens": 210781886.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0017,
+      "num_tokens": 223961466.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 354
     },
     {
@@ -9927,26 +9927,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1258.0,
-      "completions/max_terminated_length": 1258.0,
-      "completions/mean_length": 558.203125,
-      "completions/mean_terminated_length": 558.203125,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1820.0,
+      "completions/mean_length": 580.19921875,
+      "completions/mean_terminated_length": 574.4431762695312,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
       "epoch": 0.3178867248712783,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0020322560028473275,
-      "kl": 0.00682830810546875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.11420371586495477,
+      "kl": 0.04827880859375,
       "learning_rate": 1.4333855765228104e-06,
-      "loss": 0.0001,
-      "num_tokens": 211373974.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0157,
+      "num_tokens": 224564816.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 355
     },
     {
@@ -9955,20 +9955,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1791.0,
-      "completions/max_terminated_length": 1791.0,
-      "completions/mean_length": 622.130859375,
-      "completions/mean_terminated_length": 622.130859375,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1698.0,
+      "completions/mean_length": 630.931640625,
+      "completions/mean_terminated_length": 628.1585083007812,
+      "completions/min_length": 221.0,
+      "completions/min_terminated_length": 221.0,
       "epoch": 0.31878218043429596,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0021667727258320112,
-      "kl": 0.0065765380859375,
+      "grad_norm": 0.004304656769130002,
+      "kl": 0.0465087890625,
       "learning_rate": 1.421594660736675e-06,
-      "loss": 0.0001,
-      "num_tokens": 212004009.0,
+      "loss": 0.0005,
+      "num_tokens": 225199357.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9984,19 +9984,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1612.0,
-      "completions/max_terminated_length": 1612.0,
-      "completions/mean_length": 534.94140625,
-      "completions/mean_terminated_length": 534.94140625,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/max_length": 1453.0,
+      "completions/max_terminated_length": 1453.0,
+      "completions/mean_length": 544.44140625,
+      "completions/mean_terminated_length": 544.44140625,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.31967763599731364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002663417489634053,
-      "kl": 0.00749969482421875,
+      "grad_norm": 0.004792128064874122,
+      "kl": 0.05010986328125,
       "learning_rate": 1.4098594821780476e-06,
-      "loss": 0.0001,
-      "num_tokens": 212552091.0,
+      "loss": 0.0005,
+      "num_tokens": 225752303.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10012,25 +10012,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1600.0,
-      "completions/max_terminated_length": 1600.0,
-      "completions/mean_length": 564.744140625,
-      "completions/mean_terminated_length": 564.744140625,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/max_length": 1750.0,
+      "completions/max_terminated_length": 1750.0,
+      "completions/mean_length": 569.7265625,
+      "completions/mean_terminated_length": 569.7265625,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.3205730915603313,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0031061900289884826,
-      "kl": 0.00829315185546875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07939089082655287,
+      "kl": 0.04949951171875,
       "learning_rate": 1.3981805332315174e-06,
       "loss": 0.0001,
-      "num_tokens": 213155240.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "num_tokens": 226358003.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 358
     },
     {
@@ -10040,19 +10040,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1436.0,
-      "completions/max_terminated_length": 1436.0,
-      "completions/mean_length": 571.017578125,
-      "completions/mean_terminated_length": 571.017578125,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/max_length": 1859.0,
+      "completions/max_terminated_length": 1859.0,
+      "completions/mean_length": 586.755859375,
+      "completions/mean_terminated_length": 586.755859375,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.321468547123349,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0021493978916938975,
-      "kl": 0.006927490234375,
+      "grad_norm": 0.004747531699904538,
+      "kl": 0.04705810546875,
       "learning_rate": 1.3865583039223929e-06,
-      "loss": 0.0001,
-      "num_tokens": 213744049.0,
+      "loss": 0.0005,
+      "num_tokens": 226954870.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10067,26 +10067,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1491.0,
-      "completions/max_terminated_length": 1491.0,
-      "completions/mean_length": 547.46875,
-      "completions/mean_terminated_length": 547.46875,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1848.0,
+      "completions/mean_length": 561.67578125,
+      "completions/mean_terminated_length": 558.76708984375,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.32236400268636667,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002390722572560558,
-      "kl": 0.00682830810546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05340801461414318,
+      "kl": 0.0465087890625,
       "learning_rate": 1.374993281896137e-06,
-      "loss": 0.0001,
-      "num_tokens": 214323393.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0166,
+      "num_tokens": 227541488.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 360
     },
     {
@@ -10095,20 +10095,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1235.0,
-      "completions/mean_length": 530.275390625,
-      "completions/mean_terminated_length": 524.3235473632812,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1943.0,
+      "completions/max_terminated_length": 1943.0,
+      "completions/mean_length": 549.7890625,
+      "completions/mean_terminated_length": 549.7890625,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.3232594582493844,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002902768186402617,
-      "kl": 0.00717926025390625,
+      "grad_norm": 0.004837436430436778,
+      "kl": 0.0484619140625,
       "learning_rate": 1.3634859523979134e-06,
-      "loss": 0.0001,
-      "num_tokens": 214877838.0,
+      "loss": 0.0005,
+      "num_tokens": 228105924.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10124,19 +10124,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1317.0,
-      "completions/max_terminated_length": 1317.0,
-      "completions/mean_length": 566.4140625,
-      "completions/mean_terminated_length": 566.4140625,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/max_length": 1845.0,
+      "completions/max_terminated_length": 1845.0,
+      "completions/mean_length": 572.677734375,
+      "completions/mean_terminated_length": 572.677734375,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
       "epoch": 0.3241549138124021,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002136628038833396,
-      "kl": 0.00792694091796875,
+      "grad_norm": 0.0056382167382188035,
+      "kl": 0.04815673828125,
       "learning_rate": 1.3520367982522208e-06,
-      "loss": 0.0001,
-      "num_tokens": 215453682.0,
+      "loss": 0.0005,
+      "num_tokens": 228684975.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10153,18 +10153,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1374.0,
-      "completions/mean_length": 557.8984375,
-      "completions/mean_terminated_length": 554.9823608398438,
-      "completions/min_length": 229.0,
-      "completions/min_terminated_length": 229.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 548.544921875,
+      "completions/mean_terminated_length": 545.6105346679688,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
       "epoch": 0.32505036937541976,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002198479192435782,
-      "kl": 0.00743865966796875,
+      "grad_norm": 0.0067283143933235,
+      "kl": 0.04949951171875,
       "learning_rate": 1.3406462998426358e-06,
-      "loss": 0.0001,
-      "num_tokens": 216022158.0,
+      "loss": 0.0005,
+      "num_tokens": 229248662.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10179,26 +10179,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1581.0,
-      "completions/max_terminated_length": 1581.0,
-      "completions/mean_length": 619.244140625,
-      "completions/mean_terminated_length": 619.244140625,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1774.0,
+      "completions/mean_length": 621.30078125,
+      "completions/mean_terminated_length": 615.7059326171875,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
       "epoch": 0.32594582493843743,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002120261143082215,
-      "kl": 0.0067596435546875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.08503371465958129,
+      "kl": 0.04840087890625,
       "learning_rate": 1.3293149350916595e-06,
-      "loss": 0.0001,
-      "num_tokens": 216656363.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0103,
+      "num_tokens": 229883920.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 364
     },
     {
@@ -10207,20 +10207,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2000.0,
-      "completions/mean_length": 580.7578125,
-      "completions/mean_terminated_length": 577.886474609375,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1562.0,
+      "completions/max_terminated_length": 1562.0,
+      "completions/mean_length": 595.625,
+      "completions/mean_terminated_length": 595.625,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.3268412805014551,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008873138088250711,
-      "kl": 0.00872802734375,
+      "grad_norm": 0.0068623792592217995,
+      "kl": 0.04730224609375,
       "learning_rate": 1.3180431794406623e-06,
-      "loss": 0.0001,
-      "num_tokens": 217253119.0,
+      "loss": 0.0005,
+      "num_tokens": 230488288.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10236,19 +10236,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1617.0,
-      "completions/max_terminated_length": 1617.0,
-      "completions/mean_length": 579.234375,
-      "completions/mean_terminated_length": 579.234375,
-      "completions/min_length": 124.0,
-      "completions/min_terminated_length": 124.0,
+      "completions/max_length": 1877.0,
+      "completions/max_terminated_length": 1877.0,
+      "completions/mean_length": 582.458984375,
+      "completions/mean_terminated_length": 582.458984375,
+      "completions/min_length": 127.0,
+      "completions/min_terminated_length": 127.0,
       "epoch": 0.3277367360644728,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0021260611537859343,
-      "kl": 0.00704193115234375,
+      "grad_norm": 0.018560148818948925,
+      "kl": 0.04766845703125,
       "learning_rate": 1.3068315058299358e-06,
-      "loss": 0.0001,
-      "num_tokens": 217849367.0,
+      "loss": 0.0005,
+      "num_tokens": 231086187.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10263,20 +10263,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1519.0,
-      "completions/max_terminated_length": 1519.0,
-      "completions/mean_length": 589.966796875,
-      "completions/mean_terminated_length": 589.966796875,
-      "completions/min_length": 246.0,
-      "completions/min_terminated_length": 246.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1877.0,
+      "completions/mean_length": 613.73046875,
+      "completions/mean_terminated_length": 608.1058959960938,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.32863219162749047,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002204999600811044,
-      "kl": 0.00702667236328125,
+      "grad_norm": 0.006036312025660227,
+      "kl": 0.0472412109375,
       "learning_rate": 1.2956803846788503e-06,
-      "loss": 0.0001,
-      "num_tokens": 218453718.0,
+      "loss": 0.0005,
+      "num_tokens": 231702705.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10293,18 +10293,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1940.0,
-      "completions/mean_length": 576.056640625,
-      "completions/mean_terminated_length": 573.1761474609375,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_terminated_length": 1561.0,
+      "completions/mean_length": 575.67578125,
+      "completions/mean_terminated_length": 572.7944946289062,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.32952764719050814,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00232950684589546,
-      "kl": 0.00756072998046875,
+      "grad_norm": 0.010626533398767141,
+      "kl": 0.04766845703125,
       "learning_rate": 1.284590283866116e-06,
-      "loss": 0.0001,
-      "num_tokens": 219065395.0,
+      "loss": 0.0005,
+      "num_tokens": 232314187.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10320,19 +10320,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1794.0,
-      "completions/max_terminated_length": 1794.0,
-      "completions/mean_length": 629.060546875,
-      "completions/mean_terminated_length": 629.060546875,
-      "completions/min_length": 216.0,
-      "completions/min_terminated_length": 216.0,
+      "completions/max_length": 1877.0,
+      "completions/max_terminated_length": 1877.0,
+      "completions/mean_length": 638.849609375,
+      "completions/mean_terminated_length": 638.849609375,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.3304231027535259,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0021195870410575485,
-      "kl": 0.006866455078125,
+      "grad_norm": 0.005441410501154126,
+      "kl": 0.04730224609375,
       "learning_rate": 1.2735616687101518e-06,
-      "loss": 0.0001,
-      "num_tokens": 219763346.0,
+      "loss": 0.0005,
+      "num_tokens": 233017150.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10347,20 +10347,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1389.0,
-      "completions/max_terminated_length": 1389.0,
-      "completions/mean_length": 570.484375,
-      "completions/mean_terminated_length": 570.484375,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1908.0,
+      "completions/mean_length": 603.595703125,
+      "completions/mean_terminated_length": 595.08251953125,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.33131855831654355,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024615791853039603,
-      "kl": 0.00725555419921875,
+      "grad_norm": 0.04062997694360191,
+      "kl": 0.04681396484375,
       "learning_rate": 1.2625950019495614e-06,
-      "loss": 0.0001,
-      "num_tokens": 220362634.0,
+      "loss": 0.0005,
+      "num_tokens": 233633391.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10376,19 +10376,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1190.0,
-      "completions/max_terminated_length": 1190.0,
-      "completions/mean_length": 573.626953125,
-      "completions/mean_terminated_length": 573.626953125,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/max_length": 1448.0,
+      "completions/max_terminated_length": 1448.0,
+      "completions/mean_length": 572.203125,
+      "completions/mean_terminated_length": 572.203125,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.33221401387956123,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002016962022433684,
-      "kl": 0.00699615478515625,
+      "grad_norm": 0.00508915950543525,
+      "kl": 0.04736328125,
       "learning_rate": 1.251690743723718e-06,
-      "loss": 0.0001,
-      "num_tokens": 220964347.0,
+      "loss": 0.0005,
+      "num_tokens": 234234375.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10404,19 +10404,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1322.0,
-      "completions/max_terminated_length": 1322.0,
-      "completions/mean_length": 588.921875,
-      "completions/mean_terminated_length": 588.921875,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/max_length": 1783.0,
+      "completions/max_terminated_length": 1783.0,
+      "completions/mean_length": 571.640625,
+      "completions/mean_terminated_length": 571.640625,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.3331094694425789,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019903531026202316,
-      "kl": 0.00666046142578125,
+      "grad_norm": 0.005649723189142395,
+      "kl": 0.04681396484375,
       "learning_rate": 1.2408493515534581e-06,
-      "loss": 0.0001,
-      "num_tokens": 221552067.0,
+      "loss": 0.0005,
+      "num_tokens": 234813247.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10432,25 +10432,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1343.0,
-      "completions/max_terminated_length": 1343.0,
-      "completions/mean_length": 589.775390625,
-      "completions/mean_terminated_length": 589.775390625,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1921.0,
+      "completions/max_terminated_length": 1921.0,
+      "completions/mean_length": 582.796875,
+      "completions/mean_terminated_length": 582.796875,
+      "completions/min_length": 148.0,
+      "completions/min_terminated_length": 148.0,
       "epoch": 0.3340049250055966,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002242842785657926,
-      "kl": 0.00731658935546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07817677753397509,
+      "kl": 0.04779052734375,
       "learning_rate": 1.2300712803218834e-06,
-      "loss": 0.0001,
-      "num_tokens": 222143152.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0039,
+      "num_tokens": 235400759.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 373
     },
     {
@@ -10459,20 +10459,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1534.0,
-      "completions/max_terminated_length": 1534.0,
-      "completions/mean_length": 558.314453125,
-      "completions/mean_terminated_length": 558.314453125,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1730.0,
+      "completions/mean_length": 561.732421875,
+      "completions/mean_terminated_length": 558.8238525390625,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
       "epoch": 0.33490038056861426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003060731538706847,
-      "kl": 0.007781982421875,
+      "grad_norm": 0.005750730935584701,
+      "kl": 0.04840087890625,
       "learning_rate": 1.2193569822552772e-06,
-      "loss": 0.0001,
-      "num_tokens": 222742897.0,
+      "loss": 0.0005,
+      "num_tokens": 236002254.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10488,19 +10488,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1411.0,
-      "completions/max_terminated_length": 1411.0,
-      "completions/mean_length": 582.775390625,
-      "completions/mean_terminated_length": 582.775390625,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1538.0,
+      "completions/mean_length": 581.84765625,
+      "completions/mean_terminated_length": 581.84765625,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.33579583613163194,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0017875770315972821,
-      "kl": 0.0068359375,
+      "grad_norm": 0.005148892348158003,
+      "kl": 0.0478515625,
       "learning_rate": 1.2087069069041268e-06,
-      "loss": 0.0001,
-      "num_tokens": 223338670.0,
+      "loss": 0.0005,
+      "num_tokens": 236597552.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10515,26 +10515,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1358.0,
-      "completions/max_terminated_length": 1358.0,
-      "completions/mean_length": 617.2421875,
-      "completions/mean_terminated_length": 617.2421875,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1951.0,
+      "completions/mean_length": 613.890625,
+      "completions/mean_terminated_length": 608.7745361328125,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
       "epoch": 0.3366912916946497,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010428174741634343,
-      "kl": 0.00791168212890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 6.1250219766271465,
+      "kl": 0.0712890625,
       "learning_rate": 1.1981215011242654e-06,
-      "loss": 0.0001,
-      "num_tokens": 223964938.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0117,
+      "num_tokens": 237222104.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 376
     },
     {
@@ -10544,19 +10544,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1244.0,
-      "completions/max_terminated_length": 1244.0,
-      "completions/mean_length": 585.546875,
-      "completions/mean_terminated_length": 585.546875,
-      "completions/min_length": 108.0,
-      "completions/min_terminated_length": 108.0,
+      "completions/max_length": 1886.0,
+      "completions/max_terminated_length": 1886.0,
+      "completions/mean_length": 586.263671875,
+      "completions/mean_terminated_length": 586.263671875,
+      "completions/min_length": 112.0,
+      "completions/min_terminated_length": 112.0,
       "epoch": 0.33758674725766735,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0020485851533229055,
-      "kl": 0.00693511962890625,
+      "grad_norm": 0.00577756812887239,
+      "kl": 0.04656982421875,
       "learning_rate": 1.1876012090581184e-06,
-      "loss": 0.0001,
-      "num_tokens": 224578690.0,
+      "loss": 0.0005,
+      "num_tokens": 237836223.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10571,20 +10571,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1708.0,
-      "completions/max_terminated_length": 1708.0,
-      "completions/mean_length": 586.498046875,
-      "completions/mean_terminated_length": 586.498046875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1996.0,
+      "completions/max_terminated_length": 1996.0,
+      "completions/mean_length": 609.84765625,
+      "completions/mean_terminated_length": 607.878662109375,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.33848220282068503,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0023864026164809745,
-      "kl": 0.00688934326171875,
+      "grad_norm": 0.02365896656045564,
+      "kl": 0.0654296875,
       "learning_rate": 1.177146472116071e-06,
-      "loss": 0.0001,
-      "num_tokens": 225193745.0,
+      "loss": 0.0007,
+      "num_tokens": 238463233.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10599,26 +10599,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1357.0,
-      "completions/mean_length": 572.3984375,
-      "completions/mean_terminated_length": 569.5107421875,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1808.0,
+      "completions/max_terminated_length": 1808.0,
+      "completions/mean_length": 585.064453125,
+      "completions/mean_terminated_length": 585.064453125,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.3393776583837027,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002935718694507386,
-      "kl": 0.00739288330078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07355594175930143,
+      "kl": 0.050048828125,
       "learning_rate": 1.1667577289579462e-06,
-      "loss": 0.0001,
-      "num_tokens": 225800349.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0049,
+      "num_tokens": 239076322.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 379
     },
     {
@@ -10629,18 +10629,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1268.0,
-      "completions/mean_length": 584.03515625,
-      "completions/mean_terminated_length": 581.1702270507812,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/max_terminated_length": 1374.0,
+      "completions/mean_length": 589.796875,
+      "completions/mean_terminated_length": 586.9432373046875,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.3402731139467204,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0020063036346906656,
-      "kl": 0.00696563720703125,
+      "grad_norm": 0.005923305178076053,
+      "kl": 0.04888916015625,
       "learning_rate": 1.1564354154746007e-06,
-      "loss": 0.0001,
-      "num_tokens": 226407791.0,
+      "loss": 0.0005,
+      "num_tokens": 239686714.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10655,20 +10655,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1583.0,
-      "completions/mean_length": 567.478515625,
-      "completions/mean_terminated_length": 561.672607421875,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1704.0,
+      "completions/max_terminated_length": 1704.0,
+      "completions/mean_length": 565.86328125,
+      "completions/mean_terminated_length": 565.86328125,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.34116856950973806,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05304798394475104,
-      "kl": 0.00707244873046875,
+      "grad_norm": 0.06055954764239639,
+      "kl": 0.04974365234375,
       "learning_rate": 1.146179964769635e-06,
-      "loss": 0.0112,
-      "num_tokens": 226991780.0,
+      "loss": 0.0006,
+      "num_tokens": 240269876.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -10683,26 +10683,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1395.0,
-      "completions/max_terminated_length": 1395.0,
-      "completions/mean_length": 587.37890625,
-      "completions/mean_terminated_length": 587.37890625,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1780.0,
+      "completions/mean_length": 604.6328125,
+      "completions/mean_terminated_length": 601.8082275390625,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.34206402507275574,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0017700420725100794,
-      "kl": 0.0064697265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.057122471938020274,
+      "kl": 0.0457763671875,
       "learning_rate": 1.1359918071412195e-06,
-      "loss": 0.0001,
-      "num_tokens": 227580630.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0114,
+      "num_tokens": 240867560.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 382
     },
     {
@@ -10711,26 +10711,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1330.0,
-      "completions/max_terminated_length": 1330.0,
-      "completions/mean_length": 574.92578125,
-      "completions/mean_terminated_length": 574.92578125,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1869.0,
+      "completions/mean_length": 574.115234375,
+      "completions/mean_terminated_length": 568.3353271484375,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.3429594806357735,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002522387931367739,
-      "kl": 0.00714111328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0652873493200889,
+      "kl": 0.04638671875,
       "learning_rate": 1.1258713700640456e-06,
-      "loss": 0.0001,
-      "num_tokens": 228158640.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.014,
+      "num_tokens": 241445155.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 383
     },
     {
@@ -10739,20 +10739,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1440.0,
-      "completions/mean_length": 576.966796875,
-      "completions/mean_terminated_length": 574.0880737304688,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1976.0,
+      "completions/max_terminated_length": 1976.0,
+      "completions/mean_length": 588.0546875,
+      "completions/mean_terminated_length": 588.0546875,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.34385493619879115,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0018472409757711492,
-      "kl": 0.00701141357421875,
+      "grad_norm": 0.004754868273249351,
+      "kl": 0.047607421875,
       "learning_rate": 1.115819078171383e-06,
-      "loss": 0.0001,
-      "num_tokens": 228741151.0,
+      "loss": 0.0005,
+      "num_tokens": 242033343.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10768,19 +10768,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1622.0,
-      "completions/max_terminated_length": 1622.0,
-      "completions/mean_length": 585.87109375,
-      "completions/mean_terminated_length": 585.87109375,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/max_length": 1922.0,
+      "completions/max_terminated_length": 1922.0,
+      "completions/mean_length": 574.859375,
+      "completions/mean_terminated_length": 574.859375,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.3447503917618088,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019917553098233345,
-      "kl": 0.00733184814453125,
+      "grad_norm": 0.0056589735897611725,
+      "kl": 0.05059814453125,
       "learning_rate": 1.1058353532372667e-06,
-      "loss": 0.0001,
-      "num_tokens": 229333405.0,
+      "loss": 0.0005,
+      "num_tokens": 242619959.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10795,26 +10795,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1603.0,
-      "completions/max_terminated_length": 1603.0,
-      "completions/mean_length": 579.091796875,
-      "completions/mean_terminated_length": 579.091796875,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1501.0,
+      "completions/mean_length": 573.388671875,
+      "completions/mean_terminated_length": 563.56103515625,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.3456458473248265,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02362205706260728,
-      "kl": 0.0114593505859375,
-      "learning_rate": 1.0959206141587998e-06,
-      "loss": 0.0001,
-      "num_tokens": 229945020.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.11311586362691683,
+      "kl": 0.0516357421875,
+      "learning_rate": 1.0959206141587998e-06,
+      "loss": 0.0303,
+      "num_tokens": 243228654.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 386
     },
     {
@@ -10824,19 +10824,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1386.0,
-      "completions/max_terminated_length": 1386.0,
-      "completions/mean_length": 579.376953125,
-      "completions/mean_terminated_length": 579.376953125,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/max_length": 2034.0,
+      "completions/max_terminated_length": 2034.0,
+      "completions/mean_length": 582.330078125,
+      "completions/mean_terminated_length": 582.330078125,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.3465413028878442,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00198843994240326,
-      "kl": 0.0070648193359375,
+      "grad_norm": 0.005083694085424717,
+      "kl": 0.04852294921875,
       "learning_rate": 1.0860752769385766e-06,
-      "loss": 0.0001,
-      "num_tokens": 230530429.0,
+      "loss": 0.0005,
+      "num_tokens": 243815575.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10852,19 +10852,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1863.0,
-      "completions/max_terminated_length": 1863.0,
-      "completions/mean_length": 632.87109375,
-      "completions/mean_terminated_length": 632.87109375,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/max_length": 1655.0,
+      "completions/max_terminated_length": 1655.0,
+      "completions/mean_length": 608.525390625,
+      "completions/mean_terminated_length": 608.525390625,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.34743675845086186,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06881464441737437,
-      "kl": 0.00665283203125,
+      "grad_norm": 0.05526627161129057,
+      "kl": 0.0477294921875,
       "learning_rate": 1.0762997546672279e-06,
-      "loss": 0.01,
-      "num_tokens": 231155211.0,
+      "loss": 0.0015,
+      "num_tokens": 244427892.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -10880,25 +10880,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1730.0,
-      "completions/max_terminated_length": 1730.0,
-      "completions/mean_length": 556.984375,
-      "completions/mean_terminated_length": 556.984375,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/max_length": 1761.0,
+      "completions/max_terminated_length": 1761.0,
+      "completions/mean_length": 562.423828125,
+      "completions/mean_terminated_length": 562.423828125,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.34833221401387954,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003016609752120695,
-      "kl": 0.00732421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08080507072024797,
+      "kl": 0.048583984375,
       "learning_rate": 1.0665944575060914e-06,
-      "loss": 0.0001,
-      "num_tokens": 231748979.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.004,
+      "num_tokens": 245024445.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 389
     },
     {
@@ -10908,19 +10908,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1937.0,
-      "completions/max_terminated_length": 1937.0,
-      "completions/mean_length": 608.6484375,
-      "completions/mean_terminated_length": 608.6484375,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_length": 1832.0,
+      "completions/max_terminated_length": 1832.0,
+      "completions/mean_length": 586.203125,
+      "completions/mean_terminated_length": 586.203125,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.34922766957689727,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0022898321321825014,
-      "kl": 0.007659912109375,
+      "grad_norm": 0.009200464557926759,
+      "kl": 0.0494384765625,
       "learning_rate": 1.056959792669997e-06,
-      "loss": 0.0001,
-      "num_tokens": 232377743.0,
+      "loss": 0.0005,
+      "num_tokens": 245641717.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10935,20 +10935,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1715.0,
-      "completions/mean_length": 600.947265625,
-      "completions/mean_terminated_length": 595.2725830078125,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1634.0,
+      "completions/max_terminated_length": 1634.0,
+      "completions/mean_length": 597.11328125,
+      "completions/mean_terminated_length": 597.11328125,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.35012312513991495,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07120278792436081,
-      "kl": 0.007080078125,
+      "grad_norm": 0.05477299161437103,
+      "kl": 0.04791259765625,
       "learning_rate": 1.0473961644101856e-06,
-      "loss": 0.0087,
-      "num_tokens": 233026884.0,
+      "loss": 0.0018,
+      "num_tokens": 246288895.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -10963,26 +10963,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1701.0,
-      "completions/max_terminated_length": 1701.0,
-      "completions/mean_length": 617.69921875,
-      "completions/mean_terminated_length": 617.69921875,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1455.0,
+      "completions/mean_length": 610.310546875,
+      "completions/mean_terminated_length": 607.4970703125,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.3510185807029326,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002037737372292383,
-      "kl": 0.0066375732421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04103458864719703,
+      "kl": 0.047607421875,
       "learning_rate": 1.037903973997345e-06,
-      "loss": 0.0001,
-      "num_tokens": 233682138.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 246940366.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 392
     },
     {
@@ -10991,26 +10991,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1986.0,
-      "completions/mean_length": 600.583984375,
-      "completions/mean_terminated_length": 594.0628662109375,
-      "completions/min_length": 130.0,
-      "completions/min_terminated_length": 130.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1687.0,
+      "completions/max_terminated_length": 1687.0,
+      "completions/mean_length": 571.6171875,
+      "completions/mean_terminated_length": 571.6171875,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.3519140362659503,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 7.623259217540254,
-      "kl": 0.02263641357421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0045559010772451055,
+      "kl": 0.04620361328125,
       "learning_rate": 1.0284836197047737e-06,
-      "loss": 0.0048,
-      "num_tokens": 234310757.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 247554154.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 393
     },
     {
@@ -11020,19 +11020,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1556.0,
-      "completions/max_terminated_length": 1556.0,
-      "completions/mean_length": 554.560546875,
-      "completions/mean_terminated_length": 554.560546875,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/max_length": 1482.0,
+      "completions/max_terminated_length": 1482.0,
+      "completions/mean_length": 529.73046875,
+      "completions/mean_terminated_length": 529.73046875,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.352809491828968,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0028379250785953714,
-      "kl": 0.0075531005859375,
+      "grad_norm": 0.005620991916216765,
+      "kl": 0.05078125,
       "learning_rate": 1.0191354967916712e-06,
-      "loss": 0.0001,
-      "num_tokens": 234895028.0,
+      "loss": 0.0005,
+      "num_tokens": 248125712.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11049,18 +11049,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1442.0,
-      "completions/mean_length": 592.939453125,
-      "completions/mean_terminated_length": 590.0919799804688,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/max_terminated_length": 1438.0,
+      "completions/mean_length": 577.833984375,
+      "completions/mean_terminated_length": 574.9569702148438,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.35370494739198566,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002487976285381569,
-      "kl": 0.00714111328125,
+      "grad_norm": 0.005897057155196266,
+      "kl": 0.05084228515625,
       "learning_rate": 1.0098599974865515e-06,
-      "loss": 0.0001,
-      "num_tokens": 235493429.0,
+      "loss": 0.0005,
+      "num_tokens": 248716379.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11076,19 +11076,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1766.0,
-      "completions/max_terminated_length": 1766.0,
-      "completions/mean_length": 620.3359375,
-      "completions/mean_terminated_length": 620.3359375,
-      "completions/min_length": 221.0,
-      "completions/min_terminated_length": 221.0,
+      "completions/max_length": 1838.0,
+      "completions/max_terminated_length": 1838.0,
+      "completions/mean_length": 607.990234375,
+      "completions/mean_terminated_length": 607.990234375,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.35460040295500334,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019206177258053912,
-      "kl": 0.0068206787109375,
+      "grad_norm": 0.005227169861081531,
+      "kl": 0.04913330078125,
       "learning_rate": 1.0006575109707898e-06,
-      "loss": 0.0001,
-      "num_tokens": 236136481.0,
+      "loss": 0.0005,
+      "num_tokens": 249353110.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11104,19 +11104,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1601.0,
-      "completions/max_terminated_length": 1601.0,
-      "completions/mean_length": 569.744140625,
-      "completions/mean_terminated_length": 569.744140625,
-      "completions/min_length": 133.0,
-      "completions/min_terminated_length": 133.0,
+      "completions/max_length": 1525.0,
+      "completions/max_terminated_length": 1525.0,
+      "completions/mean_length": 534.1484375,
+      "completions/mean_terminated_length": 534.1484375,
+      "completions/min_length": 97.0,
+      "completions/min_terminated_length": 97.0,
       "epoch": 0.35549585851802107,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002326401178225782,
-      "kl": 0.00687408447265625,
+      "grad_norm": 0.009091379046012489,
+      "kl": 0.04925537109375,
       "learning_rate": 9.915284233622877e-07,
-      "loss": 0.0001,
-      "num_tokens": 236744702.0,
+      "loss": 0.0005,
+      "num_tokens": 249943106.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11132,19 +11132,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1616.0,
-      "completions/max_terminated_length": 1616.0,
-      "completions/mean_length": 577.216796875,
-      "completions/mean_terminated_length": 577.216796875,
-      "completions/min_length": 117.0,
-      "completions/min_terminated_length": 117.0,
+      "completions/max_length": 1687.0,
+      "completions/max_terminated_length": 1687.0,
+      "completions/mean_length": 554.1640625,
+      "completions/mean_terminated_length": 554.1640625,
+      "completions/min_length": 110.0,
+      "completions/min_terminated_length": 110.0,
       "epoch": 0.35639131408103875,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030862369767653917,
-      "kl": 0.006591796875,
+      "grad_norm": 0.00576347525077593,
+      "kl": 0.04864501953125,
       "learning_rate": 9.824731176992796e-07,
-      "loss": 0.0001,
-      "num_tokens": 237305933.0,
+      "loss": 0.0005,
+      "num_tokens": 250492534.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11159,26 +11159,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1612.0,
-      "completions/mean_length": 593.181640625,
-      "completions/mean_terminated_length": 587.4765014648438,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1398.0,
+      "completions/max_terminated_length": 1398.0,
+      "completions/mean_length": 580.962890625,
+      "completions/mean_terminated_length": 580.962890625,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.3572867696440564,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.11199865549877461,
-      "kl": 0.0068817138671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005684393564410186,
+      "kl": 0.05072021484375,
       "learning_rate": 9.734919739242543e-07,
-      "loss": 0.0132,
-      "num_tokens": 237912938.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0005,
+      "num_tokens": 251093283.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 399
     },
     {
@@ -11187,26 +11187,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1449.0,
-      "completions/max_terminated_length": 1449.0,
-      "completions/mean_length": 616.73828125,
-      "completions/mean_terminated_length": 616.73828125,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1882.0,
+      "completions/mean_length": 578.701171875,
+      "completions/mean_terminated_length": 572.9392700195312,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.3581822252070741,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002015596791143499,
-      "kl": 0.00691986083984375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09024918016321634,
+      "kl": 0.04998779296875,
       "learning_rate": 9.645853688680177e-07,
-      "loss": 0.0001,
-      "num_tokens": 238555476.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0057,
+      "num_tokens": 251716346.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 400
     },
     {
@@ -11216,25 +11216,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1433.0,
-      "completions/max_terminated_length": 1433.0,
-      "completions/mean_length": 559.072265625,
-      "completions/mean_terminated_length": 559.072265625,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_length": 1275.0,
+      "completions/max_terminated_length": 1275.0,
+      "completions/mean_length": 533.146484375,
+      "completions/mean_terminated_length": 533.146484375,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.3590776807700918,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0018608514139559509,
-      "kl": 0.0065155029296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07662408160218846,
+      "kl": 0.04638671875,
       "learning_rate": 9.557536762338786e-07,
-      "loss": 0.0001,
-      "num_tokens": 239128617.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0079,
+      "num_tokens": 252276213.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 401
     },
     {
@@ -11243,20 +11243,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1995.0,
-      "completions/mean_length": 571.732421875,
-      "completions/mean_terminated_length": 568.8434448242188,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1603.0,
+      "completions/max_terminated_length": 1603.0,
+      "completions/mean_length": 539.779296875,
+      "completions/mean_terminated_length": 539.779296875,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.35997313633310946,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024477307731748934,
-      "kl": 0.0071258544921875,
+      "grad_norm": 0.0047643547604445105,
+      "kl": 0.04693603515625,
       "learning_rate": 9.46997266581973e-07,
-      "loss": 0.0001,
-      "num_tokens": 239705936.0,
+      "loss": 0.0005,
+      "num_tokens": 252837172.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11271,26 +11271,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1932.0,
-      "completions/mean_length": 626.830078125,
-      "completions/mean_terminated_length": 624.0488891601562,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1944.0,
+      "completions/max_terminated_length": 1944.0,
+      "completions/mean_length": 597.201171875,
+      "completions/mean_terminated_length": 597.201171875,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.36086859189612713,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09673933475786087,
-      "kl": 0.0070343017578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004834734143787087,
+      "kl": 0.0479736328125,
       "learning_rate": 9.383165073137115e-07,
-      "loss": 0.0075,
-      "num_tokens": 240327913.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 253443979.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 403
     },
     {
@@ -11299,26 +11299,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1525.0,
-      "completions/max_terminated_length": 1525.0,
-      "completions/mean_length": 591.482421875,
-      "completions/mean_terminated_length": 590.0509033203125,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1626.0,
+      "completions/max_terminated_length": 1626.0,
+      "completions/mean_length": 566.322265625,
+      "completions/mean_terminated_length": 566.322265625,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.36176404745914487,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05941957788832946,
-      "kl": 0.01541900634765625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00579926871259698,
+      "kl": 0.04705810546875,
       "learning_rate": 9.297117626563687e-07,
-      "loss": -0.0002,
-      "num_tokens": 240926064.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 254029248.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 404
     },
     {
@@ -11328,19 +11328,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1363.0,
-      "completions/max_terminated_length": 1363.0,
-      "completions/mean_length": 601.333984375,
-      "completions/mean_terminated_length": 601.333984375,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/max_length": 1328.0,
+      "completions/max_terminated_length": 1328.0,
+      "completions/mean_length": 586.23046875,
+      "completions/mean_terminated_length": 586.23046875,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.36265950302216254,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006173233714714381,
-      "kl": 0.006805419921875,
+      "grad_norm": 0.005531919514784843,
+      "kl": 0.048583984375,
       "learning_rate": 9.211833936477957e-07,
-      "loss": 0.0001,
-      "num_tokens": 241514283.0,
+      "loss": 0.0005,
+      "num_tokens": 254609734.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11356,25 +11356,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1885.0,
-      "completions/max_terminated_length": 1885.0,
-      "completions/mean_length": 584.126953125,
-      "completions/mean_terminated_length": 584.126953125,
-      "completions/min_length": 122.0,
-      "completions/min_terminated_length": 122.0,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1538.0,
+      "completions/mean_length": 562.662109375,
+      "completions/mean_terminated_length": 562.662109375,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.3635549585851802,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.054228472972914075,
-      "kl": 0.00666046142578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00421872727224393,
+      "kl": 0.04925537109375,
       "learning_rate": 9.127317581212753e-07,
-      "loss": -0.0001,
-      "num_tokens": 242109068.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 255193529.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 406
     },
     {
@@ -11383,26 +11383,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1264.0,
-      "completions/max_terminated_length": 1264.0,
-      "completions/mean_length": 570.783203125,
-      "completions/mean_terminated_length": 570.783203125,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1312.0,
+      "completions/mean_length": 546.47265625,
+      "completions/mean_terminated_length": 542.1961059570312,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.3644504141481979,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.001946101119368566,
-      "kl": 0.00690460205078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.10098327848008554,
+      "kl": 0.05133056640625,
       "learning_rate": 9.043572106905084e-07,
-      "loss": 0.0001,
-      "num_tokens": 242724605.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.013,
+      "num_tokens": 255796619.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 407
     },
     {
@@ -11411,26 +11411,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1845.0,
-      "completions/max_terminated_length": 1845.0,
-      "completions/mean_length": 587.302734375,
-      "completions/mean_terminated_length": 587.302734375,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1818.0,
+      "completions/mean_length": 574.74609375,
+      "completions/mean_terminated_length": 571.863037109375,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.3653458697112156,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019191158558493225,
-      "kl": 0.007293701171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07221448169919696,
+      "kl": 0.04937744140625,
       "learning_rate": 8.960601027347321e-07,
-      "loss": 0.0001,
-      "num_tokens": 243331384.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0011,
+      "num_tokens": 256396969.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 408
     },
     {
@@ -11439,26 +11439,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1259.0,
-      "completions/max_terminated_length": 1259.0,
-      "completions/mean_length": 550.9140625,
-      "completions/mean_terminated_length": 550.9140625,
-      "completions/min_length": 222.0,
-      "completions/min_terminated_length": 222.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1660.0,
+      "completions/max_terminated_length": 1660.0,
+      "completions/mean_length": 531.146484375,
+      "completions/mean_terminated_length": 530.3033447265625,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.36624132527423325,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002127355767792274,
-      "kl": 0.00656890869140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06674911247595358,
+      "kl": 0.04931640625,
       "learning_rate": 8.878407823839788e-07,
-      "loss": 0.0001,
-      "num_tokens": 243899260.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0029,
+      "num_tokens": 256954724.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 409
     },
     {
@@ -11467,26 +11467,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1719.0,
-      "completions/max_terminated_length": 1719.0,
-      "completions/mean_length": 598.248046875,
-      "completions/mean_terminated_length": 598.248046875,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2013.0,
+      "completions/max_terminated_length": 2013.0,
+      "completions/mean_length": 568.6484375,
+      "completions/mean_terminated_length": 567.4246826171875,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.36713678083725093,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052649635615735615,
-      "kl": 0.0068817138671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.010132848557988017,
+      "kl": 0.05450439453125,
       "learning_rate": 8.796995945044689e-07,
-      "loss": 0.0003,
-      "num_tokens": 244511867.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 257552176.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 410
     },
     {
@@ -11496,19 +11496,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1455.0,
-      "completions/max_terminated_length": 1455.0,
-      "completions/mean_length": 605.73828125,
-      "completions/mean_terminated_length": 605.73828125,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/max_length": 1686.0,
+      "completions/max_terminated_length": 1686.0,
+      "completions/mean_length": 581.15234375,
+      "completions/mean_terminated_length": 581.15234375,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.36803223640026866,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0023921493243053795,
-      "kl": 0.00725555419921875,
+      "grad_norm": 0.00432008518918132,
+      "kl": 0.0484619140625,
       "learning_rate": 8.716368806841405e-07,
-      "loss": 0.0001,
-      "num_tokens": 245131221.0,
+      "loss": 0.0005,
+      "num_tokens": 258158942.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11523,20 +11523,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1290.0,
-      "completions/mean_length": 578.29296875,
-      "completions/mean_terminated_length": 575.4168090820312,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1425.0,
+      "completions/max_terminated_length": 1425.0,
+      "completions/mean_length": 561.341796875,
+      "completions/mean_terminated_length": 561.341796875,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.36892769196328634,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07554786393229868,
-      "kl": 0.00659942626953125,
+      "grad_norm": 0.06693413618462407,
+      "kl": 0.0494384765625,
       "learning_rate": 8.636529792183171e-07,
-      "loss": 0.0104,
-      "num_tokens": 245720827.0,
+      "loss": -0.001,
+      "num_tokens": 258739869.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -11551,20 +11551,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1502.0,
-      "completions/mean_length": 595.05859375,
-      "completions/mean_terminated_length": 590.431396484375,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1556.0,
+      "completions/max_terminated_length": 1556.0,
+      "completions/mean_length": 560.208984375,
+      "completions/mean_terminated_length": 560.208984375,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
       "epoch": 0.369823147526304,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.019885190989181234,
-      "kl": 0.01816558837890625,
+      "grad_norm": 0.004847298445441672,
+      "kl": 0.04901123046875,
       "learning_rate": 8.557482250955144e-07,
-      "loss": 0.0002,
-      "num_tokens": 246333385.0,
+      "loss": 0.0005,
+      "num_tokens": 259334584.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11580,19 +11580,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1871.0,
-      "completions/max_terminated_length": 1871.0,
-      "completions/mean_length": 607.44140625,
-      "completions/mean_terminated_length": 607.44140625,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 1558.0,
+      "completions/max_terminated_length": 1558.0,
+      "completions/mean_length": 611.892578125,
+      "completions/mean_terminated_length": 611.892578125,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.3707186030893217,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002526845240160502,
-      "kl": 0.0074310302734375,
+      "grad_norm": 0.005198359727407881,
+      "kl": 0.04888916015625,
       "learning_rate": 8.479229499833844e-07,
-      "loss": 0.0001,
-      "num_tokens": 246968763.0,
+      "loss": 0.0005,
+      "num_tokens": 259972241.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11608,19 +11608,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1883.0,
-      "completions/max_terminated_length": 1883.0,
-      "completions/mean_length": 569.833984375,
-      "completions/mean_terminated_length": 569.833984375,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_length": 1332.0,
+      "completions/max_terminated_length": 1332.0,
+      "completions/mean_length": 529.185546875,
+      "completions/mean_terminated_length": 529.185546875,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.3716140586523394,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00203404503319221,
-      "kl": 0.00699615478515625,
+      "grad_norm": 0.005096616784905531,
+      "kl": 0.04815673828125,
       "learning_rate": 8.401774822147976e-07,
-      "loss": 0.0001,
-      "num_tokens": 247495238.0,
+      "loss": 0.0005,
+      "num_tokens": 260477904.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11636,25 +11636,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1630.0,
-      "completions/max_terminated_length": 1630.0,
-      "completions/mean_length": 587.775390625,
-      "completions/mean_terminated_length": 587.775390625,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/max_length": 1589.0,
+      "completions/max_terminated_length": 1589.0,
+      "completions/mean_length": 574.904296875,
+      "completions/mean_terminated_length": 574.904296875,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
       "epoch": 0.37250951421535705,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002779264425300328,
-      "kl": 0.00716400146484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06747598747854207,
+      "kl": 0.04888916015625,
       "learning_rate": 8.325121467740695e-07,
-      "loss": 0.0001,
-      "num_tokens": 248117171.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.001,
+      "num_tokens": 261093247.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 416
     },
     {
@@ -11664,19 +11664,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1172.0,
-      "completions/max_terminated_length": 1172.0,
-      "completions/mean_length": 575.365234375,
-      "completions/mean_terminated_length": 575.365234375,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/max_length": 1710.0,
+      "completions/max_terminated_length": 1710.0,
+      "completions/mean_length": 573.78515625,
+      "completions/mean_terminated_length": 573.78515625,
+      "completions/min_length": 122.0,
+      "completions/min_terminated_length": 122.0,
       "epoch": 0.37340496977837473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0018850938887697785,
-      "kl": 0.00659942626953125,
+      "grad_norm": 0.004344603358267009,
+      "kl": 0.0465087890625,
       "learning_rate": 8.249272652833226e-07,
-      "loss": 0.0001,
-      "num_tokens": 248737774.0,
+      "loss": 0.0005,
+      "num_tokens": 261713041.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11692,25 +11692,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1624.0,
-      "completions/max_terminated_length": 1624.0,
-      "completions/mean_length": 543.0390625,
-      "completions/mean_terminated_length": 543.0390625,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/max_length": 1463.0,
+      "completions/max_terminated_length": 1463.0,
+      "completions/mean_length": 533.013671875,
+      "completions/mean_terminated_length": 533.013671875,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.3743004253413924,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07549294657539767,
-      "kl": 0.0076904296875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004507059752267276,
+      "kl": 0.0506591796875,
       "learning_rate": 8.174231559889931e-07,
-      "loss": -0.0014,
-      "num_tokens": 249311714.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 262281848.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 418
     },
     {
@@ -11720,19 +11720,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1836.0,
-      "completions/max_terminated_length": 1836.0,
-      "completions/mean_length": 528.9765625,
-      "completions/mean_terminated_length": 528.9765625,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/max_length": 1396.0,
+      "completions/max_terminated_length": 1396.0,
+      "completions/mean_length": 517.865234375,
+      "completions/mean_terminated_length": 517.865234375,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.37519588090441014,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00231041407725594,
-      "kl": 0.00714111328125,
+      "grad_norm": 0.005351908786787514,
+      "kl": 0.04852294921875,
       "learning_rate": 8.100001337484787e-07,
-      "loss": 0.0001,
-      "num_tokens": 249846646.0,
+      "loss": 0.0005,
+      "num_tokens": 262811091.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11748,25 +11748,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1219.0,
-      "completions/max_terminated_length": 1219.0,
-      "completions/mean_length": 559.14453125,
-      "completions/mean_terminated_length": 559.14453125,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/max_length": 1760.0,
+      "completions/max_terminated_length": 1760.0,
+      "completions/mean_length": 561.669921875,
+      "completions/mean_terminated_length": 561.669921875,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.3760913364674278,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024977967915594834,
-      "kl": 0.00714874267578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.058773060593634845,
+      "kl": 0.0477294921875,
       "learning_rate": 8.026585100169251e-07,
-      "loss": 0.0001,
-      "num_tokens": 250454832.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0004,
+      "num_tokens": 263420570.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 420
     },
     {
@@ -11775,26 +11775,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1959.0,
-      "completions/mean_length": 621.669921875,
-      "completions/mean_terminated_length": 618.878662109375,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1659.0,
+      "completions/max_terminated_length": 1659.0,
+      "completions/mean_length": 601.13671875,
+      "completions/mean_terminated_length": 601.13671875,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.3769867920304455,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.06166413332258108,
-      "kl": 0.0085296630859375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04780470323490276,
+      "kl": 0.04840087890625,
       "learning_rate": 7.953985928341601e-07,
-      "loss": 0.012,
-      "num_tokens": 251094071.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": -0.0011,
+      "num_tokens": 264049296.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 421
     },
     {
@@ -11804,19 +11804,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1575.0,
-      "completions/max_terminated_length": 1575.0,
-      "completions/mean_length": 600.337890625,
-      "completions/mean_terminated_length": 600.337890625,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/max_length": 1901.0,
+      "completions/max_terminated_length": 1901.0,
+      "completions/mean_length": 587.703125,
+      "completions/mean_terminated_length": 587.703125,
+      "completions/min_length": 122.0,
+      "completions/min_terminated_length": 122.0,
       "epoch": 0.37788224759346317,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024472020636609228,
-      "kl": 0.00649261474609375,
+      "grad_norm": 0.015584951802210437,
+      "kl": 0.0504150390625,
       "learning_rate": 7.882206868117693e-07,
-      "loss": 0.0001,
-      "num_tokens": 251700772.0,
+      "loss": 0.0005,
+      "num_tokens": 264649528.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11831,20 +11831,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1759.0,
-      "completions/mean_length": 579.787109375,
-      "completions/mean_terminated_length": 576.9138793945312,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1931.0,
+      "completions/max_terminated_length": 1931.0,
+      "completions/mean_length": 575.669921875,
+      "completions/mean_terminated_length": 575.669921875,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.37877770315648085,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0025695653842804063,
-      "kl": 0.0072021484375,
+      "grad_norm": 0.008202181121387101,
+      "kl": 0.04742431640625,
       "learning_rate": 7.81125093120313e-07,
-      "loss": 0.0001,
-      "num_tokens": 252319143.0,
+      "loss": 0.0005,
+      "num_tokens": 265265791.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11860,25 +11860,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1465.0,
-      "completions/max_terminated_length": 1465.0,
-      "completions/mean_length": 563.19140625,
-      "completions/mean_terminated_length": 563.19140625,
-      "completions/min_length": 251.0,
-      "completions/min_terminated_length": 251.0,
+      "completions/max_length": 1773.0,
+      "completions/max_terminated_length": 1773.0,
+      "completions/mean_length": 554.826171875,
+      "completions/mean_terminated_length": 554.826171875,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.3796731587194985,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024216522522944213,
-      "kl": 0.00678253173828125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07411872102018555,
+      "kl": 0.04827880859375,
       "learning_rate": 7.741121094766916e-07,
-      "loss": 0.0001,
-      "num_tokens": 252933833.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0036,
+      "num_tokens": 265876198.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 424
     },
     {
@@ -11888,19 +11888,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1950.0,
-      "completions/max_terminated_length": 1950.0,
-      "completions/mean_length": 542.798828125,
-      "completions/mean_terminated_length": 542.798828125,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/max_length": 1806.0,
+      "completions/max_terminated_length": 1806.0,
+      "completions/mean_length": 531.36328125,
+      "completions/mean_terminated_length": 531.36328125,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
       "epoch": 0.3805686142825162,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003995445221210926,
-      "kl": 0.00765228271484375,
+      "grad_norm": 0.00483447721173187,
+      "kl": 0.049072265625,
       "learning_rate": 7.671820301316532e-07,
-      "loss": 0.0001,
-      "num_tokens": 253488418.0,
+      "loss": 0.0005,
+      "num_tokens": 266424928.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11916,19 +11916,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1771.0,
-      "completions/max_terminated_length": 1771.0,
-      "completions/mean_length": 558.029296875,
-      "completions/mean_terminated_length": 558.029296875,
-      "completions/min_length": 230.0,
-      "completions/min_terminated_length": 230.0,
+      "completions/max_length": 1398.0,
+      "completions/max_terminated_length": 1398.0,
+      "completions/mean_length": 547.166015625,
+      "completions/mean_terminated_length": 547.166015625,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.38146406984553394,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030221782126988606,
-      "kl": 0.00658416748046875,
+      "grad_norm": 0.004354827538356667,
+      "kl": 0.04656982421875,
       "learning_rate": 7.603351458574474e-07,
-      "loss": 0.0001,
-      "num_tokens": 254065425.0,
+      "loss": 0.0005,
+      "num_tokens": 266996373.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11943,26 +11943,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1650.0,
-      "completions/mean_length": 569.091796875,
-      "completions/mean_terminated_length": 561.3104248046875,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/max_terminated_length": 1578.0,
+      "completions/mean_length": 560.0390625,
+      "completions/mean_terminated_length": 557.127197265625,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.3823595254085516,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08739034946246164,
-      "kl": 0.0099334716796875,
+      "grad_norm": 0.052963479074057004,
+      "kl": 0.05413818359375,
       "learning_rate": 7.535717439356255e-07,
-      "loss": 0.012,
-      "num_tokens": 254736080.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.0009,
+      "num_tokens": 267662393.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 427
     },
     {
@@ -11972,19 +11972,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1306.0,
-      "completions/max_terminated_length": 1306.0,
-      "completions/mean_length": 532.998046875,
-      "completions/mean_terminated_length": 532.998046875,
-      "completions/min_length": 118.0,
-      "completions/min_terminated_length": 118.0,
+      "completions/max_length": 1869.0,
+      "completions/max_terminated_length": 1869.0,
+      "completions/mean_length": 532.46484375,
+      "completions/mean_terminated_length": 532.46484375,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.3832549809715693,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032836914118878156,
-      "kl": 0.00711822509765625,
+      "grad_norm": 0.14959490402999703,
+      "kl": 0.0740966796875,
       "learning_rate": 7.46892108144986e-07,
-      "loss": 0.0001,
-      "num_tokens": 255296959.0,
+      "loss": 0.0007,
+      "num_tokens": 268222999.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11999,26 +11999,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1549.0,
-      "completions/max_terminated_length": 1549.0,
-      "completions/mean_length": 565.09765625,
-      "completions/mean_terminated_length": 565.09765625,
-      "completions/min_length": 205.0,
-      "completions/min_terminated_length": 205.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1312.0,
+      "completions/mean_length": 572.41015625,
+      "completions/mean_terminated_length": 569.5225219726562,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.38415043653458697,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.028838282068600722,
-      "kl": 0.0084075927734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005582201559752547,
+      "kl": 0.05035400390625,
       "learning_rate": 7.402965187496697e-07,
-      "loss": -0.0028,
-      "num_tokens": 255905585.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 268835369.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 429
     },
     {
@@ -12027,26 +12027,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1784.0,
-      "completions/mean_length": 580.353515625,
-      "completions/mean_terminated_length": 574.5980834960938,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1578.0,
+      "completions/max_terminated_length": 1578.0,
+      "completions/mean_length": 572.984375,
+      "completions/mean_terminated_length": 572.984375,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.38504589209760465,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027135705194481804,
-      "kl": 0.006805419921875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05576338510401166,
+      "kl": 0.0494384765625,
       "learning_rate": 7.337852524873974e-07,
-      "loss": 0.0001,
-      "num_tokens": 256514022.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0035,
+      "num_tokens": 269440033.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 430
     },
     {
@@ -12055,26 +12055,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1552.0,
-      "completions/mean_length": 555.763671875,
-      "completions/mean_terminated_length": 552.8434448242188,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1423.0,
+      "completions/max_terminated_length": 1423.0,
+      "completions/mean_length": 562.912109375,
+      "completions/mean_terminated_length": 562.912109375,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.3859413476606223,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0035836046428584386,
-      "kl": 0.00743865966796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08650265338397081,
+      "kl": 0.04925537109375,
       "learning_rate": 7.273585825578608e-07,
-      "loss": 0.0001,
-      "num_tokens": 257069757.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0027,
+      "num_tokens": 269999428.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 431
     },
     {
@@ -12084,19 +12084,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1635.0,
-      "completions/max_terminated_length": 1635.0,
-      "completions/mean_length": 585.365234375,
-      "completions/mean_terminated_length": 585.365234375,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/max_length": 1740.0,
+      "completions/max_terminated_length": 1740.0,
+      "completions/mean_length": 583.6328125,
+      "completions/mean_terminated_length": 583.6328125,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.38683680322364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026310922549361436,
-      "kl": 0.00643157958984375,
+      "grad_norm": 0.0041924114517845655,
+      "kl": 0.04949951171875,
       "learning_rate": 7.21016778611259e-07,
-      "loss": 0.0001,
-      "num_tokens": 257680792.0,
+      "loss": 0.0005,
+      "num_tokens": 270609576.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12112,19 +12112,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1261.0,
-      "completions/max_terminated_length": 1261.0,
-      "completions/mean_length": 557.9375,
-      "completions/mean_terminated_length": 557.9375,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/max_length": 1501.0,
+      "completions/max_terminated_length": 1501.0,
+      "completions/mean_length": 555.6171875,
+      "completions/mean_terminated_length": 555.6171875,
+      "completions/min_length": 119.0,
+      "completions/min_terminated_length": 119.0,
       "epoch": 0.38773225878665774,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00411242161557962,
-      "kl": 0.00650787353515625,
+      "grad_norm": 0.004051015296435293,
+      "kl": 0.04632568359375,
       "learning_rate": 7.147601067369835e-07,
-      "loss": 0.0001,
-      "num_tokens": 258265336.0,
+      "loss": 0.0005,
+      "num_tokens": 271192932.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12140,19 +12140,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1468.0,
-      "completions/max_terminated_length": 1468.0,
-      "completions/mean_length": 583.52734375,
-      "completions/mean_terminated_length": 583.52734375,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/max_length": 1703.0,
+      "completions/max_terminated_length": 1703.0,
+      "completions/mean_length": 600.853515625,
+      "completions/mean_terminated_length": 600.853515625,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.3886277143496754,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026550287253777143,
-      "kl": 0.00661468505859375,
+      "grad_norm": 0.003873089336388401,
+      "kl": 0.047607421875,
       "learning_rate": 7.085888294524561e-07,
-      "loss": 0.0001,
-      "num_tokens": 258870726.0,
+      "loss": 0.0005,
+      "num_tokens": 271807193.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12168,25 +12168,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1904.0,
-      "completions/max_terminated_length": 1904.0,
-      "completions/mean_length": 593.146484375,
-      "completions/mean_terminated_length": 593.146484375,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_length": 1880.0,
+      "completions/max_terminated_length": 1880.0,
+      "completions/mean_length": 589.169921875,
+      "completions/mean_terminated_length": 589.169921875,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.3895231699126931,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005815059007833509,
-      "kl": 0.00765228271484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06598911812763071,
+      "kl": 0.04876708984375,
       "learning_rate": 7.025032056921117e-07,
-      "loss": 0.0001,
-      "num_tokens": 259478753.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0017,
+      "num_tokens": 272413184.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 435
     },
     {
@@ -12195,20 +12195,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1576.0,
-      "completions/max_terminated_length": 1576.0,
-      "completions/mean_length": 570.83203125,
-      "completions/mean_terminated_length": 569.1976318359375,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1621.0,
+      "completions/max_terminated_length": 1621.0,
+      "completions/mean_length": 554.048828125,
+      "completions/mean_terminated_length": 554.048828125,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.39041862547571077,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.7255448560508342,
-      "kl": 0.04613494873046875,
+      "grad_norm": 0.005011629822964105,
+      "kl": 0.04913330078125,
       "learning_rate": 6.965034907965349e-07,
       "loss": 0.0005,
-      "num_tokens": 260092971.0,
+      "num_tokens": 273018809.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12223,26 +12223,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1158.0,
-      "completions/max_terminated_length": 1158.0,
-      "completions/mean_length": 552.009765625,
-      "completions/mean_terminated_length": 552.009765625,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1509.0,
+      "completions/mean_length": 546.236328125,
+      "completions/mean_terminated_length": 540.3471069335938,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.39131408103872845,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030963488401591077,
-      "kl": 0.00754547119140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05460977307454807,
+      "kl": 0.0478515625,
       "learning_rate": 6.905899365017462e-07,
-      "loss": 0.0001,
-      "num_tokens": 260662080.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0158,
+      "num_tokens": 273584962.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 437
     },
     {
@@ -12252,19 +12252,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1591.0,
-      "completions/max_terminated_length": 1591.0,
-      "completions/mean_length": 578.076171875,
-      "completions/mean_terminated_length": 576.890380859375,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 561.583984375,
+      "completions/mean_terminated_length": 558.6751708984375,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.3922095366017461,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02801037982963157,
-      "kl": 0.0189666748046875,
+      "grad_norm": 0.0062424353732860646,
+      "kl": 0.05084228515625,
       "learning_rate": 6.847627909286409e-07,
-      "loss": 0.0002,
-      "num_tokens": 261299687.0,
+      "loss": 0.0005,
+      "num_tokens": 274214125.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12280,19 +12280,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1346.0,
-      "completions/max_terminated_length": 1346.0,
-      "completions/mean_length": 577.248046875,
-      "completions/mean_terminated_length": 577.248046875,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/max_length": 1474.0,
+      "completions/max_terminated_length": 1474.0,
+      "completions/mean_length": 580.083984375,
+      "completions/mean_terminated_length": 580.083984375,
+      "completions/min_length": 133.0,
+      "completions/min_terminated_length": 133.0,
       "epoch": 0.3931049921647638,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026557993041991315,
-      "kl": 0.00693511962890625,
+      "grad_norm": 0.007684600815592728,
+      "kl": 0.0478515625,
       "learning_rate": 6.790222985725761e-07,
-      "loss": 0.0001,
-      "num_tokens": 261895174.0,
+      "loss": 0.0005,
+      "num_tokens": 274811064.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12308,19 +12308,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.75,
-      "completions/max_length": 1197.0,
-      "completions/max_terminated_length": 1197.0,
-      "completions/mean_length": 576.236328125,
-      "completions/mean_terminated_length": 556.211669921875,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/max_length": 1687.0,
+      "completions/max_terminated_length": 1687.0,
+      "completions/mean_length": 562.736328125,
+      "completions/mean_terminated_length": 551.4375,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.39400044772778153,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 15.334091278669593,
-      "kl": 0.416748046875,
+      "grad_norm": 105.4794447937914,
+      "kl": 12.5390625,
       "learning_rate": 6.733687002931141e-07,
-      "loss": 0.0042,
-      "num_tokens": 262493631.0,
+      "loss": 0.1253,
+      "num_tokens": 275402609.0,
       "reward": 0.09687499701976776,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12335,26 +12335,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1230.0,
-      "completions/max_terminated_length": 1230.0,
-      "completions/mean_length": 571.33203125,
-      "completions/mean_terminated_length": 571.33203125,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1720.0,
+      "completions/mean_length": 574.7421875,
+      "completions/mean_terminated_length": 568.9647216796875,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.3948959032907992,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002444742961490628,
-      "kl": 0.00711822509765625,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.15188148089603382,
+      "kl": 0.0477294921875,
       "learning_rate": 6.678022333039158e-07,
-      "loss": 0.0001,
-      "num_tokens": 263071657.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.024,
+      "num_tokens": 275982381.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 441
     },
     {
@@ -12363,26 +12363,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1682.0,
-      "completions/mean_length": 620.861328125,
-      "completions/mean_terminated_length": 616.933349609375,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1923.0,
+      "completions/max_terminated_length": 1923.0,
+      "completions/mean_length": 606.69921875,
+      "completions/mean_terminated_length": 606.69921875,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.3957913588538169,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011201392862572901,
-      "kl": 0.014007568359375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06622735966778687,
+      "kl": 0.04669189453125,
       "learning_rate": 6.623231311627876e-07,
-      "loss": 0.0001,
-      "num_tokens": 263709330.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0043,
+      "num_tokens": 276612803.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 442
     },
     {
@@ -12392,19 +12392,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1731.0,
-      "completions/max_terminated_length": 1731.0,
-      "completions/mean_length": 610.037109375,
-      "completions/mean_terminated_length": 610.037109375,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/max_length": 1895.0,
+      "completions/max_terminated_length": 1895.0,
+      "completions/mean_length": 589.787109375,
+      "completions/mean_terminated_length": 589.787109375,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.39668681441683457,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0021737767712625513,
-      "kl": 0.0066070556640625,
+      "grad_norm": 0.004054937917658486,
+      "kl": 0.04730224609375,
       "learning_rate": 6.569316237618811e-07,
-      "loss": 0.0001,
-      "num_tokens": 264310517.0,
+      "loss": 0.0005,
+      "num_tokens": 277203622.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12419,26 +12419,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1484.0,
-      "completions/max_terminated_length": 1484.0,
-      "completions/mean_length": 605.30078125,
-      "completions/mean_terminated_length": 605.30078125,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1854.0,
+      "completions/max_terminated_length": 1660.0,
+      "completions/mean_length": 602.287109375,
+      "completions/mean_terminated_length": 597.3784790039062,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.39758226997985224,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003722959764073968,
-      "kl": 0.00848388671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07219201096985092,
+      "kl": 0.04864501953125,
       "learning_rate": 6.516279373180499e-07,
-      "loss": 0.0001,
-      "num_tokens": 264956351.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0084,
+      "num_tokens": 277847913.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 444
     },
     {
@@ -12447,20 +12447,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1252.0,
-      "completions/max_terminated_length": 1252.0,
-      "completions/mean_length": 549.498046875,
-      "completions/mean_terminated_length": 549.498046875,
-      "completions/min_length": 162.0,
-      "completions/min_terminated_length": 162.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1784.0,
+      "completions/mean_length": 554.794921875,
+      "completions/mean_terminated_length": 551.872802734375,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.3984777255428699,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00215060811327745,
-      "kl": 0.0074615478515625,
+      "grad_norm": 0.004333441625319936,
+      "kl": 0.0489501953125,
       "learning_rate": 6.464122943633543e-07,
-      "loss": 0.0001,
-      "num_tokens": 265566782.0,
+      "loss": 0.0005,
+      "num_tokens": 278461056.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12475,26 +12475,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1516.0,
-      "completions/max_terminated_length": 1516.0,
-      "completions/mean_length": 594.345703125,
-      "completions/mean_terminated_length": 594.345703125,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1661.0,
+      "completions/mean_length": 592.8828125,
+      "completions/mean_terminated_length": 590.0352172851562,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.3993731811058876,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.001984884141250643,
-      "kl": 0.00677490234375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07587832755649311,
+      "kl": 0.0469970703125,
       "learning_rate": 6.412849137357271e-07,
-      "loss": 0.0001,
-      "num_tokens": 266189727.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0016,
+      "num_tokens": 279083252.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 446
     },
     {
@@ -12504,25 +12504,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1500.0,
-      "completions/max_terminated_length": 1500.0,
-      "completions/mean_length": 560.974609375,
-      "completions/mean_terminated_length": 559.136962890625,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_length": 1835.0,
+      "completions/max_terminated_length": 1835.0,
+      "completions/mean_length": 558.845703125,
+      "completions/mean_terminated_length": 558.0822143554688,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.40026863666890533,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012607005481500596,
-      "kl": 0.0207061767578125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 5.034003120090257,
+      "kl": 0.0579833984375,
       "learning_rate": 6.3624601056979e-07,
-      "loss": 0.0002,
-      "num_tokens": 266789154.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0124,
+      "num_tokens": 279681589.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 447
     },
     {
@@ -12532,19 +12532,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1827.0,
-      "completions/max_terminated_length": 1827.0,
-      "completions/mean_length": 601.62890625,
-      "completions/mean_terminated_length": 600.0332641601562,
-      "completions/min_length": 242.0,
-      "completions/min_terminated_length": 242.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1953.0,
+      "completions/mean_length": 614.783203125,
+      "completions/mean_terminated_length": 611.9784545898438,
+      "completions/min_length": 248.0,
+      "completions/min_terminated_length": 248.0,
       "epoch": 0.401164092231923,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 7.872691267196718,
-      "kl": 0.02170562744140625,
+      "grad_norm": 0.04124554822003232,
+      "kl": 0.048583984375,
       "learning_rate": 6.312957962878278e-07,
-      "loss": 0.0116,
-      "num_tokens": 267369940.0,
+      "loss": -0.0048,
+      "num_tokens": 280269110.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -12560,19 +12560,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1537.0,
-      "completions/max_terminated_length": 1537.0,
-      "completions/mean_length": 577.041015625,
-      "completions/mean_terminated_length": 577.041015625,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1602.0,
+      "completions/max_terminated_length": 1602.0,
+      "completions/mean_length": 570.455078125,
+      "completions/mean_terminated_length": 570.455078125,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.4020595477949407,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0026289677378162537,
-      "kl": 0.007049560546875,
+      "grad_norm": 0.004252061047808049,
+      "kl": 0.0467529296875,
       "learning_rate": 6.264344785909181e-07,
-      "loss": 0.0001,
-      "num_tokens": 267959161.0,
+      "loss": 0.0005,
+      "num_tokens": 280854959.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12587,26 +12587,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 2023.0,
-      "completions/max_terminated_length": 2023.0,
-      "completions/mean_length": 628.451171875,
-      "completions/mean_terminated_length": 628.451171875,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1850.0,
+      "completions/mean_length": 615.158203125,
+      "completions/mean_terminated_length": 610.9019775390625,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
       "epoch": 0.40295500335795836,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0020281249287714726,
-      "kl": 0.0069122314453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 4.155482563277789,
+      "kl": 0.0538330078125,
       "learning_rate": 6.216622614502149e-07,
-      "loss": 0.0001,
-      "num_tokens": 268619264.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0073,
+      "num_tokens": 281508256.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 450
     },
     {
@@ -12615,26 +12615,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1761.0,
-      "completions/max_terminated_length": 1761.0,
-      "completions/mean_length": 631.857421875,
-      "completions/mean_terminated_length": 631.857421875,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 618.771484375,
+      "completions/mean_terminated_length": 615.9745483398438,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.40385045892097604,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019557582531050284,
-      "kl": 0.006744384765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.069402280555639,
+      "kl": 0.0484619140625,
       "learning_rate": 6.169793450983916e-07,
-      "loss": 0.0001,
-      "num_tokens": 269257159.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0053,
+      "num_tokens": 282139451.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 451
     },
     {
@@ -12643,20 +12643,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1748.0,
-      "completions/mean_length": 632.13671875,
-      "completions/mean_terminated_length": 629.365966796875,
-      "completions/min_length": 238.0,
-      "completions/min_terminated_length": 238.0,
+      "completions/max_terminated_length": 1601.0,
+      "completions/mean_length": 629.6171875,
+      "completions/mean_terminated_length": 624.054931640625,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.4047459144839937,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07942781207559449,
-      "kl": 0.0071868896484375,
+      "grad_norm": 0.05619296797430902,
+      "kl": 0.04974365234375,
       "learning_rate": 6.123859260212393e-07,
-      "loss": 0.0115,
-      "num_tokens": 269924477.0,
+      "loss": -0.0006,
+      "num_tokens": 282805479.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -12672,25 +12672,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1287.0,
-      "completions/max_terminated_length": 1287.0,
-      "completions/mean_length": 553.845703125,
-      "completions/mean_terminated_length": 553.845703125,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/max_length": 1345.0,
+      "completions/max_terminated_length": 1345.0,
+      "completions/mean_length": 551.099609375,
+      "completions/mean_terminated_length": 551.099609375,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.4056413700470114,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002260666693581993,
-      "kl": 0.00655364990234375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06716769755390013,
+      "kl": 0.047607421875,
       "learning_rate": 6.07882196949423e-07,
-      "loss": 0.0001,
-      "num_tokens": 270508718.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 283388314.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 453
     },
     {
@@ -12699,20 +12699,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1669.0,
-      "completions/max_terminated_length": 1669.0,
-      "completions/mean_length": 605.912109375,
-      "completions/mean_terminated_length": 605.912109375,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1887.0,
+      "completions/mean_length": 585.65625,
+      "completions/mean_terminated_length": 582.7944946289062,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.40653682561002913,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002447637498558581,
-      "kl": 0.00658416748046875,
+      "grad_norm": 0.004697335407404541,
+      "kl": 0.04742431640625,
       "learning_rate": 6.034683468503948e-07,
-      "loss": 0.0001,
-      "num_tokens": 271098977.0,
+      "loss": 0.0005,
+      "num_tokens": 283968202.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12727,22 +12727,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.734375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1368.0,
-      "completions/mean_length": 615.451171875,
-      "completions/mean_terminated_length": 592.5556030273438,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -6.75,
+      "completions/max_length": 1486.0,
+      "completions/max_terminated_length": 1486.0,
+      "completions/mean_length": 585.796875,
+      "completions/mean_terminated_length": 574.6612548828125,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.4074322811730468,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 2.4369472254890088,
-      "kl": 0.1077880859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 1.027380515611281,
+      "kl": 0.306640625,
       "learning_rate": 5.991445609204641e-07,
-      "loss": 0.022,
-      "num_tokens": 271757272.0,
+      "loss": 0.0031,
+      "num_tokens": 284611314.0,
       "reward": 0.09687499701976776,
-      "reward_std": 0.0015625000232830644,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
       "rewards/format_reward/mean": 0.96875,
@@ -12755,26 +12755,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1747.0,
-      "completions/max_terminated_length": 1747.0,
-      "completions/mean_length": 581.82421875,
-      "completions/mean_terminated_length": 579.5440063476562,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1545.0,
+      "completions/max_terminated_length": 1545.0,
+      "completions/mean_length": 546.5234375,
+      "completions/mean_terminated_length": 546.5234375,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.4083277367360645,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.08771054633359765,
-      "kl": 0.0449371337890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0556622348002105,
+      "kl": 0.04693603515625,
       "learning_rate": 5.949110205770292e-07,
-      "loss": 0.0004,
-      "num_tokens": 272388462.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0011,
+      "num_tokens": 285224430.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 456
     },
     {
@@ -12784,25 +12784,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1665.0,
-      "completions/max_terminated_length": 1665.0,
-      "completions/mean_length": 602.40625,
-      "completions/mean_terminated_length": 602.40625,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/max_length": 1539.0,
+      "completions/max_terminated_length": 1539.0,
+      "completions/mean_length": 569.07421875,
+      "completions/mean_terminated_length": 569.07421875,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.40922319229908216,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004411391036507665,
-      "kl": 0.00644683837890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.038169107742384197,
+      "kl": 0.0482177734375,
       "learning_rate": 5.90767903450964e-07,
-      "loss": 0.0001,
-      "num_tokens": 273004302.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0023,
+      "num_tokens": 285823204.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 457
     },
     {
@@ -12813,18 +12813,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1573.0,
-      "completions/mean_length": 608.103515625,
-      "completions/mean_terminated_length": 605.2857055664062,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/max_terminated_length": 1509.0,
+      "completions/mean_length": 579.447265625,
+      "completions/mean_terminated_length": 576.5733642578125,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.41011864786209984,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0017929355017531458,
-      "kl": 0.00606536865234375,
+      "grad_norm": 0.005065842696717293,
+      "kl": 0.047607421875,
       "learning_rate": 5.867153833791652e-07,
-      "loss": 0.0001,
-      "num_tokens": 273598435.0,
+      "loss": 0.0005,
+      "num_tokens": 286402665.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12839,26 +12839,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1965.0,
-      "completions/mean_length": 606.548828125,
-      "completions/mean_terminated_length": 601.058837890625,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/max_terminated_length": 1992.0,
+      "completions/mean_length": 614.25,
+      "completions/mean_terminated_length": 605.7996215820312,
+      "completions/min_length": 112.0,
+      "completions/min_terminated_length": 112.0,
       "epoch": 0.4110141034251175,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 202.70294884152713,
-      "kl": 47.255271911621094,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003912700365631719,
+      "kl": 0.04583740234375,
       "learning_rate": 5.827536303972587e-07,
-      "loss": 0.4874,
-      "num_tokens": 274179068.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 286987241.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 459
     },
     {
@@ -12868,19 +12868,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1687.0,
-      "completions/max_terminated_length": 1687.0,
-      "completions/mean_length": 597.40625,
-      "completions/mean_terminated_length": 597.40625,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/max_length": 1813.0,
+      "completions/max_terminated_length": 1813.0,
+      "completions/mean_length": 560.013671875,
+      "completions/mean_terminated_length": 560.013671875,
+      "completions/min_length": 69.0,
+      "completions/min_terminated_length": 69.0,
       "epoch": 0.4119095589881352,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.001968268256640002,
-      "kl": 0.007049560546875,
+      "grad_norm": 0.004495489669199859,
+      "kl": 0.04718017578125,
       "learning_rate": 5.78882810732465e-07,
-      "loss": 0.0001,
-      "num_tokens": 274766508.0,
+      "loss": 0.0005,
+      "num_tokens": 287555536.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12895,20 +12895,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1438.0,
-      "completions/max_terminated_length": 1438.0,
-      "completions/mean_length": 565.31640625,
-      "completions/mean_terminated_length": 565.31640625,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1580.0,
+      "completions/mean_length": 558.630859375,
+      "completions/mean_terminated_length": 555.7162475585938,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.4128050145511529,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.001874408283947134,
-      "kl": 0.00655364990234375,
+      "grad_norm": 0.0046506916985020715,
+      "kl": 0.0484619140625,
       "learning_rate": 5.75103086796625e-07,
-      "loss": 0.0001,
-      "num_tokens": 275377278.0,
+      "loss": 0.0005,
+      "num_tokens": 288162883.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12924,19 +12924,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1562.0,
-      "completions/max_terminated_length": 1562.0,
-      "completions/mean_length": 656.626953125,
-      "completions/mean_terminated_length": 656.626953125,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
+      "completions/max_length": 1877.0,
+      "completions/max_terminated_length": 1877.0,
+      "completions/mean_length": 624.595703125,
+      "completions/mean_terminated_length": 624.595703125,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
       "epoch": 0.4137004701141706,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0018452101093293508,
-      "kl": 0.00627899169921875,
+      "grad_norm": 0.0049626766542558935,
+      "kl": 0.0458984375,
       "learning_rate": 5.714146171793846e-07,
-      "loss": 0.0001,
-      "num_tokens": 276057343.0,
+      "loss": 0.0005,
+      "num_tokens": 288826548.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12951,26 +12951,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1889.0,
-      "completions/mean_length": 695.85546875,
-      "completions/mean_terminated_length": 693.2094116210938,
-      "completions/min_length": 231.0,
-      "completions/min_terminated_length": 231.0,
+      "completions/max_terminated_length": 1807.0,
+      "completions/mean_length": 675.462890625,
+      "completions/mean_terminated_length": 670.0804443359375,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.4145959256771883,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0656524479101266,
-      "kl": 0.00646209716796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.008119931529559445,
+      "kl": 0.049072265625,
       "learning_rate": 5.678175566415422e-07,
-      "loss": 0.0074,
-      "num_tokens": 276779237.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 289538001.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 463
     },
     {
@@ -12980,19 +12980,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1756.0,
-      "completions/max_terminated_length": 1756.0,
-      "completions/mean_length": 615.26953125,
-      "completions/mean_terminated_length": 615.26953125,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/max_length": 1913.0,
+      "completions/max_terminated_length": 1913.0,
+      "completions/mean_length": 582.626953125,
+      "completions/mean_terminated_length": 582.626953125,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.41549138124020596,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002116149555581525,
-      "kl": 0.00698089599609375,
+      "grad_norm": 0.004928378991447442,
+      "kl": 0.049072265625,
       "learning_rate": 5.643120561085528e-07,
-      "loss": 0.0001,
-      "num_tokens": 277421967.0,
+      "loss": 0.0005,
+      "num_tokens": 290164018.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13007,20 +13007,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1530.0,
-      "completions/max_terminated_length": 1530.0,
-      "completions/mean_length": 616.572265625,
-      "completions/mean_terminated_length": 615.7123413085938,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1680.0,
+      "completions/max_terminated_length": 1680.0,
+      "completions/mean_length": 601.583984375,
+      "completions/mean_terminated_length": 601.583984375,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
       "epoch": 0.41638683680322364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 12.454797402118519,
-      "kl": 0.9109954833984375,
+      "grad_norm": 0.008088857980013029,
+      "kl": 0.04791259765625,
       "learning_rate": 5.608982626641991e-07,
-      "loss": 0.0091,
-      "num_tokens": 278065972.0,
+      "loss": 0.0005,
+      "num_tokens": 290800349.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13036,19 +13036,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1607.0,
-      "completions/max_terminated_length": 1607.0,
-      "completions/mean_length": 584.4609375,
-      "completions/mean_terminated_length": 584.4609375,
-      "completions/min_length": 113.0,
-      "completions/min_terminated_length": 113.0,
+      "completions/max_length": 1930.0,
+      "completions/max_terminated_length": 1930.0,
+      "completions/mean_length": 568.7578125,
+      "completions/mean_terminated_length": 568.7578125,
+      "completions/min_length": 81.0,
+      "completions/min_terminated_length": 81.0,
       "epoch": 0.4172822923662413,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0022458003083646212,
-      "kl": 0.00685882568359375,
+      "grad_norm": 0.024017793326594178,
+      "kl": 0.05133056640625,
       "learning_rate": 5.575763195444166e-07,
-      "loss": 0.0001,
-      "num_tokens": 278612816.0,
+      "loss": 0.0005,
+      "num_tokens": 291339153.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13063,20 +13063,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1844.0,
-      "completions/mean_length": 608.923828125,
-      "completions/mean_terminated_length": 606.1076049804688,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1584.0,
+      "completions/max_terminated_length": 1584.0,
+      "completions/mean_length": 581.87109375,
+      "completions/mean_terminated_length": 581.87109375,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.418177747929259,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006077576169920933,
-      "kl": 0.0076141357421875,
+      "grad_norm": 0.004137094950964047,
+      "kl": 0.04852294921875,
       "learning_rate": 5.543463661312847e-07,
-      "loss": 0.0001,
-      "num_tokens": 279243129.0,
+      "loss": 0.0005,
+      "num_tokens": 291955615.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13091,26 +13091,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1712.0,
-      "completions/max_terminated_length": 1712.0,
-      "completions/mean_length": 655.77734375,
-      "completions/mean_terminated_length": 655.77734375,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1926.0,
+      "completions/mean_length": 632.26171875,
+      "completions/mean_terminated_length": 621.1141967773438,
+      "completions/min_length": 189.0,
+      "completions/min_terminated_length": 189.0,
       "epoch": 0.41907320349227667,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00508351729525104,
-      "kl": 0.00701141357421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06436363037629371,
+      "kl": 0.04949951171875,
       "learning_rate": 5.512085379471808e-07,
-      "loss": 0.0001,
-      "num_tokens": 279923399.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0011,
+      "num_tokens": 292623845.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 468
     },
     {
@@ -13120,25 +13120,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1716.0,
-      "completions/mean_length": 567.76171875,
-      "completions/mean_terminated_length": 564.864990234375,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/max_length": 1338.0,
+      "completions/max_terminated_length": 1338.0,
+      "completions/mean_length": 540.55859375,
+      "completions/mean_terminated_length": 539.2700805664062,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.4199686590552944,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019400963630317191,
-      "kl": 0.00716400146484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.9091175781068817,
+      "kl": 0.0504150390625,
       "learning_rate": 5.481629666490903e-07,
-      "loss": 0.0001,
-      "num_tokens": 280524733.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.006,
+      "num_tokens": 293211251.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 469
     },
     {
@@ -13147,20 +13147,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1541.0,
-      "completions/mean_length": 628.064453125,
-      "completions/mean_terminated_length": 622.49609375,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1309.0,
+      "completions/max_terminated_length": 1309.0,
+      "completions/mean_length": 571.876953125,
+      "completions/mean_terminated_length": 571.876953125,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.4208641146183121,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07028124365023734,
-      "kl": 0.00714874267578125,
+      "grad_norm": 0.08882810422258497,
+      "kl": 0.04718017578125,
       "learning_rate": 5.452097800230853e-07,
-      "loss": 0.0077,
-      "num_tokens": 281160590.0,
+      "loss": 0.0039,
+      "num_tokens": 293818340.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -13175,26 +13175,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1393.0,
-      "completions/max_terminated_length": 1393.0,
-      "completions/mean_length": 614.638671875,
-      "completions/mean_terminated_length": 614.638671875,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1920.0,
+      "completions/mean_length": 618.740234375,
+      "completions/mean_terminated_length": 615.9432373046875,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.42175957018132976,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002522479085201929,
-      "kl": 0.00661468505859375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.08441184034971604,
+      "kl": 0.0458984375,
       "learning_rate": 5.423491019789623e-07,
-      "loss": 0.0001,
-      "num_tokens": 281791189.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0047,
+      "num_tokens": 294451039.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 471
     },
     {
@@ -13203,20 +13203,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1525.0,
-      "completions/mean_length": 595.560546875,
-      "completions/mean_terminated_length": 592.7182006835938,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1859.0,
+      "completions/max_terminated_length": 1859.0,
+      "completions/mean_length": 571.236328125,
+      "completions/mean_terminated_length": 571.236328125,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.42265502574434743,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002388905997514264,
-      "kl": 0.00653076171875,
+      "grad_norm": 0.004151029586595816,
+      "kl": 0.047607421875,
       "learning_rate": 5.395810525450425e-07,
-      "loss": 0.0001,
-      "num_tokens": 282402516.0,
+      "loss": 0.0005,
+      "num_tokens": 295049912.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13232,19 +13232,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1697.0,
-      "completions/max_terminated_length": 1697.0,
-      "completions/mean_length": 606.19921875,
-      "completions/mean_terminated_length": 606.19921875,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/max_length": 1622.0,
+      "completions/max_terminated_length": 1622.0,
+      "completions/mean_length": 585.880859375,
+      "completions/mean_terminated_length": 585.880859375,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.4235504813073651,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002615205230137125,
-      "kl": 0.00702667236328125,
+      "grad_norm": 0.006773189374311078,
+      "kl": 0.0477294921875,
       "learning_rate": 5.369057478631359e-07,
-      "loss": 0.0001,
-      "num_tokens": 283050778.0,
+      "loss": 0.0005,
+      "num_tokens": 295687771.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13259,26 +13259,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1604.0,
-      "completions/max_terminated_length": 1604.0,
-      "completions/mean_length": 594.029296875,
-      "completions/mean_terminated_length": 594.029296875,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1519.0,
+      "completions/max_terminated_length": 1519.0,
+      "completions/mean_length": 559.310546875,
+      "completions/mean_terminated_length": 555.547119140625,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.4244459368703828,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002635326102091178,
-      "kl": 0.00766754150390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05238104977419995,
+      "kl": 0.0484619140625,
       "learning_rate": 5.343233001836694e-07,
-      "loss": 0.0001,
-      "num_tokens": 283675753.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0077,
+      "num_tokens": 296294970.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 474
     },
     {
@@ -13287,26 +13287,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1640.0,
-      "completions/mean_length": 635.048828125,
-      "completions/mean_terminated_length": 625.1538696289062,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_terminated_length": 1596.0,
+      "completions/mean_length": 597.630859375,
+      "completions/mean_terminated_length": 591.0942993164062,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.42534139243340047,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07288263680226181,
-      "kl": 0.00714874267578125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04626310549442998,
+      "kl": 0.0478515625,
       "learning_rate": 5.318338178609754e-07,
-      "loss": 0.0314,
-      "num_tokens": 284316898.0,
-      "reward": 0.09902343899011612,
-      "reward_std": 0.0021787926089018583,
+      "loss": 0.0124,
+      "num_tokens": 296916957.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 475
     },
     {
@@ -13315,20 +13315,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1498.0,
-      "completions/max_terminated_length": 1498.0,
-      "completions/mean_length": 568.4375,
-      "completions/mean_terminated_length": 568.4375,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1873.0,
+      "completions/mean_length": 556.599609375,
+      "completions/mean_terminated_length": 550.7510375976562,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.4262368479964182,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024556427027511017,
-      "kl": 0.00702667236328125,
+      "grad_norm": 0.0053069795960833605,
+      "kl": 0.04669189453125,
       "learning_rate": 5.294374053487459e-07,
-      "loss": 0.0001,
-      "num_tokens": 284891058.0,
+      "loss": 0.0005,
+      "num_tokens": 297485056.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13344,19 +13344,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1848.0,
-      "completions/max_terminated_length": 1848.0,
-      "completions/mean_length": 553.7578125,
-      "completions/mean_terminated_length": 553.7578125,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_length": 1817.0,
+      "completions/max_terminated_length": 1817.0,
+      "completions/mean_length": 534.4140625,
+      "completions/mean_terminated_length": 534.4140625,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.4271323035594359,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002743355850282284,
-      "kl": 0.00701141357421875,
+      "grad_norm": 0.004148165474532734,
+      "kl": 0.04693603515625,
       "learning_rate": 5.271341631956511e-07,
-      "loss": 0.0001,
-      "num_tokens": 285455366.0,
+      "loss": 0.0005,
+      "num_tokens": 298039460.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13371,26 +13371,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1466.0,
-      "completions/max_terminated_length": 1466.0,
-      "completions/mean_length": 570.58984375,
-      "completions/mean_terminated_length": 570.58984375,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1695.0,
+      "completions/mean_length": 564.353515625,
+      "completions/mean_terminated_length": 561.4500732421875,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.42802775912245355,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006622609032114254,
-      "kl": 0.00746917724609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0361604804039374,
+      "kl": 0.04638671875,
       "learning_rate": 5.249241880411181e-07,
-      "loss": 0.0001,
-      "num_tokens": 286065652.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0153,
+      "num_tokens": 298646553.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 478
     },
     {
@@ -13401,24 +13401,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 627.298828125,
-      "completions/mean_terminated_length": 624.5186157226562,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/max_terminated_length": 1789.0,
+      "completions/mean_length": 595.029296875,
+      "completions/mean_terminated_length": 592.1859130859375,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.42892321468547123,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07069570079703043,
-      "kl": 0.006683349609375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07503338366587108,
+      "kl": 0.048583984375,
       "learning_rate": 5.228075726112785e-07,
-      "loss": 0.0087,
-      "num_tokens": 286721645.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0007,
+      "num_tokens": 299286024.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 479
     },
     {
@@ -13427,26 +13427,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1667.0,
-      "completions/max_terminated_length": 1667.0,
-      "completions/mean_length": 615.96484375,
-      "completions/mean_terminated_length": 614.74951171875,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1694.0,
+      "completions/max_terminated_length": 1694.0,
+      "completions/mean_length": 589.59375,
+      "completions/mean_terminated_length": 589.59375,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.4298186702484889,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.5664508010867297,
-      "kl": 0.00927734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003932622151853824,
+      "kl": 0.048095703125,
       "learning_rate": 5.207844057150768e-07,
-      "loss": 0.0052,
-      "num_tokens": 287365339.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 299916216.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 480
     },
     {
@@ -13455,20 +13455,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1478.0,
-      "completions/max_terminated_length": 1478.0,
-      "completions/mean_length": 594.375,
-      "completions/mean_terminated_length": 594.375,
-      "completions/min_length": 121.0,
-      "completions/min_terminated_length": 121.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1600.0,
+      "completions/mean_length": 556.908203125,
+      "completions/mean_terminated_length": 553.990234375,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
       "epoch": 0.4307141258115066,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0022399177386503632,
-      "kl": 0.00719451904296875,
+      "grad_norm": 0.0046160433450423335,
+      "kl": 0.05029296875,
       "learning_rate": 5.188547722405437e-07,
-      "loss": 0.0001,
-      "num_tokens": 287997531.0,
+      "loss": 0.0005,
+      "num_tokens": 300529225.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13484,19 +13484,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1655.0,
-      "completions/max_terminated_length": 1655.0,
-      "completions/mean_length": 620.845703125,
-      "completions/mean_terminated_length": 620.845703125,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1835.0,
+      "completions/max_terminated_length": 1835.0,
+      "completions/mean_length": 611.91796875,
+      "completions/mean_terminated_length": 611.91796875,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.43160958137452426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0017809752727835867,
-      "kl": 0.00690460205078125,
+      "grad_norm": 0.004081618380481847,
+      "kl": 0.04681396484375,
       "learning_rate": 5.170187531512351e-07,
-      "loss": 0.0001,
-      "num_tokens": 288647484.0,
+      "loss": 0.0005,
+      "num_tokens": 301174607.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13512,25 +13512,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1235.0,
-      "completions/max_terminated_length": 1235.0,
-      "completions/mean_length": 587.107421875,
-      "completions/mean_terminated_length": 587.107421875,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1538.0,
+      "completions/mean_length": 561.107421875,
+      "completions/mean_terminated_length": 561.107421875,
+      "completions/min_length": 127.0,
+      "completions/min_terminated_length": 127.0,
       "epoch": 0.432505036937542,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07380820125009971,
-      "kl": 0.006591796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005272474133392292,
+      "kl": 0.05059814453125,
       "learning_rate": 5.152764254828348e-07,
-      "loss": 0.0026,
-      "num_tokens": 289269523.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 301783334.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 483
     },
     {
@@ -13539,26 +13539,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1385.0,
-      "completions/max_terminated_length": 1385.0,
-      "completions/mean_length": 592.1484375,
-      "completions/mean_terminated_length": 591.4285888671875,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1796.0,
+      "completions/max_terminated_length": 1796.0,
+      "completions/mean_length": 575.103515625,
+      "completions/mean_terminated_length": 575.103515625,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.4334004925005597,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.367785098713213,
-      "kl": 0.0121002197265625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004371253746007476,
+      "kl": 0.04931640625,
       "learning_rate": 5.136278623399225e-07,
-      "loss": 0.0037,
-      "num_tokens": 289900063.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 302405147.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 484
     },
     {
@@ -13567,20 +13567,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1317.0,
-      "completions/max_terminated_length": 1317.0,
-      "completions/mean_length": 572.5546875,
-      "completions/mean_terminated_length": 572.5546875,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1587.0,
+      "completions/max_terminated_length": 1587.0,
+      "completions/mean_length": 557.376953125,
+      "completions/mean_terminated_length": 555.73779296875,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.43429594806357735,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0019026093751945668,
-      "kl": 0.00656890869140625,
+      "grad_norm": 0.00631760392876256,
+      "kl": 0.04669189453125,
       "learning_rate": 5.120731328929058e-07,
-      "loss": 0.0001,
-      "num_tokens": 290502075.0,
+      "loss": 0.0005,
+      "num_tokens": 302999388.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13596,19 +13596,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1751.0,
-      "completions/max_terminated_length": 1751.0,
-      "completions/mean_length": 610.455078125,
-      "completions/mean_terminated_length": 609.8023681640625,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/max_length": 1474.0,
+      "completions/max_terminated_length": 1474.0,
+      "completions/mean_length": 579.818359375,
+      "completions/mean_terminated_length": 578.4951171875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.43519140362659503,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.277845158123168,
-      "kl": 0.01782989501953125,
+      "grad_norm": 0.7675240642119238,
+      "kl": 0.05047607421875,
       "learning_rate": 5.106123023751187e-07,
-      "loss": 0.0043,
-      "num_tokens": 291112692.0,
+      "loss": 0.0069,
+      "num_tokens": 303594319.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -13623,20 +13623,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1923.0,
-      "completions/max_terminated_length": 1923.0,
-      "completions/mean_length": 634.310546875,
-      "completions/mean_terminated_length": 634.310546875,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1803.0,
+      "completions/mean_length": 610.142578125,
+      "completions/mean_terminated_length": 604.5039672851562,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.4360868591896127,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029675438707230098,
-      "kl": 0.00637054443359375,
+      "grad_norm": 0.003966772271342612,
+      "kl": 0.04791259765625,
       "learning_rate": 5.092454320800833e-07,
-      "loss": 0.0001,
-      "num_tokens": 291779299.0,
+      "loss": 0.0005,
+      "num_tokens": 304248552.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13651,26 +13651,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1793.0,
-      "completions/mean_length": 582.9453125,
-      "completions/mean_terminated_length": 580.0782470703125,
-      "completions/min_length": 133.0,
-      "completions/min_terminated_length": 133.0,
+      "completions/max_terminated_length": 1912.0,
+      "completions/mean_length": 583.4921875,
+      "completions/mean_terminated_length": 574.8605346679688,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.4369823147526304,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002218209993171591,
-      "kl": 0.0069427490234375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06718584348509721,
+      "kl": 0.04815673828125,
       "learning_rate": 5.079725793589405e-07,
-      "loss": 0.0001,
-      "num_tokens": 292388183.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0119,
+      "num_tokens": 304857716.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 488
     },
     {
@@ -13679,26 +13679,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1947.0,
-      "completions/max_terminated_length": 1947.0,
-      "completions/mean_length": 619.53125,
-      "completions/mean_terminated_length": 619.53125,
-      "completions/min_length": 247.0,
-      "completions/min_terminated_length": 247.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1631.0,
+      "completions/mean_length": 587.619140625,
+      "completions/mean_terminated_length": 584.76123046875,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.43787777031564806,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002973729317383643,
-      "kl": 0.00652313232421875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06044390352276465,
+      "kl": 0.04400634765625,
       "learning_rate": 5.067937976180407e-07,
-      "loss": 0.0001,
-      "num_tokens": 293009943.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0005,
+      "num_tokens": 305463137.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 489
     },
     {
@@ -13708,19 +13708,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1907.0,
-      "completions/max_terminated_length": 1907.0,
-      "completions/mean_length": 606.29296875,
-      "completions/mean_terminated_length": 606.29296875,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/max_length": 1466.0,
+      "completions/max_terminated_length": 1466.0,
+      "completions/mean_length": 587.826171875,
+      "completions/mean_terminated_length": 587.826171875,
+      "completions/min_length": 148.0,
+      "completions/min_terminated_length": 148.0,
       "epoch": 0.4387732258786658,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002162476695238198,
-      "kl": 0.0070037841796875,
+      "grad_norm": 0.004340416142239811,
+      "kl": 0.04534912109375,
       "learning_rate": 5.057091363167046e-07,
-      "loss": 0.0001,
-      "num_tokens": 293637293.0,
+      "loss": 0.0005,
+      "num_tokens": 306081032.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13735,20 +13735,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1456.0,
-      "completions/mean_length": 564.47265625,
-      "completions/mean_terminated_length": 561.5694580078125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1335.0,
+      "completions/max_terminated_length": 1335.0,
+      "completions/mean_length": 544.626953125,
+      "completions/mean_terminated_length": 544.626953125,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.4396686814416835,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0024995506608288215,
-      "kl": 0.00746917724609375,
+      "grad_norm": 0.005717450638553965,
+      "kl": 0.0474853515625,
       "learning_rate": 5.047186409651489e-07,
-      "loss": 0.0001,
-      "num_tokens": 294222911.0,
+      "loss": 0.0005,
+      "num_tokens": 306656489.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13763,26 +13763,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1721.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
       "completions/max_terminated_length": 1721.0,
-      "completions/mean_length": 635.138671875,
-      "completions/mean_terminated_length": 635.138671875,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/mean_length": 613.29296875,
+      "completions/mean_terminated_length": 607.6666870117188,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.44056413700470115,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003279801989016285,
-      "kl": 0.0062103271484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05273242178002334,
+      "kl": 0.0477294921875,
       "learning_rate": 5.038223531225742e-07,
-      "loss": 0.0001,
-      "num_tokens": 294842518.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0115,
+      "num_tokens": 307264911.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 492
     },
     {
@@ -13792,25 +13792,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1358.0,
-      "completions/max_terminated_length": 1358.0,
-      "completions/mean_length": 588.244140625,
-      "completions/mean_terminated_length": 588.244140625,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/max_length": 1694.0,
+      "completions/max_terminated_length": 1694.0,
+      "completions/mean_length": 554.16796875,
+      "completions/mean_terminated_length": 554.16796875,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
       "epoch": 0.44145959256771883,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07219605536662821,
-      "kl": 0.00728607177734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004061493509151464,
+      "kl": 0.04656982421875,
       "learning_rate": 5.030203103954232e-07,
-      "loss": 0.0006,
-      "num_tokens": 295443235.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0005,
+      "num_tokens": 307848181.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 493
     },
     {
@@ -13820,19 +13820,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1510.0,
-      "completions/max_terminated_length": 1510.0,
-      "completions/mean_length": 594.796875,
-      "completions/mean_terminated_length": 594.796875,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/max_length": 1524.0,
+      "completions/max_terminated_length": 1524.0,
+      "completions/mean_length": 558.9453125,
+      "completions/mean_terminated_length": 558.9453125,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.4423550481307365,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003095639432087293,
-      "kl": 0.0069122314453125,
+      "grad_norm": 0.004367225471207651,
+      "kl": 0.04486083984375,
       "learning_rate": 5.023125464358026e-07,
-      "loss": 0.0001,
-      "num_tokens": 296030939.0,
+      "loss": 0.0004,
+      "num_tokens": 308417529.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13848,19 +13848,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1350.0,
-      "completions/max_terminated_length": 1350.0,
-      "completions/mean_length": 586.44921875,
-      "completions/mean_terminated_length": 586.44921875,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/max_length": 1677.0,
+      "completions/max_terminated_length": 1677.0,
+      "completions/mean_length": 551.27734375,
+      "completions/mean_terminated_length": 551.27734375,
+      "completions/min_length": 133.0,
+      "completions/min_terminated_length": 133.0,
       "epoch": 0.4432505036937542,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0032109704817087368,
-      "kl": 0.006134033203125,
+      "grad_norm": 0.00418532184825478,
+      "kl": 0.047119140625,
       "learning_rate": 5.016990909400709e-07,
-      "loss": 0.0001,
-      "num_tokens": 296606673.0,
+      "loss": 0.0005,
+      "num_tokens": 308975255.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13876,19 +13876,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1649.0,
-      "completions/max_terminated_length": 1649.0,
-      "completions/mean_length": 647.8203125,
-      "completions/mean_terminated_length": 647.8203125,
-      "completions/min_length": 271.0,
-      "completions/min_terminated_length": 271.0,
+      "completions/max_length": 1497.0,
+      "completions/max_terminated_length": 1497.0,
+      "completions/mean_length": 601.00390625,
+      "completions/mean_terminated_length": 601.00390625,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.44414595925677186,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0030476575163748028,
-      "kl": 0.0064697265625,
+      "grad_norm": 0.004602536366235129,
+      "kl": 0.04852294921875,
       "learning_rate": 5.011799696475915e-07,
-      "loss": 0.0001,
-      "num_tokens": 297232949.0,
+      "loss": 0.0005,
+      "num_tokens": 309577561.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13905,18 +13905,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1572.0,
-      "completions/mean_length": 567.07421875,
-      "completions/mean_terminated_length": 564.1761474609375,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 531.177734375,
+      "completions/mean_terminated_length": 528.2094116210938,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.4450414148197896,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00517923432566091,
-      "kl": 0.00667572021484375,
+      "grad_norm": 0.004030212878424051,
+      "kl": 0.04644775390625,
       "learning_rate": 5.007552043396547e-07,
-      "loss": 0.0001,
-      "num_tokens": 297824987.0,
+      "loss": 0.0005,
+      "num_tokens": 310151220.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13931,26 +13931,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1440.0,
-      "completions/mean_length": 593.87109375,
-      "completions/mean_terminated_length": 588.1686401367188,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_terminated_length": 1416.0,
+      "completions/mean_length": 545.96875,
+      "completions/mean_terminated_length": 543.0293579101562,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
       "epoch": 0.44593687038280727,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002854169465541202,
-      "kl": 0.00717926025390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04970801239600626,
+      "kl": 0.047119140625,
       "learning_rate": 5.004248128385618e-07,
-      "loss": 0.0001,
-      "num_tokens": 298453993.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0006,
+      "num_tokens": 310755700.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 498
     },
     {
@@ -13960,19 +13960,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1089.0,
-      "completions/max_terminated_length": 1089.0,
-      "completions/mean_length": 588.5625,
-      "completions/mean_terminated_length": 588.5625,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_length": 1445.0,
+      "completions/max_terminated_length": 1445.0,
+      "completions/mean_length": 556.638671875,
+      "completions/mean_terminated_length": 556.638671875,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.44683232594582495,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002690500225541875,
-      "kl": 0.006744384765625,
+      "grad_norm": 0.004571033839926497,
+      "kl": 0.0477294921875,
       "learning_rate": 5.001888090068784e-07,
-      "loss": 0.0001,
-      "num_tokens": 299039593.0,
+      "loss": 0.0005,
+      "num_tokens": 311324955.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13988,19 +13988,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1525.0,
-      "completions/max_terminated_length": 1525.0,
-      "completions/mean_length": 610.150390625,
-      "completions/mean_terminated_length": 610.150390625,
-      "completions/min_length": 238.0,
-      "completions/min_terminated_length": 238.0,
+      "completions/max_length": 1470.0,
+      "completions/max_terminated_length": 1470.0,
+      "completions/mean_length": 580.17578125,
+      "completions/mean_terminated_length": 580.17578125,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.4477277815088426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0029109843458220755,
-      "kl": 0.006561279296875,
+      "grad_norm": 0.004267414031586906,
+      "kl": 0.04730224609375,
       "learning_rate": 5.000472027468528e-07,
-      "loss": 0.0001,
-      "num_tokens": 299697222.0,
+      "loss": 0.0005,
+      "num_tokens": 311967237.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14016,25 +14016,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1554.0,
-      "completions/max_terminated_length": 1554.0,
-      "completions/mean_length": 629.177734375,
-      "completions/mean_terminated_length": 629.177734375,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/max_length": 1609.0,
+      "completions/max_terminated_length": 1609.0,
+      "completions/mean_length": 606.591796875,
+      "completions/mean_terminated_length": 606.591796875,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.4486232370718603,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003181584471698959,
-      "kl": 0.006500244140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05086077162905058,
+      "kl": 0.0474853515625,
       "learning_rate": 5.000000000000001e-07,
-      "loss": 0.0001,
-      "num_tokens": 300322849.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0011,
+      "num_tokens": 312581300.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 501
     },
     {
@@ -14045,18 +14045,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1848.0,
-      "completions/mean_length": 628.576171875,
-      "completions/mean_terminated_length": 625.7984008789062,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_terminated_length": 1779.0,
+      "completions/mean_length": 599.373046875,
+      "completions/mean_terminated_length": 596.5381469726562,
+      "completions/min_length": 108.0,
+      "completions/min_terminated_length": 108.0,
       "epoch": 0.449518692634878,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.03514131702249418,
-      "kl": 0.00738525390625,
+      "grad_norm": 0.056542864188258934,
+      "kl": 0.0482177734375,
       "learning_rate": 5.000472027468528e-07,
-      "loss": 0.0154,
-      "num_tokens": 300992888.0,
+      "loss": 0.0146,
+      "num_tokens": 313236387.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -14072,19 +14072,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1356.0,
-      "completions/max_terminated_length": 1356.0,
-      "completions/mean_length": 598.283203125,
-      "completions/mean_terminated_length": 598.283203125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_length": 1390.0,
+      "completions/max_terminated_length": 1390.0,
+      "completions/mean_length": 565.26171875,
+      "completions/mean_terminated_length": 565.26171875,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.45041414819789566,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0028078886176980417,
-      "kl": 0.00676727294921875,
+      "grad_norm": 0.003944884798198514,
+      "kl": 0.04736328125,
       "learning_rate": 5.001888090068784e-07,
-      "loss": 0.0001,
-      "num_tokens": 301585241.0,
+      "loss": 0.0005,
+      "num_tokens": 313811833.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14100,19 +14100,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1856.0,
-      "completions/max_terminated_length": 1856.0,
-      "completions/mean_length": 600.822265625,
-      "completions/mean_terminated_length": 600.822265625,
-      "completions/min_length": 253.0,
-      "completions/min_terminated_length": 253.0,
+      "completions/max_length": 1529.0,
+      "completions/max_terminated_length": 1529.0,
+      "completions/mean_length": 568.341796875,
+      "completions/mean_terminated_length": 568.341796875,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.4513096037609134,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002869992594641441,
-      "kl": 0.0061187744140625,
+      "grad_norm": 0.005105667807983009,
+      "kl": 0.0458984375,
       "learning_rate": 5.004248128385618e-07,
-      "loss": 0.0001,
-      "num_tokens": 302189934.0,
+      "loss": 0.0005,
+      "num_tokens": 314399896.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14127,20 +14127,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1485.0,
-      "completions/mean_length": 642.93359375,
-      "completions/mean_terminated_length": 640.1839599609375,
-      "completions/min_length": 253.0,
-      "completions/min_terminated_length": 253.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1788.0,
+      "completions/max_terminated_length": 1788.0,
+      "completions/mean_length": 609.244140625,
+      "completions/mean_terminated_length": 609.244140625,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.45220505932393107,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0033069623666026116,
-      "kl": 0.00682830810546875,
+      "grad_norm": 0.0043507543970794085,
+      "kl": 0.0474853515625,
       "learning_rate": 5.007552043396547e-07,
-      "loss": 0.0001,
-      "num_tokens": 302869084.0,
+      "loss": 0.0005,
+      "num_tokens": 315061797.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14155,26 +14155,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1852.0,
-      "completions/max_terminated_length": 1852.0,
-      "completions/mean_length": 608.78515625,
-      "completions/mean_terminated_length": 608.78515625,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1824.0,
+      "completions/mean_length": 574.73828125,
+      "completions/mean_terminated_length": 571.8551635742188,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.45310051488694875,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.019256567921365382,
-      "kl": 0.0113372802734375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08176921037298442,
+      "kl": 0.0460205078125,
       "learning_rate": 5.011799696475915e-07,
-      "loss": 0.0001,
-      "num_tokens": 303501582.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0138,
+      "num_tokens": 315676863.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 506
     },
     {
@@ -14183,20 +14183,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1764.0,
-      "completions/max_terminated_length": 1764.0,
-      "completions/mean_length": 621.85546875,
-      "completions/mean_terminated_length": 621.85546875,
-      "completions/min_length": 257.0,
-      "completions/min_terminated_length": 257.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1696.0,
+      "completions/mean_length": 594.41796875,
+      "completions/mean_terminated_length": 588.7176513671875,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.4539959704499664,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004709655692336034,
-      "kl": 0.006866455078125,
+      "grad_norm": 0.004202191868376494,
+      "kl": 0.04669189453125,
       "learning_rate": 5.016990909400706e-07,
-      "loss": 0.0001,
-      "num_tokens": 304138148.0,
+      "loss": 0.0005,
+      "num_tokens": 316299381.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14211,26 +14211,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1990.0,
-      "completions/mean_length": 596.705078125,
-      "completions/mean_terminated_length": 593.864990234375,
-      "completions/min_length": 237.0,
-      "completions/min_terminated_length": 237.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1558.0,
+      "completions/max_terminated_length": 1558.0,
+      "completions/mean_length": 567.14453125,
+      "completions/mean_terminated_length": 567.14453125,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.4548914260129841,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0027508053263992434,
-      "kl": 0.00679779052734375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.06400276605298859,
+      "kl": 0.0452880859375,
       "learning_rate": 5.023125464358026e-07,
-      "loss": 0.0001,
-      "num_tokens": 304715677.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0009,
+      "num_tokens": 316861775.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 508
     },
     {
@@ -14240,19 +14240,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1464.0,
-      "completions/max_terminated_length": 1464.0,
-      "completions/mean_length": 579.806640625,
-      "completions/mean_terminated_length": 579.806640625,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
+      "completions/max_length": 1631.0,
+      "completions/max_terminated_length": 1631.0,
+      "completions/mean_length": 544.9765625,
+      "completions/mean_terminated_length": 544.9765625,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.4557868815760018,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.002570745420249915,
-      "kl": 0.00681304931640625,
+      "grad_norm": 0.004374618149752765,
+      "kl": 0.04608154296875,
       "learning_rate": 5.03020310395423e-07,
-      "loss": 0.0001,
-      "num_tokens": 305322634.0,
+      "loss": 0.0005,
+      "num_tokens": 317450899.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14262,46 +14262,18 @@
       "step": 509
     },
     {
-      "clip_ratio/high_max": 0.0,
-      "clip_ratio/high_mean": 0.0,
-      "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1611.0,
-      "completions/max_terminated_length": 1611.0,
-      "completions/mean_length": 572.6640625,
-      "completions/mean_terminated_length": 572.6640625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
-      "epoch": 0.45668233713901946,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05233213840958361,
-      "kl": 0.00714874267578125,
-      "learning_rate": 5.038223531225739e-07,
-      "loss": 0.0036,
-      "num_tokens": 305889150.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
-      "rewards/code_reward/mean": 0.0,
-      "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
-      "step": 510
-    },
-    {
-      "epoch": 0.45668233713901946,
-      "step": 510,
+      "epoch": 0.4557868815760018,
+      "step": 509,
       "total_flos": 0.0,
-      "train_loss": 7.054507823697492e-06,
-      "train_runtime": 74.8551,
-      "train_samples_per_second": 3419.94,
-      "train_steps_per_second": 6.68
+      "train_loss": 9.052564064315237e-07,
+      "train_runtime": 75.296,
+      "train_samples_per_second": 3399.916,
+      "train_steps_per_second": 6.64
     }
   ],
   "logging_steps": 1,
   "max_steps": 500,
-  "num_input_tokens_seen": 305889150,
+  "num_input_tokens_seen": 317450899,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {