diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -24,10 +24,10 @@
       "completions/min_terminated_length": 212.0,
       "epoch": 0.0008954555630176853,
       "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.3225010711337324,
-      "kl": 0.05487060546875,
+      "grad_norm": 1.2952850427924405,
+      "kl": 0.0005941390991210938,
       "learning_rate": 0.0,
-      "loss": 0.0165,
+      "loss": 0.016,
       "num_tokens": 614764.0,
       "reward": 0.05976562947034836,
       "reward_std": 0.0456097275018692,
@@ -52,10 +52,10 @@
       "completions/min_terminated_length": 144.0,
       "epoch": 0.0017909111260353706,
       "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.1878297521431689,
-      "kl": 0.05389404296875,
+      "grad_norm": 1.1654835256642486,
+      "kl": 0.0006380081176757812,
       "learning_rate": 3.3333333333333335e-07,
-      "loss": 0.0218,
+      "loss": 0.0213,
       "num_tokens": 1251315.0,
       "reward": 0.06015624850988388,
       "reward_std": 0.04386558383703232,
@@ -72,25 +72,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1771.0,
-      "completions/max_terminated_length": 1771.0,
-      "completions/mean_length": 641.310546875,
-      "completions/mean_terminated_length": 640.5048828125,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1898.0,
+      "completions/mean_length": 633.169921875,
+      "completions/mean_terminated_length": 630.4011840820312,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.002686366689053056,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 6.374360193421834,
-      "kl": 0.060302734375,
+      "grad_norm": 1.1432983033631348,
+      "kl": 0.0006580352783203125,
       "learning_rate": 6.666666666666667e-07,
-      "loss": 0.0187,
-      "num_tokens": 1906866.0,
-      "reward": 0.05839844048023224,
-      "reward_std": 0.04519660770893097,
+      "loss": 0.0288,
+      "num_tokens": 1902698.0,
+      "reward": 0.05878906697034836,
+      "reward_std": 0.04603128135204315,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.583984375,
-      "rewards/format_reward/std": 0.493378221988678,
+      "rewards/format_reward/mean": 0.587890625,
+      "rewards/format_reward/std": 0.49269601702690125,
       "step": 3
     },
     {
@@ -99,26 +99,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1589.0,
-      "completions/mean_length": 641.28125,
-      "completions/mean_terminated_length": 630.2047119140625,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_terminated_length": 1800.0,
+      "completions/mean_length": 631.861328125,
+      "completions/mean_terminated_length": 629.0900268554688,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.003581822252070741,
-      "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.0358462731569396,
-      "kl": 0.0552978515625,
+      "frac_reward_zero_std": 0.0625,
+      "grad_norm": 0.8408256552350213,
+      "kl": 0.0007734298706054688,
       "learning_rate": 1.0000000000000002e-06,
-      "loss": 0.032,
-      "num_tokens": 2560530.0,
-      "reward": 0.06679687649011612,
-      "reward_std": 0.045152708888053894,
+      "loss": 0.0369,
+      "num_tokens": 2551539.0,
+      "reward": 0.07343749701976776,
+      "reward_std": 0.0411534309387207,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.66796875,
-      "rewards/format_reward/std": 0.47140273451805115,
+      "rewards/format_reward/mean": 0.734375,
+      "rewards/format_reward/std": 0.44209739565849304,
       "step": 4
     },
     {
@@ -127,26 +127,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1533.0,
-      "completions/mean_length": 589.8203125,
-      "completions/mean_terminated_length": 586.9667358398438,
-      "completions/min_length": 225.0,
-      "completions/min_terminated_length": 225.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1767.0,
+      "completions/max_terminated_length": 1767.0,
+      "completions/mean_length": 579.943359375,
+      "completions/mean_terminated_length": 579.943359375,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.0044772778150884264,
-      "frac_reward_zero_std": 0.0625,
-      "grad_norm": 0.8048559127300979,
-      "kl": 0.05413818359375,
+      "frac_reward_zero_std": 0.21875,
+      "grad_norm": 0.5439705548720983,
+      "kl": 0.0013751983642578125,
       "learning_rate": 1.3333333333333334e-06,
-      "loss": 0.0198,
-      "num_tokens": 3145030.0,
-      "reward": 0.07988281548023224,
-      "reward_std": 0.03725838661193848,
+      "loss": 0.0314,
+      "num_tokens": 3130982.0,
+      "reward": 0.08925781399011612,
+      "reward_std": 0.026584036648273468,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.798828125,
-      "rewards/format_reward/std": 0.4012683033943176,
+      "rewards/format_reward/mean": 0.892578125,
+      "rewards/format_reward/std": 0.30995169281959534,
       "step": 5
     },
     {
@@ -155,26 +155,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1625.0,
-      "completions/max_terminated_length": 1625.0,
-      "completions/mean_length": 602.583984375,
-      "completions/mean_terminated_length": 601.2994384765625,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1988.0,
+      "completions/mean_length": 599.826171875,
+      "completions/mean_terminated_length": 591.290771484375,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.005372733378106112,
-      "frac_reward_zero_std": 0.03125,
-      "grad_norm": 1.1964223192844583,
-      "kl": 0.0577392578125,
+      "frac_reward_zero_std": 0.28125,
+      "grad_norm": 0.35924800717698446,
+      "kl": 0.001567840576171875,
       "learning_rate": 1.6666666666666667e-06,
-      "loss": 0.0274,
-      "num_tokens": 3768913.0,
-      "reward": 0.087890625,
-      "reward_std": 0.03155777230858803,
+      "loss": 0.03,
+      "num_tokens": 3753453.0,
+      "reward": 0.08671875298023224,
+      "reward_std": 0.02724354714155197,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.87890625,
-      "rewards/format_reward/std": 0.3265552520751953,
+      "rewards/format_reward/mean": 0.8671875,
+      "rewards/format_reward/std": 0.33970388770103455,
       "step": 6
     },
     {
@@ -183,26 +183,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1803.0,
-      "completions/max_terminated_length": 1803.0,
-      "completions/mean_length": 625.125,
-      "completions/mean_terminated_length": 625.125,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1959.0,
+      "completions/mean_length": 629.521484375,
+      "completions/mean_terminated_length": 626.74560546875,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.006268188941123797,
-      "frac_reward_zero_std": 0.4375,
-      "grad_norm": 0.3470545070077696,
-      "kl": 0.05816650390625,
+      "frac_reward_zero_std": 0.3125,
+      "grad_norm": 0.35624534475901054,
+      "kl": 0.00205230712890625,
       "learning_rate": 2.0000000000000003e-06,
-      "loss": 0.0427,
-      "num_tokens": 4486049.0,
-      "reward": 0.09316406399011612,
-      "reward_std": 0.018348829820752144,
+      "loss": 0.0283,
+      "num_tokens": 4472840.0,
+      "reward": 0.09335938096046448,
+      "reward_std": 0.0204335805028677,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.931640625,
-      "rewards/format_reward/std": 0.25260838866233826,
+      "rewards/format_reward/mean": 0.93359375,
+      "rewards/format_reward/std": 0.2492343932390213,
       "step": 7
     },
     {
@@ -211,26 +211,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1970.0,
-      "completions/mean_length": 564.080078125,
-      "completions/mean_terminated_length": 561.1761474609375,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1685.0,
+      "completions/max_terminated_length": 1685.0,
+      "completions/mean_length": 544.69921875,
+      "completions/mean_terminated_length": 544.69921875,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.007163644504141482,
       "frac_reward_zero_std": 0.5,
-      "grad_norm": 0.31201491179112845,
-      "kl": 0.056396484375,
+      "grad_norm": 0.3450929543712783,
+      "kl": 0.00217437744140625,
       "learning_rate": 2.3333333333333336e-06,
-      "loss": 0.0299,
-      "num_tokens": 5086362.0,
-      "reward": 0.09589843451976776,
-      "reward_std": 0.013836899772286415,
+      "loss": 0.0152,
+      "num_tokens": 5063230.0,
+      "reward": 0.09531249105930328,
+      "reward_std": 0.014789125882089138,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.958984375,
-      "rewards/format_reward/std": 0.19852031767368317,
+      "rewards/format_reward/mean": 0.953125,
+      "rewards/format_reward/std": 0.21157780289649963,
       "step": 8
     },
     {
@@ -239,26 +239,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1808.0,
-      "completions/mean_length": 541.20703125,
-      "completions/mean_terminated_length": 538.25830078125,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/max_terminated_length": 1935.0,
+      "completions/mean_length": 532.423828125,
+      "completions/mean_terminated_length": 526.4804077148438,
+      "completions/min_length": 158.0,
+      "completions/min_terminated_length": 158.0,
       "epoch": 0.008059100067159167,
-      "frac_reward_zero_std": 0.78125,
-      "grad_norm": 0.1881523338020528,
-      "kl": 0.0604248046875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.1839405208353318,
+      "kl": 0.00618743896484375,
       "learning_rate": 2.666666666666667e-06,
-      "loss": 0.0108,
-      "num_tokens": 5656484.0,
-      "reward": 0.0986328125,
-      "reward_std": 0.00546875037252903,
+      "loss": 0.0253,
+      "num_tokens": 5628855.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0026298905722796917,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.986328125,
-      "rewards/format_reward/std": 0.1162383034825325,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 9
     },
     {
@@ -267,26 +267,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1424.0,
-      "completions/mean_length": 522.45703125,
-      "completions/mean_terminated_length": 519.4716186523438,
-      "completions/min_length": 93.0,
-      "completions/min_terminated_length": 93.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1702.0,
+      "completions/max_terminated_length": 1702.0,
+      "completions/mean_length": 487.248046875,
+      "completions/mean_terminated_length": 487.248046875,
+      "completions/min_length": 97.0,
+      "completions/min_terminated_length": 97.0,
       "epoch": 0.008954555630176853,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10940100591695406,
-      "kl": 0.069091796875,
+      "grad_norm": 0.12236768536967425,
+      "kl": 0.0093231201171875,
       "learning_rate": 3e-06,
-      "loss": 0.0085,
-      "num_tokens": 6209150.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": -0.0057,
+      "num_tokens": 6163494.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 10
     },
     {
@@ -295,26 +295,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1976.0,
-      "completions/mean_length": 568.38671875,
-      "completions/mean_terminated_length": 565.4912109375,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1580.0,
+      "completions/max_terminated_length": 1580.0,
+      "completions/mean_length": 545.07421875,
+      "completions/mean_terminated_length": 545.07421875,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.009850011193194537,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 0.14514624791380354,
-      "kl": 0.0689697265625,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.12766105862016214,
+      "kl": 0.009033203125,
       "learning_rate": 3.3333333333333333e-06,
-      "loss": 0.0119,
-      "num_tokens": 6820596.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0031250000465661287,
+      "loss": -0.0035,
+      "num_tokens": 6763004.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 11
     },
     {
@@ -324,19 +324,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1487.0,
-      "completions/max_terminated_length": 1487.0,
-      "completions/mean_length": 470.0703125,
-      "completions/mean_terminated_length": 470.0703125,
-      "completions/min_length": 71.0,
-      "completions/min_terminated_length": 71.0,
+      "completions/max_length": 1583.0,
+      "completions/max_terminated_length": 1583.0,
+      "completions/mean_length": 458.548828125,
+      "completions/mean_terminated_length": 458.548828125,
+      "completions/min_length": 104.0,
+      "completions/min_terminated_length": 104.0,
       "epoch": 0.010745466756212223,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09849439146568009,
-      "kl": 0.07958984375,
+      "grad_norm": 0.2226030448008434,
+      "kl": 0.012603759765625,
       "learning_rate": 3.6666666666666666e-06,
-      "loss": 0.0002,
-      "num_tokens": 7366488.0,
+      "loss": 0.0003,
+      "num_tokens": 7302997.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -352,25 +352,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1582.0,
-      "completions/max_terminated_length": 1582.0,
-      "completions/mean_length": 516.744140625,
-      "completions/mean_terminated_length": 516.744140625,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/max_length": 1771.0,
+      "completions/max_terminated_length": 1771.0,
+      "completions/mean_length": 478.6875,
+      "completions/mean_terminated_length": 478.6875,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.011640922319229908,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.1514482044995211,
-      "kl": 0.079345703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.10169433318921767,
+      "kl": 0.0145111083984375,
       "learning_rate": 4.000000000000001e-06,
-      "loss": 0.0022,
-      "num_tokens": 7945893.0,
-      "reward": 0.09921874850988388,
-      "reward_std": 0.0026298905722796917,
+      "loss": -0.0032,
+      "num_tokens": 7862917.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 13
     },
     {
@@ -379,26 +379,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.703125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2021.0,
-      "completions/mean_length": 531.853515625,
-      "completions/mean_terminated_length": 528.886474609375,
-      "completions/min_length": 91.0,
-      "completions/min_terminated_length": 91.0,
+      "completions/max_terminated_length": 1984.0,
+      "completions/mean_length": 593.982421875,
+      "completions/mean_terminated_length": 537.9451904296875,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.012536377882247594,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.28626495216884457,
-      "kl": 0.1065673828125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.12608218316972675,
+      "kl": 0.034210205078125,
       "learning_rate": 4.333333333333334e-06,
-      "loss": -0.0048,
-      "num_tokens": 8573210.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0026,
+      "num_tokens": 8522044.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 14
     },
     {
@@ -407,26 +407,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.5,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1480.0,
-      "completions/mean_length": 483.9921875,
-      "completions/mean_terminated_length": 480.0843505859375,
-      "completions/min_length": 124.0,
-      "completions/min_terminated_length": 124.0,
+      "completions/max_terminated_length": 2048.0,
+      "completions/mean_length": 627.28515625,
+      "completions/mean_terminated_length": 532.5708618164062,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
       "epoch": 0.013431833445265278,
-      "frac_reward_zero_std": 0.84375,
-      "grad_norm": 0.31691147197037534,
-      "kl": 0.1241455078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 15.775692571832229,
+      "kl": 0.079345703125,
       "learning_rate": 4.666666666666667e-06,
-      "loss": 0.019,
-      "num_tokens": 9140438.0,
-      "reward": 0.09843750298023224,
-      "reward_std": 0.004670868627727032,
+      "loss": 0.0104,
+      "num_tokens": 9162638.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.984375,
-      "rewards/format_reward/std": 0.12414088100194931,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 15
     },
     {
@@ -435,26 +435,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1855.0,
-      "completions/max_terminated_length": 1855.0,
-      "completions/mean_length": 519.79296875,
-      "completions/mean_terminated_length": 512.6004028320312,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -6.546875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2002.0,
+      "completions/mean_length": 607.8515625,
+      "completions/mean_terminated_length": 526.5134887695312,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
       "epoch": 0.014327289008282965,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.1648930734651514,
-      "kl": 0.158447265625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 18276.22528094759,
+      "kl": 717.5,
       "learning_rate": 5e-06,
-      "loss": 0.0016,
-      "num_tokens": 9731676.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 7.1764,
+      "num_tokens": 9798962.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 16
     },
     {
@@ -463,26 +463,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
+      "completions/clipped_ratio": -6.5625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2041.0,
-      "completions/mean_length": 504.20703125,
-      "completions/mean_terminated_length": 493.0751037597656,
-      "completions/min_length": 94.0,
-      "completions/min_terminated_length": 94.0,
+      "completions/max_terminated_length": 1943.0,
+      "completions/mean_length": 605.16015625,
+      "completions/mean_terminated_length": 522.7809448242188,
+      "completions/min_length": 46.0,
+      "completions/min_terminated_length": 46.0,
       "epoch": 0.015222744571300649,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.4330453571845125,
-      "kl": 0.161865234375,
+      "grad_norm": 784.6537568296061,
+      "kl": 15.96484375,
       "learning_rate": 4.999952797253148e-06,
-      "loss": -0.0019,
-      "num_tokens": 10274966.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.1602,
+      "num_tokens": 10393940.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 17
     },
     {
@@ -491,26 +491,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.796875,
+      "completions/clipped_ratio": -6.703125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2017.0,
-      "completions/mean_length": 615.103515625,
-      "completions/mean_terminated_length": 580.6132202148438,
-      "completions/min_length": 128.0,
-      "completions/min_terminated_length": 128.0,
+      "completions/max_terminated_length": 1984.0,
+      "completions/mean_length": 598.755859375,
+      "completions/mean_terminated_length": 543.7788696289062,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.016118200134318333,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.110370207766606,
-      "kl": 0.204833984375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 20.79955935344091,
+      "kl": 1.15966796875,
       "learning_rate": 4.9998111909931225e-06,
-      "loss": 0.002,
-      "num_tokens": 10932827.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0205,
+      "num_tokens": 11043431.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 18
     },
     {
@@ -519,26 +519,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.65625,
+      "completions/clipped_ratio": -6.40625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1965.0,
-      "completions/mean_length": 600.470703125,
-      "completions/mean_terminated_length": 537.7999877929688,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_terminated_length": 1975.0,
+      "completions/mean_length": 595.953125,
+      "completions/mean_terminated_length": 481.9282531738281,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.01701365569733602,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 0.4145181105492008,
-      "kl": 0.245361328125,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 30.502487683490465,
+      "kl": 4.1640625,
       "learning_rate": 4.999575187161439e-06,
-      "loss": 0.0078,
-      "num_tokens": 11526668.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0031250000465661287,
+      "loss": 0.0478,
+      "num_tokens": 11634959.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 19
     },
     {
@@ -547,26 +547,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
+      "completions/clipped_ratio": -6.078125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2014.0,
-      "completions/mean_length": 584.4921875,
-      "completions/mean_terminated_length": 549.3680419921875,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/max_terminated_length": 2045.0,
+      "completions/mean_length": 666.94140625,
+      "completions/mean_terminated_length": 487.06842041015625,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.017909111260353706,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 1.2229303856627542,
-      "kl": 0.143310546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 9.128603733807386,
+      "kl": 1.1630859375,
       "learning_rate": 4.9992447956603455e-06,
-      "loss": -0.0046,
-      "num_tokens": 12119544.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0073,
+      "num_tokens": 12270049.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 20
     },
     {
@@ -575,26 +575,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -5.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1949.0,
-      "completions/mean_length": 480.513671875,
-      "completions/mean_terminated_length": 471.2750549316406,
-      "completions/min_length": 108.0,
-      "completions/min_terminated_length": 108.0,
+      "completions/max_terminated_length": 1939.0,
+      "completions/mean_length": 685.447265625,
+      "completions/mean_terminated_length": 487.8009033203125,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.01880456682337139,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10797165131936656,
-      "kl": 0.093505859375,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 2.0053797855307804,
+      "kl": 0.22998046875,
       "learning_rate": 4.998820030352409e-06,
-      "loss": 0.0073,
-      "num_tokens": 12646255.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": -0.0081,
+      "num_tokens": 12901686.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 21
     },
     {
@@ -603,26 +603,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.265625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1508.0,
-      "completions/mean_length": 522.076171875,
-      "completions/mean_terminated_length": 513.08251953125,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/max_terminated_length": 2000.0,
+      "completions/mean_length": 641.787109375,
+      "completions/mean_terminated_length": 501.0129089355469,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.019700022386389075,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.1267456048768045,
-      "kl": 0.0908203125,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 12.479987888813367,
+      "kl": 0.74072265625,
       "learning_rate": 4.998300909059929e-06,
-      "loss": 0.0084,
-      "num_tokens": 13210582.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.034,
+      "num_tokens": 13527305.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 22
     },
     {
@@ -631,26 +631,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.65625,
+      "completions/clipped_ratio": -6.375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1926.0,
-      "completions/mean_length": 593.544921875,
-      "completions/mean_terminated_length": 528.2428588867188,
-      "completions/min_length": 108.0,
-      "completions/min_terminated_length": 108.0,
+      "completions/max_terminated_length": 1882.0,
+      "completions/mean_length": 565.125,
+      "completions/mean_terminated_length": 445.1525573730469,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
       "epoch": 0.02059547794940676,
-      "frac_reward_zero_std": 0.5625,
-      "grad_norm": 3.0222232303189247,
-      "kl": 0.0986328125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 6.963355920650139,
+      "kl": 0.6787109375,
       "learning_rate": 4.997687453564198e-06,
-      "loss": 0.0604,
-      "num_tokens": 13784925.0,
-      "reward": 0.09121093899011612,
-      "reward_std": 0.015414923429489136,
+      "loss": 0.0195,
+      "num_tokens": 14087097.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.912109375,
-      "rewards/format_reward/std": 0.2834126651287079,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 23
     },
     {
@@ -659,20 +659,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
+      "completions/clipped_ratio": -6.640625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1936.0,
-      "completions/mean_length": 547.720703125,
-      "completions/mean_terminated_length": 526.9247436523438,
-      "completions/min_length": 133.0,
-      "completions/min_terminated_length": 133.0,
+      "completions/max_terminated_length": 2022.0,
+      "completions/mean_length": 528.884765625,
+      "completions/mean_terminated_length": 466.11041259765625,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
       "epoch": 0.021490933512424447,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.12835045627092287,
-      "kl": 0.0906982421875,
+      "grad_norm": 0.23487749119933354,
+      "kl": 0.29736328125,
       "learning_rate": 4.9969796896045775e-06,
-      "loss": 0.0009,
-      "num_tokens": 14418606.0,
+      "loss": 0.003,
+      "num_tokens": 14711134.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -687,26 +687,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.796875,
+      "completions/clipped_ratio": -6.84375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1878.0,
-      "completions/mean_length": 543.8203125,
-      "completions/mean_terminated_length": 504.6332702636719,
-      "completions/min_length": 104.0,
-      "completions/min_terminated_length": 104.0,
+      "completions/max_terminated_length": 1501.0,
+      "completions/mean_length": 411.025390625,
+      "completions/mean_terminated_length": 393.637451171875,
+      "completions/min_length": 123.0,
+      "completions/min_terminated_length": 123.0,
       "epoch": 0.02238638907544213,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.035870702477187856,
-      "kl": 0.091064453125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 7.283851009962948,
+      "kl": 0.1751708984375,
       "learning_rate": 4.996177646877426e-06,
-      "loss": 0.0009,
-      "num_tokens": 14964642.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0117,
+      "num_tokens": 15189179.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 25
     },
     {
@@ -715,26 +715,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.6875,
+      "completions/clipped_ratio": -6.78125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1726.0,
-      "completions/mean_length": 569.7109375,
-      "completions/mean_terminated_length": 509.61785888671875,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/max_terminated_length": 1775.0,
+      "completions/mean_length": 471.109375,
+      "completions/mean_terminated_length": 441.7851257324219,
+      "completions/min_length": 131.0,
+      "completions/min_terminated_length": 131.0,
       "epoch": 0.023281844638459816,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.06374825486217874,
-      "kl": 0.0888671875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.595269313741645,
+      "kl": 0.2216796875,
       "learning_rate": 4.995281359034851e-06,
-      "loss": 0.0009,
-      "num_tokens": 15560446.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0083,
+      "num_tokens": 15734499.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 26
     },
     {
@@ -743,26 +743,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.5625,
+      "completions/clipped_ratio": -6.859375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2001.0,
-      "completions/mean_length": 578.88671875,
-      "completions/mean_terminated_length": 493.89666748046875,
-      "completions/min_length": 122.0,
-      "completions/min_terminated_length": 122.0,
+      "completions/max_terminated_length": 926.0,
+      "completions/mean_length": 366.5703125,
+      "completions/mean_terminated_length": 358.7117004394531,
+      "completions/min_length": 68.0,
+      "completions/min_terminated_length": 68.0,
       "epoch": 0.0241773002014775,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.04349623576785538,
-      "kl": 0.08447265625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 3.211143343202539,
+      "kl": 0.07269287109375,
       "learning_rate": 4.994290863683296e-06,
-      "loss": 0.0008,
-      "num_tokens": 16127044.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0146,
+      "num_tokens": 16192391.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 27
     },
     {
@@ -771,26 +771,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.515625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2029.0,
-      "completions/mean_length": 635.462890625,
-      "completions/mean_terminated_length": 544.4262084960938,
-      "completions/min_length": 134.0,
-      "completions/min_terminated_length": 134.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 1249.0,
+      "completions/max_terminated_length": 1249.0,
+      "completions/mean_length": 425.619140625,
+      "completions/mean_terminated_length": 424.0589599609375,
+      "completions/min_length": 74.0,
+      "completions/min_terminated_length": 74.0,
       "epoch": 0.025072755764495188,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10869987782546223,
-      "kl": 0.0858154296875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.041783599036276396,
+      "kl": 0.03924560546875,
       "learning_rate": 4.99320620238196e-06,
-      "loss": 0.0015,
-      "num_tokens": 16737169.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0004,
+      "num_tokens": 16695076.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 28
     },
     {
@@ -799,20 +799,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.828125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1984.0,
-      "completions/mean_length": 549.55859375,
-      "completions/mean_terminated_length": 516.65869140625,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/max_terminated_length": 1408.0,
+      "completions/mean_length": 464.380859375,
+      "completions/mean_terminated_length": 461.28179931640625,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
       "epoch": 0.025968211327512872,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.060121213159789126,
-      "kl": 0.0743408203125,
+      "grad_norm": 0.07454353504766084,
+      "kl": 0.02850341796875,
       "learning_rate": 4.99202742064106e-06,
-      "loss": 0.0017,
-      "num_tokens": 17372495.0,
+      "loss": 0.0192,
+      "num_tokens": 17286791.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -827,26 +827,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1435.0,
-      "completions/mean_length": 512.666015625,
-      "completions/mean_terminated_length": 494.4604797363281,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1740.0,
+      "completions/max_terminated_length": 1740.0,
+      "completions/mean_length": 448.654296875,
+      "completions/mean_terminated_length": 448.654296875,
+      "completions/min_length": 63.0,
+      "completions/min_terminated_length": 63.0,
       "epoch": 0.026863666890530557,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08729934209278342,
-      "kl": 0.0672607421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.022840042186935416,
+      "kl": 0.025787353515625,
       "learning_rate": 4.990754567919917e-06,
-      "loss": 0.0012,
-      "num_tokens": 17937572.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0003,
+      "num_tokens": 17819094.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 30
     },
     {
@@ -856,19 +856,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1663.0,
-      "completions/max_terminated_length": 1663.0,
-      "completions/mean_length": 490.388671875,
-      "completions/mean_terminated_length": 490.388671875,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/max_length": 1376.0,
+      "completions/max_terminated_length": 1376.0,
+      "completions/mean_length": 435.240234375,
+      "completions/mean_terminated_length": 435.240234375,
+      "completions/min_length": 69.0,
+      "completions/min_terminated_length": 69.0,
       "epoch": 0.02775912245354824,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014912187748278594,
-      "kl": 0.065185546875,
+      "grad_norm": 0.0201442073656093,
+      "kl": 0.023193359375,
       "learning_rate": 4.989387697624881e-06,
-      "loss": 0.0007,
-      "num_tokens": 18491403.0,
+      "loss": 0.0002,
+      "num_tokens": 18344689.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -883,26 +883,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1426.0,
-      "completions/mean_length": 563.92578125,
-      "completions/mean_terminated_length": 561.0215454101562,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/max_terminated_length": 1879.0,
+      "completions/mean_length": 529.171875,
+      "completions/mean_terminated_length": 524.1078491210938,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.02865457801656593,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.10732893682943874,
-      "kl": 0.069091796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 11726.835967429213,
+      "kl": 544.0148620605469,
       "learning_rate": 4.987926867107095e-06,
-      "loss": 0.0117,
-      "num_tokens": 19124901.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 5.464,
+      "num_tokens": 18960393.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 32
     },
     {
@@ -911,20 +911,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1379.0,
-      "completions/max_terminated_length": 1379.0,
-      "completions/mean_length": 527.953125,
-      "completions/mean_terminated_length": 527.953125,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1324.0,
+      "completions/max_terminated_length": 1324.0,
+      "completions/mean_length": 500.341796875,
+      "completions/mean_terminated_length": 497.57257080078125,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.029550033579583614,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.013880347239008294,
-      "kl": 0.0657958984375,
+      "grad_norm": 4.4073495095910635,
+      "kl": 0.8916015625,
       "learning_rate": 4.986372137660078e-06,
-      "loss": 0.0007,
-      "num_tokens": 19712285.0,
+      "loss": 0.0089,
+      "num_tokens": 19533640.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -939,26 +939,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1420.0,
-      "completions/max_terminated_length": 1420.0,
-      "completions/mean_length": 478.341796875,
-      "completions/mean_terminated_length": 478.341796875,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1168.0,
+      "completions/max_terminated_length": 1168.0,
+      "completions/mean_length": 430.51953125,
+      "completions/mean_terminated_length": 429.8884582519531,
+      "completions/min_length": 83.0,
+      "completions/min_terminated_length": 83.0,
       "epoch": 0.030445489142601298,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.11245191546489028,
-      "kl": 0.06451416015625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09236567784912826,
+      "kl": 0.028533935546875,
       "learning_rate": 4.984723574517165e-06,
-      "loss": 0.0043,
-      "num_tokens": 20235964.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": -0.0062,
+      "num_tokens": 20032834.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 34
     },
     {
@@ -967,26 +967,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1743.0,
-      "completions/max_terminated_length": 1743.0,
-      "completions/mean_length": 538.541015625,
-      "completions/mean_terminated_length": 538.541015625,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1237.0,
+      "completions/max_terminated_length": 1237.0,
+      "completions/mean_length": 499.416015625,
+      "completions/mean_terminated_length": 498.78277587890625,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.03134094470561898,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.10103218237797797,
-      "kl": 0.0614013671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.01651332440268518,
+      "kl": 0.02386474609375,
       "learning_rate": 4.9829812468487655e-06,
-      "loss": -0.0031,
-      "num_tokens": 20831809.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0002,
+      "num_tokens": 20608647.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 35
     },
     {
@@ -995,26 +995,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.765625,
-      "completions/max_length": 1589.0,
-      "completions/max_terminated_length": 1589.0,
-      "completions/mean_length": 534.1484375,
-      "completions/mean_terminated_length": 521.9295654296875,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
+      "completions/clipped_ratio": -6.734375,
+      "completions/max_length": 1281.0,
+      "completions/max_terminated_length": 1281.0,
+      "completions/mean_length": 500.65234375,
+      "completions/mean_terminated_length": 480.89093017578125,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.03223640026863667,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 20.200206186476855,
-      "kl": 2.783203125,
+      "grad_norm": 2529.782303345979,
+      "kl": 48.9375,
       "learning_rate": 4.981145227759457e-06,
-      "loss": 0.0329,
-      "num_tokens": 21423245.0,
-      "reward": 0.09707031399011612,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.5007,
+      "num_tokens": 21182933.0,
+      "reward": 0.09726563096046448,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.970703125,
-      "rewards/format_reward/std": 0.16880230605602264,
+      "rewards/format_reward/mean": 0.97265625,
+      "rewards/format_reward/std": 0.16324250400066376,
       "step": 36
     },
     {
@@ -1023,20 +1023,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1672.0,
-      "completions/max_terminated_length": 1672.0,
-      "completions/mean_length": 572.818359375,
-      "completions/mean_terminated_length": 571.872802734375,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 1611.0,
+      "completions/max_terminated_length": 1611.0,
+      "completions/mean_length": 537.484375,
+      "completions/mean_terminated_length": 530.6324462890625,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.03313185583165435,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.6440547223266587,
-      "kl": 0.12445068359375,
+      "grad_norm": 2.4928540218102038,
+      "kl": 0.14483642578125,
       "learning_rate": 4.979215594284924e-06,
-      "loss": 0.0065,
-      "num_tokens": 22021680.0,
+      "loss": 0.0115,
+      "num_tokens": 21763277.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1051,26 +1051,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1848.0,
-      "completions/max_terminated_length": 1848.0,
-      "completions/mean_length": 580.0234375,
-      "completions/mean_terminated_length": 580.0234375,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1866.0,
+      "completions/mean_length": 540.787109375,
+      "completions/mean_terminated_length": 532.1146240234375,
+      "completions/min_length": 148.0,
+      "completions/min_terminated_length": 148.0,
       "epoch": 0.03402731139467204,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.058757743235516635,
-      "kl": 0.06072998046875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 1.980183337733373,
+      "kl": 0.057220458984375,
       "learning_rate": 4.977192427388722e-06,
-      "loss": 0.0025,
-      "num_tokens": 22643772.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0236,
+      "num_tokens": 22365280.0,
+      "reward": 0.09785156697034836,
+      "reward_std": 0.00316358613781631,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.978515625,
+      "rewards/format_reward/std": 0.14513419568538666,
       "step": 38
     },
     {
@@ -1080,19 +1080,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1413.0,
-      "completions/max_terminated_length": 1413.0,
-      "completions/mean_length": 548.609375,
-      "completions/mean_terminated_length": 548.609375,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_length": 1825.0,
+      "completions/max_terminated_length": 1825.0,
+      "completions/mean_length": 511.64453125,
+      "completions/mean_terminated_length": 511.64453125,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.03492276695768973,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012598270654231074,
-      "kl": 0.058837890625,
+      "grad_norm": 0.018814791012791984,
+      "kl": 0.019622802734375,
       "learning_rate": 4.9750758119588824e-06,
-      "loss": 0.0006,
-      "num_tokens": 23225860.0,
+      "loss": 0.0002,
+      "num_tokens": 22928442.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1107,26 +1107,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1677.0,
-      "completions/max_terminated_length": 1677.0,
-      "completions/mean_length": 580.900390625,
-      "completions/mean_terminated_length": 580.900390625,
-      "completions/min_length": 237.0,
-      "completions/min_terminated_length": 237.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1528.0,
+      "completions/mean_length": 538.15625,
+      "completions/mean_terminated_length": 533.9725952148438,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.03581822252070741,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012729652253403266,
-      "kl": 0.05987548828125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09930995271707406,
+      "kl": 0.02301025390625,
       "learning_rate": 4.972865836804349e-06,
-      "loss": 0.0006,
-      "num_tokens": 23813585.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0122,
+      "num_tokens": 23494282.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 40
     },
     {
@@ -1135,26 +1135,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1436.0,
-      "completions/max_terminated_length": 1436.0,
-      "completions/mean_length": 509.9296875,
-      "completions/mean_terminated_length": 509.9296875,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1301.0,
+      "completions/mean_length": 460.962890625,
+      "completions/mean_terminated_length": 452.0592956542969,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.036713678083725096,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07495467584140274,
-      "kl": 0.05712890625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 41.394496617699204,
+      "kl": 6.682464599609375,
       "learning_rate": 4.970562594651254e-06,
-      "loss": 0.0019,
-      "num_tokens": 24347917.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0667,
+      "num_tokens": 24003543.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 41
     },
     {
@@ -1163,20 +1163,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1801.0,
-      "completions/max_terminated_length": 1801.0,
-      "completions/mean_length": 552.830078125,
-      "completions/mean_terminated_length": 552.830078125,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1939.0,
+      "completions/mean_length": 475.712890625,
+      "completions/mean_terminated_length": 466.44598388671875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.03760913364674278,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014256239329774286,
-      "kl": 0.05963134765625,
+      "grad_norm": 24.391100211271684,
+      "kl": 1.68402099609375,
       "learning_rate": 4.968166182139026e-06,
-      "loss": 0.0006,
-      "num_tokens": 24958934.0,
+      "loss": 0.0169,
+      "num_tokens": 24575076.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1191,20 +1191,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1480.0,
-      "completions/max_terminated_length": 1480.0,
-      "completions/mean_length": 481.08984375,
-      "completions/mean_terminated_length": 481.08984375,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1772.0,
+      "completions/mean_length": 434.248046875,
+      "completions/mean_terminated_length": 429.7013854980469,
+      "completions/min_length": 79.0,
+      "completions/min_terminated_length": 79.0,
       "epoch": 0.038504589209760465,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012410355612420237,
-      "kl": 0.06182861328125,
+      "grad_norm": 0.5910587074093245,
+      "kl": 0.1524658203125,
       "learning_rate": 4.9656766998163306e-06,
-      "loss": 0.0006,
-      "num_tokens": 25469028.0,
+      "loss": 0.0015,
+      "num_tokens": 25061187.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1219,26 +1219,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.9375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1624.0,
-      "completions/mean_length": 538.57421875,
-      "completions/mean_terminated_length": 533.486328125,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/max_terminated_length": 2036.0,
+      "completions/mean_length": 470.498046875,
+      "completions/mean_terminated_length": 464.6909484863281,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.03940004477277815,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.11410330840934924,
-      "kl": 0.081298828125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.14956360476131622,
+      "kl": 0.06591796875,
       "learning_rate": 4.963094252136865e-06,
-      "loss": 0.0112,
-      "num_tokens": 26044170.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0007,
+      "num_tokens": 25601474.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 44
     },
     {
@@ -1247,26 +1247,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1548.0,
-      "completions/max_terminated_length": 1548.0,
-      "completions/mean_length": 526.244140625,
-      "completions/mean_terminated_length": 526.244140625,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -6.78125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1885.0,
+      "completions/mean_length": 506.697265625,
+      "completions/mean_terminated_length": 471.13250732421875,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
       "epoch": 0.040295500335795834,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008982764516293115,
-      "kl": 0.06011962890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 16.11234098312743,
+      "kl": 1.211181640625,
       "learning_rate": 4.960418947454958e-06,
-      "loss": 0.0006,
-      "num_tokens": 26641527.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0129,
+      "num_tokens": 26188823.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 45
     },
     {
@@ -1275,20 +1275,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1526.0,
-      "completions/max_terminated_length": 1526.0,
-      "completions/mean_length": 545.90234375,
-      "completions/mean_terminated_length": 545.90234375,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 1863.0,
+      "completions/max_terminated_length": 1863.0,
+      "completions/mean_length": 490.3203125,
+      "completions/mean_terminated_length": 479.1208190917969,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.04119095589881352,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011841241698184731,
-      "kl": 0.0606689453125,
+      "grad_norm": 0.9840541759000635,
+      "kl": 0.1519775390625,
       "learning_rate": 4.957650898021038e-06,
-      "loss": 0.0006,
-      "num_tokens": 27219845.0,
+      "loss": 0.0015,
+      "num_tokens": 26738683.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -1305,24 +1305,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1706.0,
-      "completions/mean_length": 566.806640625,
-      "completions/mean_terminated_length": 563.9080200195312,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/max_terminated_length": 1553.0,
+      "completions/mean_length": 475.044921875,
+      "completions/mean_terminated_length": 471.96673583984375,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.04208641146183121,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.11052168377753285,
-      "kl": 0.0595703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.01990518206423815,
+      "kl": 0.032440185546875,
       "learning_rate": 4.954790219976915e-06,
-      "loss": 0.0108,
-      "num_tokens": 27821794.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0003,
+      "num_tokens": 27293650.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 47
     },
     {
@@ -1331,26 +1331,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
+      "completions/clipped_ratio": -6.859375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1283.0,
-      "completions/mean_length": 598.818359375,
-      "completions/mean_terminated_length": 588.4580078125,
-      "completions/min_length": 240.0,
-      "completions/min_terminated_length": 240.0,
+      "completions/max_terminated_length": 1737.0,
+      "completions/mean_length": 533.146484375,
+      "completions/mean_terminated_length": 514.7256469726562,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.042981867024848894,
-      "frac_reward_zero_std": 0.875,
-      "grad_norm": 3.73818212098571,
-      "kl": 0.09808349609375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 7.314941473605262,
+      "kl": 0.32373046875,
       "learning_rate": 4.95183703335091e-06,
-      "loss": 0.0196,
-      "num_tokens": 28483477.0,
-      "reward": 0.09726563096046448,
-      "reward_std": 0.003839729819446802,
+      "loss": 0.0205,
+      "num_tokens": 27921709.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.97265625,
-      "rewards/format_reward/std": 0.16324250400066376,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 48
     },
     {
@@ -1359,26 +1359,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.90625,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1792.0,
-      "completions/mean_length": 584.134765625,
-      "completions/mean_terminated_length": 581.2700805664062,
-      "completions/min_length": 257.0,
-      "completions/min_terminated_length": 257.0,
+      "completions/max_terminated_length": 1629.0,
+      "completions/mean_length": 502.583984375,
+      "completions/mean_terminated_length": 488.8458557128906,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.04387732258786658,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009853203927592551,
-      "kl": 0.05950927734375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 4.053172103770152,
+      "kl": 0.665435791015625,
       "learning_rate": 4.948791462052819e-06,
-      "loss": 0.0006,
-      "num_tokens": 29061482.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0282,
+      "num_tokens": 28457960.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 49
     },
     {
@@ -1387,26 +1387,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1504.0,
-      "completions/max_terminated_length": 1504.0,
-      "completions/mean_length": 558.048828125,
-      "completions/mean_terminated_length": 558.048828125,
-      "completions/min_length": 146.0,
-      "completions/min_terminated_length": 146.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1479.0,
+      "completions/max_terminated_length": 1479.0,
+      "completions/mean_length": 491.30078125,
+      "completions/mean_terminated_length": 487.6062927246094,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
       "epoch": 0.04477277815088426,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06158028008338189,
-      "kl": 0.06207275390625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.032153653495619945,
+      "kl": 0.06005859375,
       "learning_rate": 4.945653633868716e-06,
-      "loss": 0.0002,
-      "num_tokens": 29667203.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0006,
+      "num_tokens": 29029506.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 50
     },
     {
@@ -1415,26 +1415,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1496.0,
-      "completions/max_terminated_length": 1496.0,
-      "completions/mean_length": 589.54296875,
-      "completions/mean_terminated_length": 589.54296875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -6.890625,
+      "completions/max_length": 1245.0,
+      "completions/max_terminated_length": 1245.0,
+      "completions/mean_length": 523.353515625,
+      "completions/mean_terminated_length": 514.160400390625,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.04566823371390195,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.085918900087203,
-      "kl": 0.06072998046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.04442617311439247,
+      "kl": 0.08599853515625,
       "learning_rate": 4.942423680455584e-06,
-      "loss": 0.0021,
-      "num_tokens": 30277769.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0009,
+      "num_tokens": 29606183.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 51
     },
     {
@@ -1444,19 +1444,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1400.0,
-      "completions/max_terminated_length": 1400.0,
-      "completions/mean_length": 566.83203125,
-      "completions/mean_terminated_length": 565.9589233398438,
-      "completions/min_length": 120.0,
-      "completions/min_terminated_length": 120.0,
+      "completions/max_length": 1283.0,
+      "completions/max_terminated_length": 1283.0,
+      "completions/mean_length": 494.517578125,
+      "completions/mean_terminated_length": 493.626220703125,
+      "completions/min_length": 128.0,
+      "completions/min_terminated_length": 128.0,
       "epoch": 0.04656368927691963,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 13.53527608696407,
-      "kl": 0.06634521484375,
+      "grad_norm": 0.057162846752272095,
+      "kl": 0.036529541015625,
       "learning_rate": 4.939101737335802e-06,
-      "loss": 0.0112,
-      "num_tokens": 30857379.0,
+      "loss": -0.0034,
+      "num_tokens": 30148768.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1472,25 +1472,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2036.0,
-      "completions/max_terminated_length": 2036.0,
-      "completions/mean_length": 568.298828125,
-      "completions/mean_terminated_length": 566.6731567382812,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1307.0,
+      "completions/max_terminated_length": 1307.0,
+      "completions/mean_length": 488.09765625,
+      "completions/mean_terminated_length": 487.25439453125,
+      "completions/min_length": 104.0,
+      "completions/min_terminated_length": 104.0,
       "epoch": 0.047459144839937316,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 753.4933997875953,
-      "kl": 40.29571533203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05128260945429217,
+      "kl": 0.037628173828125,
       "learning_rate": 4.935687943891447e-06,
-      "loss": 0.404,
-      "num_tokens": 31445404.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0017,
+      "num_tokens": 30695730.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 53
     },
     {
@@ -1499,20 +1499,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1762.0,
-      "completions/mean_length": 606.25,
-      "completions/mean_terminated_length": 603.4285888671875,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2043.0,
+      "completions/max_terminated_length": 2043.0,
+      "completions/mean_length": 547.087890625,
+      "completions/mean_terminated_length": 547.087890625,
+      "completions/min_length": 116.0,
+      "completions/min_terminated_length": 116.0,
       "epoch": 0.048354600402955,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07880783979440148,
-      "kl": 0.0592041015625,
+      "grad_norm": 0.06726520989605488,
+      "kl": 0.023834228515625,
       "learning_rate": 4.932182443358458e-06,
-      "loss": 0.0101,
-      "num_tokens": 32091612.0,
+      "loss": -0.001,
+      "num_tokens": 31311647.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1527,26 +1527,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1750.0,
-      "completions/mean_length": 554.294921875,
-      "completions/mean_terminated_length": 548.4373168945312,
-      "completions/min_length": 157.0,
-      "completions/min_terminated_length": 157.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 1871.0,
+      "completions/max_terminated_length": 1871.0,
+      "completions/mean_length": 508.474609375,
+      "completions/mean_terminated_length": 499.8717956542969,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.04925005596597269,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08726278881698582,
-      "kl": 0.05682373046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.031009644101716777,
+      "kl": 0.079254150390625,
       "learning_rate": 4.928585382820616e-06,
-      "loss": 0.0059,
-      "num_tokens": 32635283.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0008,
+      "num_tokens": 31831858.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 55
     },
     {
@@ -1555,26 +1555,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1862.0,
-      "completions/max_terminated_length": 1862.0,
-      "completions/mean_length": 616.517578125,
-      "completions/mean_terminated_length": 616.517578125,
-      "completions/min_length": 278.0,
-      "completions/min_terminated_length": 278.0,
+      "completions/clipped_ratio": -6.90625,
+      "completions/max_length": 1503.0,
+      "completions/max_terminated_length": 1503.0,
+      "completions/mean_length": 577.771484375,
+      "completions/mean_terminated_length": 570.49609375,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.050145511528990376,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00787517172635136,
-      "kl": 0.05615234375,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.17520768141349025,
+      "kl": 0.1038818359375,
       "learning_rate": 4.924896913203376e-06,
-      "loss": 0.0006,
-      "num_tokens": 33261820.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.001,
+      "num_tokens": 32438557.0,
+      "reward": 0.09921875596046448,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 56
     },
     {
@@ -1583,26 +1583,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1443.0,
-      "completions/mean_length": 579.927734375,
-      "completions/mean_terminated_length": 577.0548095703125,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/max_terminated_length": 2004.0,
+      "completions/mean_length": 564.052734375,
+      "completions/mean_terminated_length": 560.36474609375,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.05104096709200806,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09467657721643793,
-      "kl": 0.0556640625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10349796740366579,
+      "kl": 0.03631591796875,
       "learning_rate": 4.921117189267535e-06,
-      "loss": 0.0123,
-      "num_tokens": 33867159.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0141,
+      "num_tokens": 33035768.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0018486406188458204,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 57
     },
     {
@@ -1612,25 +1612,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1406.0,
-      "completions/max_terminated_length": 1406.0,
-      "completions/mean_length": 573.78125,
-      "completions/mean_terminated_length": 573.78125,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_length": 1589.0,
+      "completions/max_terminated_length": 1589.0,
+      "completions/mean_length": 542.61328125,
+      "completions/mean_terminated_length": 542.61328125,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.051936422655025745,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007058775392801076,
-      "kl": 0.05938720703125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.08585433577443426,
+      "kl": 0.021392822265625,
       "learning_rate": 4.917246369602742e-06,
-      "loss": 0.0006,
-      "num_tokens": 34475127.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0013,
+      "num_tokens": 33627778.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 58
     },
     {
@@ -1639,20 +1639,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1731.0,
-      "completions/max_terminated_length": 1731.0,
-      "completions/mean_length": 574.1796875,
-      "completions/mean_terminated_length": 572.4011840820312,
-      "completions/min_length": 261.0,
-      "completions/min_terminated_length": 261.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1705.0,
+      "completions/max_terminated_length": 1705.0,
+      "completions/mean_length": 545.0625,
+      "completions/mean_terminated_length": 539.7598266601562,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.05283187821804343,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 6.081099094392287,
-      "kl": 0.35699462890625,
+      "grad_norm": 0.06357442394169845,
+      "kl": 0.1036376953125,
       "learning_rate": 4.9132846166208355e-06,
-      "loss": 0.0019,
-      "num_tokens": 35081475.0,
+      "loss": -0.0002,
+      "num_tokens": 34219218.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1667,27 +1667,27 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1659.0,
-      "completions/max_terminated_length": 1659.0,
-      "completions/mean_length": 580.228515625,
-      "completions/mean_terminated_length": 580.228515625,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 1646.0,
+      "completions/max_terminated_length": 1646.0,
+      "completions/mean_length": 561.75,
+      "completions/mean_terminated_length": 558.19091796875,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.053727333781061114,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007985885460781206,
-      "kl": 0.05706787109375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09726885501749043,
+      "kl": 0.11737060546875,
       "learning_rate": 4.9092320965490365e-06,
       "loss": 0.0006,
-      "num_tokens": 35702376.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "num_tokens": 34830658.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
-      "step": 60
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
+      "step": 60
     },
     {
       "clip_ratio/high_max": 0.0,
@@ -1695,26 +1695,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1325.0,
-      "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 575.150390625,
-      "completions/mean_terminated_length": 575.150390625,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2037.0,
+      "completions/max_terminated_length": 2037.0,
+      "completions/mean_length": 563.16015625,
+      "completions/mean_terminated_length": 557.4133911132812,
+      "completions/min_length": 108.0,
+      "completions/min_terminated_length": 108.0,
       "epoch": 0.0546227893440788,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008989013474146063,
-      "kl": 0.05889892578125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 7.603189195829842,
+      "kl": 0.100677490234375,
       "learning_rate": 4.905088979422971e-06,
-      "loss": 0.0006,
-      "num_tokens": 36318869.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0128,
+      "num_tokens": 35441012.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 61
     },
     {
@@ -1723,26 +1723,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1401.0,
-      "completions/max_terminated_length": 1401.0,
-      "completions/mean_length": 543.685546875,
-      "completions/mean_terminated_length": 542.7866821289062,
-      "completions/min_length": 97.0,
-      "completions/min_terminated_length": 97.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1732.0,
+      "completions/max_terminated_length": 1732.0,
+      "completions/mean_length": 518.525390625,
+      "completions/mean_terminated_length": 518.525390625,
+      "completions/min_length": 119.0,
+      "completions/min_terminated_length": 119.0,
       "epoch": 0.05551824490709648,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01919284660807941,
-      "kl": 0.06610107421875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10269453437417964,
+      "kl": 0.021270751953125,
       "learning_rate": 4.900855439079536e-06,
-      "loss": 0.0007,
-      "num_tokens": 36900388.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0029,
+      "num_tokens": 36009649.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 62
     },
     {
@@ -1751,20 +1751,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1739.0,
-      "completions/max_terminated_length": 1739.0,
-      "completions/mean_length": 568.287109375,
-      "completions/mean_terminated_length": 568.287109375,
-      "completions/min_length": 236.0,
-      "completions/min_terminated_length": 236.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1435.0,
+      "completions/max_terminated_length": 1272.0,
+      "completions/mean_length": 542.779296875,
+      "completions/mean_terminated_length": 540.3529663085938,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.056413700470114174,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07846205338958102,
-      "kl": 0.05694580078125,
+      "grad_norm": 0.07525577027012759,
+      "kl": 0.019866943359375,
       "learning_rate": 4.8965316531496055e-06,
-      "loss": -0.0019,
-      "num_tokens": 37504423.0,
+      "loss": 0.0009,
+      "num_tokens": 36600624.0,
       "reward": 0.099609375,
       "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
@@ -1779,26 +1779,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1904.0,
-      "completions/max_terminated_length": 1904.0,
-      "completions/mean_length": 603.353515625,
-      "completions/mean_terminated_length": 601.3953247070312,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1666.0,
+      "completions/max_terminated_length": 1666.0,
+      "completions/mean_length": 603.5859375,
+      "completions/mean_terminated_length": 599.5882568359375,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.05730915603313186,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 4.810189704077544,
-      "kl": 0.062744140625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.7608650680571825,
+      "kl": 0.17962646484375,
       "learning_rate": 4.892117803050578e-06,
-      "loss": 0.0096,
-      "num_tokens": 38096204.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0018,
+      "num_tokens": 37192524.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 64
     },
     {
@@ -1808,19 +1808,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1312.0,
-      "completions/max_terminated_length": 1312.0,
-      "completions/mean_length": 587.44921875,
-      "completions/mean_terminated_length": 586.2974243164062,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1343.0,
+      "completions/mean_length": 553.01953125,
+      "completions/mean_terminated_length": 550.0939331054688,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.05820461159614954,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 6.992893766082191,
-      "kl": 0.5133056640625,
+      "grad_norm": 0.04778415655998196,
+      "kl": 0.0208740234375,
       "learning_rate": 4.887614073978761e-06,
-      "loss": 0.0079,
-      "num_tokens": 38695746.0,
+      "loss": 0.0126,
+      "num_tokens": 37774438.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -1835,26 +1835,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1648.0,
-      "completions/max_terminated_length": 1648.0,
-      "completions/mean_length": 590.52734375,
-      "completions/mean_terminated_length": 590.52734375,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1554.0,
+      "completions/mean_length": 564.185546875,
+      "completions/mean_terminated_length": 561.2817993164062,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
       "epoch": 0.05910006715916723,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006834195519283801,
-      "kl": 0.05694580078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09161873092433927,
+      "kl": 0.020843505859375,
       "learning_rate": 4.883020654901609e-06,
-      "loss": 0.0006,
-      "num_tokens": 39327728.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0103,
+      "num_tokens": 38392933.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 66
     },
     {
@@ -1863,26 +1863,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1189.0,
-      "completions/max_terminated_length": 1189.0,
-      "completions/mean_length": 539.552734375,
-      "completions/mean_terminated_length": 538.74169921875,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1349.0,
+      "completions/max_terminated_length": 1349.0,
+      "completions/mean_length": 525.9296875,
+      "completions/mean_terminated_length": 525.9296875,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
       "epoch": 0.05999552272218491,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1445536.0669629306,
-      "kl": 23296.041015625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.012620561722483294,
+      "kl": 0.022552490234375,
       "learning_rate": 4.878337738549785e-06,
-      "loss": 233.4053,
-      "num_tokens": 39910283.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 38968513.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 67
     },
     {
@@ -1891,26 +1891,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1645.0,
-      "completions/max_terminated_length": 1645.0,
-      "completions/mean_length": 603.494140625,
-      "completions/mean_terminated_length": 603.494140625,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1809.0,
+      "completions/mean_length": 588.72265625,
+      "completions/mean_terminated_length": 585.866943359375,
+      "completions/min_length": 218.0,
+      "completions/min_terminated_length": 218.0,
       "epoch": 0.060890978285202596,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007074653107084449,
-      "kl": 0.05511474609375,
+      "frac_reward_zero_std": 0.875,
+      "grad_norm": 0.1361201287941182,
+      "kl": 0.018341064453125,
       "learning_rate": 4.873565521409082e-06,
-      "loss": 0.0006,
-      "num_tokens": 40534392.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0149,
+      "num_tokens": 39585059.0,
+      "reward": 0.09921874850988388,
+      "reward_std": 0.0031250000465661287,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.9921875,
+      "rewards/format_reward/std": 0.08812850713729858,
       "step": 68
     },
     {
@@ -1919,26 +1919,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1478.0,
-      "completions/max_terminated_length": 1478.0,
-      "completions/mean_length": 589.14453125,
-      "completions/mean_terminated_length": 589.14453125,
-      "completions/min_length": 221.0,
-      "completions/min_terminated_length": 221.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1505.0,
+      "completions/max_terminated_length": 1505.0,
+      "completions/mean_length": 562.810546875,
+      "completions/mean_terminated_length": 561.7357788085938,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.06178643384822028,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.10404347225648633,
-      "kl": 0.05804443359375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.019277916524184716,
+      "kl": 0.035003662109375,
       "learning_rate": 4.868704203712173e-06,
-      "loss": 0.0054,
-      "num_tokens": 41128066.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0003,
+      "num_tokens": 40165250.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 69
     },
     {
@@ -1948,25 +1948,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1608.0,
-      "completions/max_terminated_length": 1608.0,
-      "completions/mean_length": 570.591796875,
-      "completions/mean_terminated_length": 570.591796875,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/max_length": 1305.0,
+      "completions/max_terminated_length": 1305.0,
+      "completions/mean_length": 560.72265625,
+      "completions/mean_terminated_length": 560.72265625,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.06268188941123796,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06680192103628023,
-      "kl": 0.05828857421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009928765161792547,
+      "kl": 0.017822265625,
       "learning_rate": 4.86375398943021e-06,
-      "loss": 0.002,
-      "num_tokens": 41723441.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 40755572.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 70
     },
     {
@@ -1975,26 +1975,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1842.0,
-      "completions/max_terminated_length": 1842.0,
-      "completions/mean_length": 564.57421875,
-      "completions/mean_terminated_length": 562.3902587890625,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1758.0,
+      "completions/max_terminated_length": 1758.0,
+      "completions/mean_length": 550.47265625,
+      "completions/mean_terminated_length": 550.47265625,
+      "completions/min_length": 221.0,
+      "completions/min_terminated_length": 221.0,
       "epoch": 0.06357734497425566,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2.1606000379295507,
-      "kl": 0.06298828125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.08245008848029971,
+      "kl": 0.017364501953125,
       "learning_rate": 4.858715086264274e-06,
-      "loss": 0.0042,
-      "num_tokens": 42320855.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0008,
+      "num_tokens": 41345766.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 71
     },
     {
@@ -2004,25 +2004,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1384.0,
-      "completions/max_terminated_length": 1384.0,
-      "completions/mean_length": 597.32421875,
-      "completions/mean_terminated_length": 597.32421875,
-      "completions/min_length": 241.0,
-      "completions/min_terminated_length": 241.0,
+      "completions/max_length": 1483.0,
+      "completions/max_terminated_length": 1483.0,
+      "completions/mean_length": 575.744140625,
+      "completions/mean_terminated_length": 575.744140625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.06447280053727333,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06791730290876942,
-      "kl": 0.05657958984375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009114395477044235,
+      "kl": 0.01678466796875,
       "learning_rate": 4.853587705636646e-06,
-      "loss": 0.0004,
-      "num_tokens": 42959485.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 41973347.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 72
     },
     {
@@ -2032,19 +2032,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1228.0,
-      "completions/max_terminated_length": 1228.0,
-      "completions/mean_length": 530.748046875,
-      "completions/mean_terminated_length": 529.383544921875,
-      "completions/min_length": 231.0,
-      "completions/min_terminated_length": 231.0,
+      "completions/max_length": 1467.0,
+      "completions/max_terminated_length": 1467.0,
+      "completions/mean_length": 509.81640625,
+      "completions/mean_terminated_length": 508.53619384765625,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.06536825610029103,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 937.0925069160255,
-      "kl": 59.2928466796875,
+      "grad_norm": 0.01945409715205245,
+      "kl": 0.033905029296875,
       "learning_rate": 4.84837206268195e-06,
-      "loss": 0.5913,
-      "num_tokens": 43516316.0,
+      "loss": 0.0003,
+      "num_tokens": 42519461.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2059,26 +2059,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1156.0,
-      "completions/mean_length": 516.6015625,
-      "completions/mean_terminated_length": 513.6046752929688,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1322.0,
+      "completions/max_terminated_length": 1322.0,
+      "completions/mean_length": 497.779296875,
+      "completions/mean_terminated_length": 497.779296875,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.0662637116633087,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10313094352636988,
-      "kl": 0.056884765625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.012293864921911191,
+      "kl": 0.02044677734375,
       "learning_rate": 4.8430683762381195e-06,
-      "loss": 0.0177,
-      "num_tokens": 44058032.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 43051540.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 74
     },
     {
@@ -2087,26 +2087,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1437.0,
-      "completions/max_terminated_length": 1437.0,
-      "completions/mean_length": 566.34765625,
-      "completions/mean_terminated_length": 566.34765625,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1970.0,
+      "completions/mean_length": 557.455078125,
+      "completions/mean_terminated_length": 551.60986328125,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.0671591672263264,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00747549164256916,
-      "kl": 0.05645751953125,
+      "frac_reward_zero_std": 0.84375,
+      "grad_norm": 0.13630913617793994,
+      "kl": 0.0151824951171875,
       "learning_rate": 4.837676868837213e-06,
-      "loss": 0.0006,
-      "num_tokens": 44658914.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0097,
+      "num_tokens": 43647869.0,
+      "reward": 0.09882812201976776,
+      "reward_std": 0.004192390479147434,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.98828125,
+      "rewards/format_reward/std": 0.10772226005792618,
       "step": 75
     },
     {
@@ -2115,20 +2115,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1354.0,
-      "completions/mean_length": 527.84375,
-      "completions/mean_terminated_length": 520.9234008789062,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_terminated_length": 1829.0,
+      "completions/mean_length": 522.078125,
+      "completions/mean_terminated_length": 519.0919799804688,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.06805462278934409,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06719001887293087,
-      "kl": 0.06622314453125,
+      "grad_norm": 0.06750608349497755,
+      "kl": 0.017974853515625,
       "learning_rate": 4.832197766696085e-06,
-      "loss": 0.0199,
-      "num_tokens": 45217522.0,
+      "loss": 0.0069,
+      "num_tokens": 44203525.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2143,26 +2143,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1606.0,
-      "completions/max_terminated_length": 1606.0,
-      "completions/mean_length": 570.078125,
-      "completions/mean_terminated_length": 568.4344482421875,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1706.0,
+      "completions/mean_length": 554.322265625,
+      "completions/mean_terminated_length": 548.4647216796875,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.06895007835236176,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.03159919437264003,
-      "kl": 0.060546875,
+      "frac_reward_zero_std": 0.90625,
+      "grad_norm": 0.10947781516811841,
+      "kl": 0.015289306640625,
       "learning_rate": 4.826631299706887e-06,
-      "loss": 0.0006,
-      "num_tokens": 45816602.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0199,
+      "num_tokens": 44794538.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0023437500931322575,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 77
     },
     {
@@ -2172,25 +2172,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1261.0,
-      "completions/max_terminated_length": 1261.0,
-      "completions/mean_length": 537.560546875,
-      "completions/mean_terminated_length": 537.560546875,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/max_length": 1439.0,
+      "completions/max_terminated_length": 1439.0,
+      "completions/mean_length": 561.875,
+      "completions/mean_terminated_length": 561.875,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.06984553391537945,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0670951539644897,
-      "kl": 0.05682373046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.012482978592863939,
+      "kl": 0.01763916015625,
       "learning_rate": 4.820977701427424e-06,
-      "loss": 0.0036,
-      "num_tokens": 46374281.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 45364666.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 78
     },
     {
@@ -2200,25 +2200,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1410.0,
-      "completions/max_terminated_length": 1410.0,
-      "completions/mean_length": 548.822265625,
-      "completions/mean_terminated_length": 548.822265625,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/max_length": 1624.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 549.216796875,
+      "completions/mean_terminated_length": 549.216796875,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.07074098947839713,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00830534868975707,
-      "kl": 0.05743408203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05673428927364843,
+      "kl": 0.01702880859375,
       "learning_rate": 4.81523720907136e-06,
-      "loss": 0.0006,
-      "num_tokens": 46976702.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 45967289.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 79
     },
     {
@@ -2228,19 +2228,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1503.0,
-      "completions/max_terminated_length": 1503.0,
-      "completions/mean_length": 526.49609375,
-      "completions/mean_terminated_length": 526.49609375,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/max_length": 1561.0,
+      "completions/max_terminated_length": 1561.0,
+      "completions/mean_length": 518.8046875,
+      "completions/mean_terminated_length": 518.8046875,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.07163644504141482,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01078274311527768,
-      "kl": 0.0574951171875,
+      "grad_norm": 0.006457555530544499,
+      "kl": 0.016326904296875,
       "learning_rate": 4.809410063498254e-06,
-      "loss": 0.0006,
-      "num_tokens": 47558924.0,
+      "loss": 0.0002,
+      "num_tokens": 46545573.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2256,25 +2256,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1105.0,
-      "completions/max_terminated_length": 1105.0,
-      "completions/mean_length": 527.138671875,
-      "completions/mean_terminated_length": 527.138671875,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/max_length": 1591.0,
+      "completions/max_terminated_length": 1591.0,
+      "completions/mean_length": 538.142578125,
+      "completions/mean_terminated_length": 538.142578125,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.0725319006044325,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011012657853872506,
-      "kl": 0.0562744140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06228818790381435,
+      "kl": 0.016632080078125,
       "learning_rate": 4.8034965092034656e-06,
-      "loss": 0.0006,
-      "num_tokens": 48152355.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0002,
+      "num_tokens": 47144638.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 81
     },
     {
@@ -2283,20 +2283,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1796.0,
-      "completions/max_terminated_length": 1796.0,
-      "completions/mean_length": 570.82421875,
-      "completions/mean_terminated_length": 570.82421875,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2037.0,
+      "completions/mean_length": 554.693359375,
+      "completions/mean_terminated_length": 551.7710571289062,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.07342735616745019,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05913765032733167,
-      "kl": 0.0567626953125,
+      "grad_norm": 0.062287259542847764,
+      "kl": 0.0159454345703125,
       "learning_rate": 4.797496794307889e-06,
-      "loss": -0.0007,
-      "num_tokens": 48749561.0,
+      "loss": 0.0149,
+      "num_tokens": 47733585.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2311,26 +2311,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1441.0,
-      "completions/max_terminated_length": 1441.0,
-      "completions/mean_length": 566.939453125,
-      "completions/mean_terminated_length": 565.4774780273438,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1470.0,
+      "completions/max_terminated_length": 1470.0,
+      "completions/mean_length": 566.072265625,
+      "completions/mean_terminated_length": 566.072265625,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.07432281173046787,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 3.166488626408786,
-      "kl": 0.0667724609375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006352211521854682,
+      "kl": 0.0157623291015625,
       "learning_rate": 4.791411170547545e-06,
-      "loss": 0.0085,
-      "num_tokens": 49367402.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 48350982.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 83
     },
     {
@@ -2339,26 +2339,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1595.0,
-      "completions/mean_length": 560.765625,
-      "completions/mean_terminated_length": 557.8551635742188,
-      "completions/min_length": 246.0,
-      "completions/min_terminated_length": 246.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1757.0,
+      "completions/max_terminated_length": 1757.0,
+      "completions/mean_length": 543.951171875,
+      "completions/mean_terminated_length": 543.951171875,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.07521826729348556,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08297988092224323,
-      "kl": 0.05584716796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007813740834546406,
+      "kl": 0.016326904296875,
       "learning_rate": 4.785239893263017e-06,
-      "loss": 0.0134,
-      "num_tokens": 49983250.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 48958221.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 84
     },
     {
@@ -2368,25 +2368,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1030.0,
-      "completions/max_terminated_length": 1030.0,
-      "completions/mean_length": 545.970703125,
-      "completions/mean_terminated_length": 545.970703125,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/max_length": 1216.0,
+      "completions/max_terminated_length": 1216.0,
+      "completions/mean_length": 550.677734375,
+      "completions/mean_terminated_length": 550.677734375,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.07611372285650325,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007356660460353614,
-      "kl": 0.05523681640625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.052051410170601047,
+      "kl": 0.016571044921875,
       "learning_rate": 4.778983221388742e-06,
-      "loss": 0.0006,
-      "num_tokens": 50557955.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0018,
+      "num_tokens": 49535336.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 85
     },
     {
@@ -2396,25 +2396,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1424.0,
-      "completions/max_terminated_length": 1424.0,
-      "completions/mean_length": 578.482421875,
-      "completions/mean_terminated_length": 578.482421875,
-      "completions/min_length": 216.0,
-      "completions/min_terminated_length": 216.0,
+      "completions/max_length": 1647.0,
+      "completions/max_terminated_length": 1647.0,
+      "completions/mean_length": 565.21484375,
+      "completions/mean_terminated_length": 565.21484375,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.07700917841952093,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007809110580486689,
-      "kl": 0.0545654296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.062482518099836674,
+      "kl": 0.0159912109375,
       "learning_rate": 4.77264141744214e-06,
-      "loss": 0.0005,
-      "num_tokens": 51175082.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0016,
+      "num_tokens": 50145670.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 86
     },
     {
@@ -2423,26 +2423,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1490.0,
-      "completions/max_terminated_length": 1490.0,
-      "completions/mean_length": 591.57421875,
-      "completions/mean_terminated_length": 591.57421875,
-      "completions/min_length": 259.0,
-      "completions/min_terminated_length": 259.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1536.0,
+      "completions/mean_length": 573.24609375,
+      "completions/mean_terminated_length": 570.3600463867188,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.07790463398253862,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04977783029837793,
-      "kl": 0.05291748046875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.1095124911093521,
+      "kl": 0.0146026611328125,
       "learning_rate": 4.766214747512603e-06,
-      "loss": 0.0013,
-      "num_tokens": 51787840.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0114,
+      "num_tokens": 50749044.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 87
     },
     {
@@ -2452,25 +2452,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1299.0,
-      "completions/max_terminated_length": 1299.0,
-      "completions/mean_length": 559.4375,
-      "completions/mean_terminated_length": 559.4375,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/max_length": 1443.0,
+      "completions/max_terminated_length": 1443.0,
+      "completions/mean_length": 550.958984375,
+      "completions/mean_terminated_length": 550.958984375,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.0788000895455563,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05494814305555876,
-      "kl": 0.05511474609375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006622879741401474,
+      "kl": 0.014495849609375,
       "learning_rate": 4.759703481250331e-06,
-      "loss": -0.0037,
-      "num_tokens": 52393072.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 51349935.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 88
     },
     {
@@ -2479,26 +2479,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1503.0,
-      "completions/max_terminated_length": 1503.0,
-      "completions/mean_length": 579.380859375,
-      "completions/mean_terminated_length": 578.4148559570312,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1532.0,
+      "completions/max_terminated_length": 1532.0,
+      "completions/mean_length": 546.904296875,
+      "completions/mean_terminated_length": 546.904296875,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.07969554510857399,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 160.94730744375263,
-      "kl": 30.91864013671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005845451074109807,
+      "kl": 0.0149993896484375,
       "learning_rate": 4.753107891855015e-06,
-      "loss": 0.3115,
-      "num_tokens": 53043635.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 51983870.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 89
     },
     {
@@ -2508,25 +2508,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1789.0,
-      "completions/max_terminated_length": 1789.0,
-      "completions/mean_length": 577.939453125,
-      "completions/mean_terminated_length": 577.939453125,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1887.0,
+      "completions/max_terminated_length": 1887.0,
+      "completions/mean_length": 548.8828125,
+      "completions/mean_terminated_length": 548.8828125,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.08059100067159167,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10489931391093958,
-      "kl": 0.0546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006245472310213368,
+      "kl": 0.01531982421875,
       "learning_rate": 4.746428256064375e-06,
-      "loss": 0.0041,
-      "num_tokens": 53658612.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 52583970.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 90
     },
     {
@@ -2536,19 +2536,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1258.0,
-      "completions/max_terminated_length": 1258.0,
-      "completions/mean_length": 549.6484375,
-      "completions/mean_terminated_length": 549.6484375,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/max_length": 1645.0,
+      "completions/max_terminated_length": 1645.0,
+      "completions/mean_length": 518.572265625,
+      "completions/mean_terminated_length": 518.572265625,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.08148645623460936,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008182419427830684,
-      "kl": 0.05682373046875,
+      "grad_norm": 0.006150422825146274,
+      "kl": 0.0150299072265625,
       "learning_rate": 4.7396648541425534e-06,
-      "loss": 0.0006,
-      "num_tokens": 54241008.0,
+      "loss": 0.0002,
+      "num_tokens": 53150455.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2563,26 +2563,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1601.0,
-      "completions/max_terminated_length": 1601.0,
-      "completions/mean_length": 556.23828125,
-      "completions/mean_terminated_length": 555.0195922851562,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1218.0,
+      "completions/max_terminated_length": 1218.0,
+      "completions/mean_length": 510.095703125,
+      "completions/mean_terminated_length": 510.095703125,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.08238191179762704,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05844731713658409,
-      "kl": 0.05755615234375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0077462527558477936,
+      "kl": 0.0150909423828125,
       "learning_rate": 4.732817969868348e-06,
-      "loss": 0.0006,
-      "num_tokens": 54844810.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 53730632.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 92
     },
     {
@@ -2592,19 +2592,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1856.0,
-      "completions/max_terminated_length": 1856.0,
-      "completions/mean_length": 608.26171875,
-      "completions/mean_terminated_length": 608.26171875,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/max_length": 1567.0,
+      "completions/max_terminated_length": 1567.0,
+      "completions/mean_length": 583.322265625,
+      "completions/mean_terminated_length": 583.322265625,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.08327736736064473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005945148181573863,
-      "kl": 0.05767822265625,
+      "grad_norm": 0.005995103203475892,
+      "kl": 0.013946533203125,
       "learning_rate": 4.7258878905233095e-06,
-      "loss": 0.0006,
-      "num_tokens": 55481952.0,
+      "loss": 0.0001,
+      "num_tokens": 54355005.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2619,26 +2619,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2026.0,
-      "completions/max_terminated_length": 2026.0,
-      "completions/mean_length": 610.75390625,
-      "completions/mean_terminated_length": 609.74560546875,
-      "completions/min_length": 242.0,
-      "completions/min_terminated_length": 242.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1284.0,
+      "completions/max_terminated_length": 1284.0,
+      "completions/mean_length": 562.65625,
+      "completions/mean_terminated_length": 562.65625,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.08417282292366242,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052266875089248634,
-      "kl": 0.0565185546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.008773903997987694,
+      "kl": 0.0150146484375,
       "learning_rate": 4.718874906879688e-06,
-      "loss": -0.0009,
-      "num_tokens": 56112434.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 54960861.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 94
     },
     {
@@ -2648,19 +2648,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1580.0,
-      "completions/max_terminated_length": 1580.0,
-      "completions/mean_length": 546.06640625,
-      "completions/mean_terminated_length": 546.06640625,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_length": 1428.0,
+      "completions/max_terminated_length": 1428.0,
+      "completions/mean_length": 510.4375,
+      "completions/mean_terminated_length": 510.4375,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.0850682784866801,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006438116887744368,
-      "kl": 0.0577392578125,
+      "grad_norm": 0.005650435077930954,
+      "kl": 0.014801025390625,
       "learning_rate": 4.711779313188231e-06,
-      "loss": 0.0006,
-      "num_tokens": 56657092.0,
+      "loss": 0.0001,
+      "num_tokens": 55487277.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2675,26 +2675,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1746.0,
-      "completions/max_terminated_length": 1746.0,
-      "completions/mean_length": 620.203125,
-      "completions/mean_terminated_length": 619.3052978515625,
-      "completions/min_length": 250.0,
-      "completions/min_terminated_length": 250.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1508.0,
+      "completions/max_terminated_length": 1508.0,
+      "completions/mean_length": 600.46875,
+      "completions/mean_terminated_length": 600.46875,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
       "epoch": 0.08596373404969779,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05996351922986414,
-      "kl": 0.05584716796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005507101143768758,
+      "kl": 0.012725830078125,
       "learning_rate": 4.70460140716584e-06,
-      "loss": -0.0007,
-      "num_tokens": 57295916.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 56115997.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 96
     },
     {
@@ -2703,26 +2703,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1329.0,
-      "completions/max_terminated_length": 1329.0,
-      "completions/mean_length": 535.82421875,
-      "completions/mean_terminated_length": 535.82421875,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1394.0,
+      "completions/mean_length": 501.939453125,
+      "completions/mean_terminated_length": 498.91387939453125,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.08685918961271547,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0065402778185692,
-      "kl": 0.05560302734375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07000735578379153,
+      "kl": 0.0143585205078125,
       "learning_rate": 4.697341489983076e-06,
-      "loss": 0.0006,
-      "num_tokens": 57849218.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0115,
+      "num_tokens": 56651950.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 97
     },
     {
@@ -2731,26 +2731,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1644.0,
-      "completions/mean_length": 605.234375,
-      "completions/mean_terminated_length": 602.4109497070312,
-      "completions/min_length": 224.0,
-      "completions/min_terminated_length": 224.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1373.0,
+      "completions/max_terminated_length": 1373.0,
+      "completions/mean_length": 582.009765625,
+      "completions/mean_terminated_length": 582.009765625,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.08775464517573316,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.055220508680084254,
-      "kl": 0.05645751953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005593438153271221,
+      "kl": 0.0135498046875,
       "learning_rate": 4.6899998662515215e-06,
-      "loss": 0.0009,
-      "num_tokens": 58475690.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 57266531.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 98
     },
     {
@@ -2759,20 +2759,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1649.0,
-      "completions/max_terminated_length": 1649.0,
-      "completions/mean_length": 539.849609375,
-      "completions/mean_terminated_length": 538.8356323242188,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1399.0,
+      "completions/max_terminated_length": 1399.0,
+      "completions/mean_length": 511.146484375,
+      "completions/mean_terminated_length": 511.146484375,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.08865010073875083,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 2.722021588876746,
-      "kl": 0.2435302734375,
+      "grad_norm": 0.005254315257483802,
+      "kl": 0.014007568359375,
       "learning_rate": 4.682576844011007e-06,
-      "loss": 0.0024,
-      "num_tokens": 59046989.0,
+      "loss": 0.0001,
+      "num_tokens": 57823134.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2788,19 +2788,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1446.0,
-      "completions/max_terminated_length": 1446.0,
-      "completions/mean_length": 559.466796875,
-      "completions/mean_terminated_length": 559.466796875,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_length": 1946.0,
+      "completions/max_terminated_length": 1946.0,
+      "completions/mean_length": 561.2421875,
+      "completions/mean_terminated_length": 561.2421875,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.08954555630176853,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006225919234576967,
-      "kl": 0.0565185546875,
+      "grad_norm": 0.0050205373550895075,
+      "kl": 0.0132904052734375,
       "learning_rate": 4.675072734716678e-06,
-      "loss": 0.0006,
-      "num_tokens": 59617372.0,
+      "loss": 0.0001,
+      "num_tokens": 58394426.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2815,20 +2815,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1832.0,
-      "completions/max_terminated_length": 1832.0,
-      "completions/mean_length": 623.28515625,
-      "completions/mean_terminated_length": 623.28515625,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1703.0,
+      "completions/mean_length": 631.42578125,
+      "completions/mean_terminated_length": 628.6536254882812,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
       "epoch": 0.09044101186478622,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07716472081979199,
-      "kl": 0.0572509765625,
+      "grad_norm": 0.0760184868440684,
+      "kl": 0.01263427734375,
       "learning_rate": 4.667487853225931e-06,
-      "loss": 0.0003,
-      "num_tokens": 60280782.0,
+      "loss": 0.0097,
+      "num_tokens": 59062004.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2843,20 +2843,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1512.0,
-      "completions/mean_length": 593.6640625,
-      "completions/mean_terminated_length": 590.8179931640625,
-      "completions/min_length": 128.0,
-      "completions/min_terminated_length": 128.0,
+      "completions/max_terminated_length": 1712.0,
+      "completions/mean_length": 610.859375,
+      "completions/mean_terminated_length": 607.2745361328125,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.0913364674278039,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.054090751280617605,
-      "kl": 0.05364990234375,
+      "grad_norm": 0.05564333998730713,
+      "kl": 0.02105712890625,
       "learning_rate": 4.659822517785203e-06,
-      "loss": 0.0106,
-      "num_tokens": 60872962.0,
+      "loss": 0.0092,
+      "num_tokens": 59662988.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -2869,28 +2869,28 @@
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1912.0,
-      "completions/max_terminated_length": 1912.0,
-      "completions/mean_length": 577.80859375,
-      "completions/mean_terminated_length": 577.80859375,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2037.0,
+      "completions/mean_length": 611.19921875,
+      "completions/mean_terminated_length": 608.387451171875,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.09223192299082159,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005765836397552098,
-      "kl": 0.055419921875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10935821324700373,
+      "kl": 0.013153076171875,
       "learning_rate": 4.6520770500166165e-06,
-      "loss": 0.0006,
-      "num_tokens": 61482432.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0109,
+      "num_tokens": 60289554.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 103
     },
     {
@@ -2900,25 +2900,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1909.0,
-      "completions/max_terminated_length": 1909.0,
-      "completions/mean_length": 625.951171875,
-      "completions/mean_terminated_length": 625.951171875,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/max_length": 1446.0,
+      "completions/max_terminated_length": 1446.0,
+      "completions/mean_length": 640.951171875,
+      "completions/mean_terminated_length": 640.951171875,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
       "epoch": 0.09312737855383926,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006272367997240394,
-      "kl": 0.0543212890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.060757676548859295,
+      "kl": 0.0118408203125,
       "learning_rate": 4.644251774904487e-06,
       "loss": 0.0005,
-      "num_tokens": 62149639.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "num_tokens": 60964441.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 104
     },
     {
@@ -2927,20 +2927,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1764.0,
-      "completions/max_terminated_length": 1764.0,
-      "completions/mean_length": 594.787109375,
-      "completions/mean_terminated_length": 594.0704345703125,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1451.0,
+      "completions/max_terminated_length": 1451.0,
+      "completions/mean_length": 592.787109375,
+      "completions/mean_terminated_length": 592.787109375,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.09402283411685695,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007591245217264678,
-      "kl": 0.0574951171875,
+      "grad_norm": 0.005244427082623526,
+      "kl": 0.011688232421875,
       "learning_rate": 4.636347020781684e-06,
-      "loss": 0.0006,
-      "num_tokens": 62741770.0,
+      "loss": 0.0001,
+      "num_tokens": 61555548.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -2957,24 +2957,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1339.0,
-      "completions/mean_length": 554.9921875,
-      "completions/mean_terminated_length": 552.0704345703125,
-      "completions/min_length": 132.0,
-      "completions/min_terminated_length": 132.0,
+      "completions/max_terminated_length": 1920.0,
+      "completions/mean_length": 576.23046875,
+      "completions/mean_terminated_length": 573.3502807617188,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.09491828967987463,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09890693315638917,
-      "kl": 0.0543212890625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08855757430577488,
+      "kl": 0.0129241943359375,
       "learning_rate": 4.6283631193158605e-06,
-      "loss": 0.0161,
-      "num_tokens": 63304470.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0125,
+      "num_tokens": 62129122.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 106
     },
     {
@@ -2984,25 +2984,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1571.0,
-      "completions/mean_length": 574.91015625,
-      "completions/mean_terminated_length": 572.0274047851562,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/max_length": 1757.0,
+      "completions/max_terminated_length": 1757.0,
+      "completions/mean_length": 587.9765625,
+      "completions/mean_terminated_length": 585.6888427734375,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.09581374524289232,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09008482271620612,
-      "kl": 0.05413818359375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.018552764774495144,
+      "kl": 0.0251617431640625,
       "learning_rate": 4.620300405495532e-06,
-      "loss": 0.0096,
-      "num_tokens": 63888856.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0003,
+      "num_tokens": 62720198.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 107
     },
     {
@@ -3012,19 +3012,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1465.0,
-      "completions/max_terminated_length": 1465.0,
-      "completions/mean_length": 572.958984375,
-      "completions/mean_terminated_length": 572.958984375,
+      "completions/max_length": 1823.0,
+      "completions/max_terminated_length": 1823.0,
+      "completions/mean_length": 590.748046875,
+      "completions/mean_terminated_length": 590.748046875,
       "completions/min_length": 176.0,
       "completions/min_terminated_length": 176.0,
       "epoch": 0.09670920080591,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009429951785588081,
-      "kl": 0.0540771484375,
+      "grad_norm": 0.011091417692320823,
+      "kl": 0.01226806640625,
       "learning_rate": 4.612159217616022e-06,
-      "loss": 0.0005,
-      "num_tokens": 64508131.0,
+      "loss": 0.0001,
+      "num_tokens": 63348581.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3039,26 +3039,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1404.0,
-      "completions/max_terminated_length": 1404.0,
-      "completions/mean_length": 548.35546875,
-      "completions/mean_terminated_length": 547.6868896484375,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1711.0,
+      "completions/max_terminated_length": 1711.0,
+      "completions/mean_length": 558.81640625,
+      "completions/mean_terminated_length": 558.81640625,
+      "completions/min_length": 249.0,
+      "completions/min_terminated_length": 249.0,
       "epoch": 0.09760465636892769,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09621891008446948,
-      "kl": 0.057861328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009595552547661724,
+      "kl": 0.0125274658203125,
       "learning_rate": 4.603939897265268e-06,
-      "loss": 0.0,
-      "num_tokens": 65096793.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 63942599.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 109
     },
     {
@@ -3067,20 +3067,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1719.0,
-      "completions/max_terminated_length": 1719.0,
-      "completions/mean_length": 533.857421875,
-      "completions/mean_terminated_length": 532.8375854492188,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1506.0,
+      "completions/max_terminated_length": 1506.0,
+      "completions/mean_length": 533.255859375,
+      "completions/mean_terminated_length": 533.255859375,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.09850011193194538,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010383900504147035,
-      "kl": 0.065673828125,
+      "grad_norm": 0.004424584359067543,
+      "kl": 0.0120849609375,
       "learning_rate": 4.595642789309492e-06,
-      "loss": 0.0007,
-      "num_tokens": 65662384.0,
+      "loss": 0.0001,
+      "num_tokens": 64507882.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3095,20 +3095,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1655.0,
-      "completions/mean_length": 553.45703125,
-      "completions/mean_terminated_length": 550.5322875976562,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1444.0,
+      "completions/max_terminated_length": 1444.0,
+      "completions/mean_length": 543.14453125,
+      "completions/mean_terminated_length": 543.14453125,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.09939556749496306,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006695037361617289,
-      "kl": 0.0556640625,
+      "grad_norm": 0.005542511266984594,
+      "kl": 0.0128021240234375,
       "learning_rate": 4.587268241878724e-06,
-      "loss": 0.0006,
-      "num_tokens": 66212682.0,
+      "loss": 0.0001,
+      "num_tokens": 65052900.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3123,26 +3123,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1947.0,
-      "completions/max_terminated_length": 1947.0,
-      "completions/mean_length": 563.484375,
-      "completions/mean_terminated_length": 563.484375,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1646.0,
+      "completions/mean_length": 572.822265625,
+      "completions/mean_terminated_length": 566.6071166992188,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.10029102305798075,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006346704211210989,
-      "kl": 0.05413818359375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10304996756847001,
+      "kl": 0.01776123046875,
       "learning_rate": 4.578816606352205e-06,
-      "loss": 0.0005,
-      "num_tokens": 66819442.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0254,
+      "num_tokens": 65664441.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 112
     },
     {
@@ -3151,26 +3151,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1462.0,
-      "completions/max_terminated_length": 1462.0,
-      "completions/mean_length": 587.166015625,
-      "completions/mean_terminated_length": 587.166015625,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1262.0,
+      "completions/max_terminated_length": 1262.0,
+      "completions/mean_length": 570.673828125,
+      "completions/mean_terminated_length": 569.7299194335938,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.10118647862099843,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.043742302560142485,
-      "kl": 0.05584716796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009706307742035009,
+      "kl": 0.0243988037109375,
       "learning_rate": 4.570288237343632e-06,
-      "loss": -0.0033,
-      "num_tokens": 67420375.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 66256930.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 113
     },
     {
@@ -3179,26 +3179,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1322.0,
-      "completions/max_terminated_length": 1322.0,
-      "completions/mean_length": 621.79296875,
-      "completions/mean_terminated_length": 621.79296875,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1857.0,
+      "completions/max_terminated_length": 1857.0,
+      "completions/mean_length": 613.189453125,
+      "completions/mean_terminated_length": 612.127197265625,
+      "completions/min_length": 225.0,
+      "completions/min_terminated_length": 225.0,
       "epoch": 0.10208193418401612,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0653506341979416,
-      "kl": 0.05206298828125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00881535183664981,
+      "kl": 0.02374267578125,
       "learning_rate": 4.561683492686289e-06,
-      "loss": 0.0038,
-      "num_tokens": 68049405.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0012597277527675033,
+      "loss": 0.0002,
+      "num_tokens": 66881555.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 114
     },
     {
@@ -3208,25 +3208,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1365.0,
-      "completions/max_terminated_length": 1365.0,
-      "completions/mean_length": 558.744140625,
-      "completions/mean_terminated_length": 558.744140625,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/max_length": 1393.0,
+      "completions/max_terminated_length": 1393.0,
+      "completions/mean_length": 577.60546875,
+      "completions/mean_terminated_length": 577.60546875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.1029773897470338,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08608131464673477,
-      "kl": 0.05255126953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004670068300340548,
+      "kl": 0.0115203857421875,
       "learning_rate": 4.5530027334180285e-06,
-      "loss": -0.0009,
-      "num_tokens": 68624954.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 67466761.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 115
     },
     {
@@ -3235,20 +3235,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1205.0,
-      "completions/max_terminated_length": 1205.0,
-      "completions/mean_length": 557.375,
-      "completions/mean_terminated_length": 556.373779296875,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1225.0,
+      "completions/max_terminated_length": 1225.0,
+      "completions/mean_length": 559.638671875,
+      "completions/mean_terminated_length": 559.638671875,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.10387284531005149,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.051558838637773656,
-      "kl": 0.0572509765625,
+      "grad_norm": 0.09101415243660085,
+      "kl": 0.0117034912109375,
       "learning_rate": 4.544246323766122e-06,
-      "loss": -0.0009,
-      "num_tokens": 69221738.0,
+      "loss": 0.0026,
+      "num_tokens": 68064704.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -3263,20 +3263,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1382.0,
-      "completions/mean_length": 599.0,
-      "completions/mean_terminated_length": 593.3176879882812,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1907.0,
+      "completions/max_terminated_length": 1907.0,
+      "completions/mean_length": 582.666015625,
+      "completions/mean_terminated_length": 582.666015625,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.10476830087306917,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011260686367210828,
-      "kl": 0.06072998046875,
+      "grad_norm": 0.005123090555119176,
+      "kl": 0.0115203857421875,
       "learning_rate": 4.535414631131983e-06,
-      "loss": 0.0006,
-      "num_tokens": 69816666.0,
+      "loss": 0.0001,
+      "num_tokens": 68651269.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3291,26 +3291,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1427.0,
-      "completions/max_terminated_length": 1427.0,
-      "completions/mean_length": 557.9140625,
-      "completions/mean_terminated_length": 557.9140625,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1942.0,
+      "completions/mean_length": 560.41015625,
+      "completions/mean_terminated_length": 557.4990234375,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.10566375643608686,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007377506198813074,
-      "kl": 0.05279541015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08107964407237964,
+      "kl": 0.01141357421875,
       "learning_rate": 4.526508026075746e-06,
-      "loss": 0.0005,
-      "num_tokens": 70407966.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0113,
+      "num_tokens": 69243847.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 118
     },
     {
@@ -3320,25 +3320,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1868.0,
-      "completions/mean_length": 604.025390625,
-      "completions/mean_terminated_length": 601.1995849609375,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/max_length": 1367.0,
+      "completions/max_terminated_length": 1367.0,
+      "completions/mean_length": 581.55078125,
+      "completions/mean_terminated_length": 580.6907958984375,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.10655921199910455,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05981685989580845,
-      "kl": 0.04998779296875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.08267105634677789,
+      "kl": 0.02154541015625,
       "learning_rate": 4.517526882300721e-06,
-      "loss": 0.0092,
-      "num_tokens": 71031707.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 69856081.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 119
     },
     {
@@ -3347,26 +3347,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1818.0,
-      "completions/mean_length": 584.580078125,
-      "completions/mean_terminated_length": 578.8411865234375,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1711.0,
+      "completions/max_terminated_length": 1711.0,
+      "completions/mean_length": 568.03125,
+      "completions/mean_terminated_length": 568.03125,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.10745466756212223,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.08493217637882901,
-      "kl": 0.0518798828125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.010809362339601222,
+      "kl": 0.0114898681640625,
       "learning_rate": 4.508471576637713e-06,
-      "loss": 0.0158,
-      "num_tokens": 71654500.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.0001,
+      "num_tokens": 70470401.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 120
     },
     {
@@ -3375,26 +3375,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1486.0,
-      "completions/max_terminated_length": 1486.0,
-      "completions/mean_length": 583.146484375,
-      "completions/mean_terminated_length": 583.146484375,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1704.0,
+      "completions/mean_length": 572.75390625,
+      "completions/mean_terminated_length": 569.866943359375,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
       "epoch": 0.10835012312513992,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009711429854862424,
-      "kl": 0.05181884765625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09457771857394989,
+      "kl": 0.0112457275390625,
       "learning_rate": 4.499342489029211e-06,
-      "loss": 0.0005,
-      "num_tokens": 72254767.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0123,
+      "num_tokens": 71065347.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 121
     },
     {
@@ -3403,20 +3403,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1591.0,
-      "completions/max_terminated_length": 1591.0,
-      "completions/mean_length": 591.98046875,
-      "completions/mean_terminated_length": 590.6927490234375,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1688.0,
+      "completions/max_terminated_length": 1688.0,
+      "completions/mean_length": 578.771484375,
+      "completions/mean_terminated_length": 578.771484375,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.1092455786881576,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006688758768353923,
-      "kl": 0.05718994140625,
+      "grad_norm": 0.009517196854567733,
+      "kl": 0.0113067626953125,
       "learning_rate": 4.490140002513449e-06,
-      "loss": 0.0006,
-      "num_tokens": 72871125.0,
+      "loss": 0.0001,
+      "num_tokens": 71674942.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3431,26 +3431,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 569.203125,
-      "completions/mean_terminated_length": 566.3092041015625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1412.0,
+      "completions/max_terminated_length": 1412.0,
+      "completions/mean_length": 555.53515625,
+      "completions/mean_terminated_length": 555.53515625,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.11014103425117529,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05402023980897781,
-      "kl": 0.05047607421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009205657515159494,
+      "kl": 0.011474609375,
       "learning_rate": 4.48086450320833e-06,
-      "loss": 0.0152,
-      "num_tokens": 73458877.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 72255696.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 123
     },
     {
@@ -3459,26 +3459,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1306.0,
-      "completions/max_terminated_length": 1306.0,
-      "completions/mean_length": 571.666015625,
-      "completions/mean_terminated_length": 569.3275146484375,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1758.0,
+      "completions/max_terminated_length": 1758.0,
+      "completions/mean_length": 553.37890625,
+      "completions/mean_terminated_length": 553.37890625,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
       "epoch": 0.11103648981419297,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.055716219457509576,
-      "kl": 0.06829833984375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00819542614576692,
+      "kl": 0.010772705078125,
       "learning_rate": 4.4715163802952266e-06,
-      "loss": -0.001,
-      "num_tokens": 74054402.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 72841858.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 124
     },
     {
@@ -3487,26 +3487,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1784.0,
-      "completions/max_terminated_length": 1784.0,
-      "completions/mean_length": 621.005859375,
-      "completions/mean_terminated_length": 621.005859375,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1450.0,
+      "completions/max_terminated_length": 1450.0,
+      "completions/mean_length": 618.697265625,
+      "completions/mean_terminated_length": 617.0704345703125,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.11193194537721066,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.059224756742812495,
-      "kl": 0.05059814453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.013122697785598695,
+      "kl": 0.0197296142578125,
       "learning_rate": 4.462096026002655e-06,
-      "loss": -0.0017,
-      "num_tokens": 74660613.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 73446887.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 125
     },
     {
@@ -3516,19 +3516,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1711.0,
-      "completions/max_terminated_length": 1711.0,
-      "completions/mean_length": 581.55078125,
-      "completions/mean_terminated_length": 581.55078125,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/max_length": 1454.0,
+      "completions/max_terminated_length": 1454.0,
+      "completions/mean_length": 570.470703125,
+      "completions/mean_terminated_length": 570.470703125,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
       "epoch": 0.11282740094022835,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006152298759927084,
-      "kl": 0.0509033203125,
+      "grad_norm": 0.0066434644747071486,
+      "kl": 0.010589599609375,
       "learning_rate": 4.4526038355898144e-06,
-      "loss": 0.0005,
-      "num_tokens": 75271119.0,
+      "loss": 0.0001,
+      "num_tokens": 74051720.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3543,26 +3543,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1253.0,
-      "completions/max_terminated_length": 1253.0,
-      "completions/mean_length": 575.66015625,
-      "completions/mean_terminated_length": 575.66015625,
-      "completions/min_length": 223.0,
-      "completions/min_terminated_length": 223.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1642.0,
+      "completions/max_terminated_length": 1642.0,
+      "completions/mean_length": 564.19921875,
+      "completions/mean_terminated_length": 562.0900268554688,
+      "completions/min_length": 222.0,
+      "completions/min_terminated_length": 222.0,
       "epoch": 0.11372285650324603,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.054134619906622765,
-      "kl": 0.0501708984375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.022334212444641555,
+      "kl": 0.0200653076171875,
       "learning_rate": 4.4430402073300035e-06,
-      "loss": 0.0006,
-      "num_tokens": 75877089.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 74651822.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 127
     },
     {
@@ -3571,20 +3571,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1672.0,
-      "completions/max_terminated_length": 1672.0,
-      "completions/mean_length": 579.533203125,
-      "completions/mean_terminated_length": 579.533203125,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1330.0,
+      "completions/max_terminated_length": 1330.0,
+      "completions/mean_length": 573.5546875,
+      "completions/mean_terminated_length": 572.8140869140625,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.11461831206626372,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005668747851169466,
-      "kl": 0.0504150390625,
+      "grad_norm": 0.005480335975480727,
+      "kl": 0.0178680419921875,
       "learning_rate": 4.433405542493909e-06,
-      "loss": 0.0005,
-      "num_tokens": 76468402.0,
+      "loss": 0.0002,
+      "num_tokens": 75240074.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3599,26 +3599,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1584.0,
-      "completions/max_terminated_length": 1584.0,
-      "completions/mean_length": 604.94921875,
-      "completions/mean_terminated_length": 603.1017456054688,
-      "completions/min_length": 229.0,
-      "completions/min_terminated_length": 229.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1416.0,
+      "completions/max_terminated_length": 1416.0,
+      "completions/mean_length": 614.1953125,
+      "completions/mean_terminated_length": 614.1953125,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.1155137676292814,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 3.323619983813528,
-      "kl": 0.05792236328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006019467357919121,
+      "kl": 0.0113525390625,
       "learning_rate": 4.4237002453327734e-06,
-      "loss": 0.0121,
-      "num_tokens": 77084744.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 75861150.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 129
     },
     {
@@ -3627,20 +3627,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1385.0,
-      "completions/max_terminated_length": 1385.0,
-      "completions/mean_length": 554.462890625,
-      "completions/mean_terminated_length": 554.462890625,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 1408.0,
+      "completions/max_terminated_length": 1408.0,
+      "completions/mean_length": 561.0,
+      "completions/mean_terminated_length": 559.1353149414062,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.11640922319229909,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005401927989012176,
-      "kl": 0.0499267578125,
+      "grad_norm": 0.016407802186772742,
+      "kl": 0.0272216796875,
       "learning_rate": 4.4139247230614245e-06,
-      "loss": 0.0005,
-      "num_tokens": 77677189.0,
+      "loss": 0.0003,
+      "num_tokens": 76456942.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3656,19 +3656,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1505.0,
-      "completions/max_terminated_length": 1505.0,
-      "completions/mean_length": 559.4375,
-      "completions/mean_terminated_length": 559.4375,
-      "completions/min_length": 232.0,
-      "completions/min_terminated_length": 232.0,
+      "completions/max_length": 1149.0,
+      "completions/max_terminated_length": 1149.0,
+      "completions/mean_length": 547.41015625,
+      "completions/mean_terminated_length": 547.41015625,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.11730467875531676,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0061677770684820275,
-      "kl": 0.05316162109375,
+      "grad_norm": 0.004386721645387016,
+      "kl": 0.01092529296875,
       "learning_rate": 4.404079385841201e-06,
-      "loss": 0.0005,
-      "num_tokens": 78251269.0,
+      "loss": 0.0001,
+      "num_tokens": 77024864.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3684,19 +3684,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1731.0,
-      "completions/max_terminated_length": 1731.0,
-      "completions/mean_length": 591.04296875,
-      "completions/mean_terminated_length": 591.04296875,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/max_length": 1368.0,
+      "completions/max_terminated_length": 1368.0,
+      "completions/mean_length": 595.248046875,
+      "completions/mean_terminated_length": 595.248046875,
+      "completions/min_length": 132.0,
+      "completions/min_terminated_length": 132.0,
       "epoch": 0.11820013431833445,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005599681736981137,
-      "kl": 0.052978515625,
+      "grad_norm": 0.0040065240940903215,
+      "kl": 0.0105133056640625,
       "learning_rate": 4.394164646762734e-06,
-      "loss": 0.0005,
-      "num_tokens": 78847275.0,
+      "loss": 0.0001,
+      "num_tokens": 77623023.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3711,20 +3711,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1358.0,
-      "completions/max_terminated_length": 1358.0,
-      "completions/mean_length": 576.123046875,
-      "completions/mean_terminated_length": 576.123046875,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1660.0,
+      "completions/max_terminated_length": 1660.0,
+      "completions/mean_length": 584.369140625,
+      "completions/mean_terminated_length": 582.9862670898438,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.11909558988135213,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005987513884752594,
-      "kl": 0.0504150390625,
+      "grad_norm": 0.01841556230332692,
+      "kl": 0.0206756591796875,
       "learning_rate": 4.384180921828618e-06,
-      "loss": 0.0005,
-      "num_tokens": 79500986.0,
+      "loss": 0.0002,
+      "num_tokens": 78280956.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3740,25 +3740,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1332.0,
-      "completions/max_terminated_length": 1332.0,
-      "completions/mean_length": 580.025390625,
-      "completions/mean_terminated_length": 578.5538330078125,
-      "completions/min_length": 203.0,
-      "completions/min_terminated_length": 203.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1502.0,
+      "completions/mean_length": 593.40625,
+      "completions/mean_terminated_length": 590.5596923828125,
+      "completions/min_length": 220.0,
+      "completions/min_terminated_length": 220.0,
       "epoch": 0.11999104544436982,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0068151723856892735,
-      "kl": 0.05657958984375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.07155830102688933,
+      "kl": 0.0109710693359375,
       "learning_rate": 4.374128629935955e-06,
-      "loss": 0.0006,
-      "num_tokens": 80102167.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0096,
+      "num_tokens": 78888988.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 134
     },
     {
@@ -3767,20 +3767,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1683.0,
-      "completions/max_terminated_length": 1683.0,
-      "completions/mean_length": 578.212890625,
-      "completions/mean_terminated_length": 576.4461669921875,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1534.0,
+      "completions/max_terminated_length": 1534.0,
+      "completions/mean_length": 585.830078125,
+      "completions/mean_terminated_length": 585.830078125,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.12088650100738751,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.03848281665656771,
-      "kl": 0.0733642578125,
+      "grad_norm": 0.004840567540768247,
+      "kl": 0.0105743408203125,
       "learning_rate": 4.364008192858781e-06,
-      "loss": 0.0007,
-      "num_tokens": 80710372.0,
+      "loss": 0.0001,
+      "num_tokens": 79501093.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3795,26 +3795,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1568.0,
-      "completions/max_terminated_length": 1568.0,
-      "completions/mean_length": 587.712890625,
-      "completions/mean_terminated_length": 585.7944946289062,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1482.0,
+      "completions/max_terminated_length": 1482.0,
+      "completions/mean_length": 592.638671875,
+      "completions/mean_terminated_length": 592.638671875,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.12178195657040519,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 1.1813951961066245,
-      "kl": 0.0977783203125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00446197838012533,
+      "kl": 0.0106964111328125,
       "learning_rate": 4.353820035230366e-06,
-      "loss": 0.0095,
-      "num_tokens": 81309089.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0001,
+      "num_tokens": 80102332.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 136
     },
     {
@@ -3824,25 +3824,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1525.0,
-      "completions/mean_length": 529.837890625,
-      "completions/mean_terminated_length": 526.866943359375,
-      "completions/min_length": 152.0,
-      "completions/min_terminated_length": 152.0,
+      "completions/max_length": 1632.0,
+      "completions/max_terminated_length": 1632.0,
+      "completions/mean_length": 536.201171875,
+      "completions/mean_terminated_length": 534.630126953125,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.12267741213342288,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10318370576581831,
-      "kl": 0.05096435546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.014022538127291095,
+      "kl": 0.0203704833984375,
       "learning_rate": 4.3435645845254e-06,
-      "loss": 0.0084,
-      "num_tokens": 81878318.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 80674819.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 137
     },
     {
@@ -3852,19 +3852,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1481.0,
-      "completions/max_terminated_length": 1481.0,
-      "completions/mean_length": 601.44140625,
-      "completions/mean_terminated_length": 601.44140625,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/max_length": 1379.0,
+      "completions/max_terminated_length": 1379.0,
+      "completions/mean_length": 611.9296875,
+      "completions/mean_terminated_length": 611.9296875,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.12357286769644056,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005667719990219518,
-      "kl": 0.05023193359375,
+      "grad_norm": 0.004263453316826543,
+      "kl": 0.010223388671875,
       "learning_rate": 4.333242271042054e-06,
-      "loss": 0.0005,
-      "num_tokens": 82537568.0,
+      "loss": 0.0001,
+      "num_tokens": 81339439.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3880,25 +3880,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1317.0,
-      "completions/max_terminated_length": 1317.0,
-      "completions/mean_length": 556.82421875,
-      "completions/mean_terminated_length": 556.82421875,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/max_length": 1569.0,
+      "completions/max_terminated_length": 1569.0,
+      "completions/mean_length": 565.08203125,
+      "completions/mean_terminated_length": 565.08203125,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.12446832325945825,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006808014727799348,
-      "kl": 0.05157470703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05663482825888845,
+      "kl": 0.009979248046875,
       "learning_rate": 4.32285352788393e-06,
-      "loss": 0.0005,
-      "num_tokens": 83142310.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0015,
+      "num_tokens": 81948409.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 139
     },
     {
@@ -3908,19 +3908,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1133.0,
-      "completions/max_terminated_length": 1133.0,
-      "completions/mean_length": 543.28125,
-      "completions/mean_terminated_length": 543.28125,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/max_length": 1278.0,
+      "completions/max_terminated_length": 1278.0,
+      "completions/mean_length": 557.494140625,
+      "completions/mean_terminated_length": 557.494140625,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.12536377882247593,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006999711096947045,
-      "kl": 0.05181884765625,
+      "grad_norm": 0.0032894992448206737,
+      "kl": 0.0096282958984375,
       "learning_rate": 4.312398790941882e-06,
-      "loss": 0.0005,
-      "num_tokens": 83735542.0,
+      "loss": 0.0001,
+      "num_tokens": 82548918.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3935,26 +3935,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1233.0,
-      "completions/mean_length": 519.869140625,
-      "completions/mean_terminated_length": 513.8765258789062,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1240.0,
+      "completions/max_terminated_length": 1240.0,
+      "completions/mean_length": 507.896484375,
+      "completions/mean_terminated_length": 507.896484375,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.1262592343854936,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.12202530599091238,
-      "kl": 0.0498046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0036947275566920268,
+      "kl": 0.010162353515625,
       "learning_rate": 4.301878498875735e-06,
-      "loss": 0.0316,
-      "num_tokens": 84288339.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0001,
+      "num_tokens": 83095585.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 141
     },
     {
@@ -3964,19 +3964,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1405.0,
-      "completions/max_terminated_length": 1405.0,
-      "completions/mean_length": 517.1953125,
-      "completions/mean_terminated_length": 517.1953125,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/max_length": 1426.0,
+      "completions/max_terminated_length": 1426.0,
+      "completions/mean_length": 519.384765625,
+      "completions/mean_terminated_length": 519.384765625,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.1271546899485113,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0060097815761899044,
-      "kl": 0.051513671875,
+      "grad_norm": 0.006416399119562177,
+      "kl": 0.0099639892578125,
       "learning_rate": 4.291293093095873e-06,
-      "loss": 0.0005,
-      "num_tokens": 84836807.0,
+      "loss": 0.0001,
+      "num_tokens": 83645174.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -3991,20 +3991,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1405.0,
-      "completions/max_terminated_length": 1405.0,
-      "completions/mean_length": 567.349609375,
-      "completions/mean_terminated_length": 567.349609375,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1506.0,
+      "completions/max_terminated_length": 1506.0,
+      "completions/mean_length": 593.76953125,
+      "completions/mean_terminated_length": 593.0410766601562,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.128050145511529,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0070273058865257835,
-      "kl": 0.05145263671875,
+      "grad_norm": 0.007294106241650529,
+      "kl": 0.015716552734375,
       "learning_rate": 4.280643017744723e-06,
-      "loss": 0.0005,
-      "num_tokens": 85470554.0,
+      "loss": 0.0002,
+      "num_tokens": 84292448.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4020,19 +4020,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1409.0,
-      "completions/max_terminated_length": 1409.0,
-      "completions/mean_length": 596.11328125,
-      "completions/mean_terminated_length": 596.11328125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_length": 1758.0,
+      "completions/max_terminated_length": 1758.0,
+      "completions/mean_length": 615.357421875,
+      "completions/mean_terminated_length": 615.357421875,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.12894560107454667,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0067549331197202115,
-      "kl": 0.05047607421875,
+      "grad_norm": 0.006310239000280281,
+      "kl": 0.0094757080078125,
       "learning_rate": 4.269928719678117e-06,
-      "loss": 0.0005,
-      "num_tokens": 86082228.0,
+      "loss": 0.0001,
+      "num_tokens": 84913975.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4048,19 +4048,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1662.0,
-      "completions/max_terminated_length": 1662.0,
-      "completions/mean_length": 539.474609375,
-      "completions/mean_terminated_length": 539.474609375,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/max_length": 1225.0,
+      "completions/max_terminated_length": 1225.0,
+      "completions/mean_length": 541.259765625,
+      "completions/mean_terminated_length": 541.259765625,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.12984105663756437,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011441613334834384,
-      "kl": 0.05242919921875,
+      "grad_norm": 0.0076739424344186005,
+      "kl": 0.009979248046875,
       "learning_rate": 4.2591506484465426e-06,
-      "loss": 0.0005,
-      "num_tokens": 86662071.0,
+      "loss": 0.0001,
+      "num_tokens": 85494732.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4077,24 +4077,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1496.0,
-      "completions/mean_length": 561.123046875,
-      "completions/mean_terminated_length": 555.2921752929688,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/max_terminated_length": 1797.0,
+      "completions/mean_length": 564.115234375,
+      "completions/mean_terminated_length": 559.4392700195312,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.13073651220058205,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0589455460705877,
-      "kl": 0.052734375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 30.466730837647507,
+      "kl": 0.0369110107421875,
       "learning_rate": 4.248309256276283e-06,
-      "loss": 0.0137,
-      "num_tokens": 87251878.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0211,
+      "num_tokens": 86086071.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 146
     },
     {
@@ -4104,19 +4104,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1170.0,
-      "completions/max_terminated_length": 1170.0,
-      "completions/mean_length": 520.42578125,
-      "completions/mean_terminated_length": 520.42578125,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/max_length": 1374.0,
+      "completions/max_terminated_length": 1374.0,
+      "completions/mean_length": 529.3046875,
+      "completions/mean_terminated_length": 529.3046875,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.13163196776359973,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.012190047453444887,
-      "kl": 0.0516357421875,
+      "grad_norm": 0.00698639674310776,
+      "kl": 0.00970458984375,
       "learning_rate": 4.23740499805044e-06,
-      "loss": 0.0005,
-      "num_tokens": 87787696.0,
+      "loss": 0.0001,
+      "num_tokens": 86626435.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4132,19 +4132,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1788.0,
-      "completions/max_terminated_length": 1788.0,
-      "completions/mean_length": 520.943359375,
-      "completions/mean_terminated_length": 520.943359375,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/max_length": 1790.0,
+      "completions/max_terminated_length": 1790.0,
+      "completions/mean_length": 532.439453125,
+      "completions/mean_terminated_length": 532.439453125,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.1325274233266174,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008192901660929772,
-      "kl": 0.05194091796875,
+      "grad_norm": 0.004340388783933389,
+      "kl": 0.009613037109375,
       "learning_rate": 4.22643833128985e-06,
-      "loss": 0.0005,
-      "num_tokens": 88361491.0,
+      "loss": 0.0001,
+      "num_tokens": 87206116.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4160,19 +4160,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1529.0,
-      "completions/max_terminated_length": 1529.0,
-      "completions/mean_length": 561.27734375,
-      "completions/mean_terminated_length": 561.27734375,
-      "completions/min_length": 223.0,
-      "completions/min_terminated_length": 223.0,
+      "completions/max_length": 1481.0,
+      "completions/max_terminated_length": 1481.0,
+      "completions/mean_length": 561.947265625,
+      "completions/mean_terminated_length": 561.947265625,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.1334228788896351,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0059915904846710765,
-      "kl": 0.05084228515625,
+      "grad_norm": 0.0032762673509562613,
+      "kl": 0.00921630859375,
       "learning_rate": 4.215409716133885e-06,
-      "loss": 0.0005,
-      "num_tokens": 88951937.0,
+      "loss": 0.0001,
+      "num_tokens": 87796905.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4188,25 +4188,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1336.0,
-      "completions/max_terminated_length": 1336.0,
-      "completions/mean_length": 582.072265625,
-      "completions/mean_terminated_length": 582.072265625,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/max_length": 1430.0,
+      "completions/max_terminated_length": 1430.0,
+      "completions/mean_length": 590.71875,
+      "completions/mean_terminated_length": 590.71875,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.1343183344526528,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006120979141360546,
-      "kl": 0.04888916015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07984510566712394,
+      "kl": 0.00885009765625,
       "learning_rate": 4.204319615321151e-06,
-      "loss": 0.0005,
-      "num_tokens": 89566550.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0036,
+      "num_tokens": 88415945.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 150
     },
     {
@@ -4215,20 +4215,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1401.0,
-      "completions/max_terminated_length": 1401.0,
-      "completions/mean_length": 570.46875,
-      "completions/mean_terminated_length": 570.46875,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1422.0,
+      "completions/max_terminated_length": 1422.0,
+      "completions/mean_length": 583.884765625,
+      "completions/mean_terminated_length": 582.7808227539062,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.13521379001567047,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005023995826478019,
-      "kl": 0.04949951171875,
+      "grad_norm": 0.0037633651975202153,
+      "kl": 0.014678955078125,
       "learning_rate": 4.193168494170065e-06,
-      "loss": 0.0005,
-      "num_tokens": 90175830.0,
+      "loss": 0.0001,
+      "num_tokens": 89032094.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4244,19 +4244,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1614.0,
-      "completions/max_terminated_length": 1614.0,
-      "completions/mean_length": 601.76953125,
-      "completions/mean_terminated_length": 601.76953125,
-      "completions/min_length": 232.0,
-      "completions/min_terminated_length": 232.0,
+      "completions/max_length": 1403.0,
+      "completions/max_terminated_length": 1403.0,
+      "completions/mean_length": 586.375,
+      "completions/mean_terminated_length": 586.375,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.13610924557868817,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006632757499928364,
-      "kl": 0.0469970703125,
+      "grad_norm": 0.0031194194696154568,
+      "kl": 0.0089874267578125,
       "learning_rate": 4.181956820559339e-06,
-      "loss": 0.0005,
-      "num_tokens": 90777184.0,
+      "loss": 0.0001,
+      "num_tokens": 89625566.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4272,25 +4272,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1444.0,
-      "completions/max_terminated_length": 1444.0,
-      "completions/mean_length": 553.236328125,
-      "completions/mean_terminated_length": 552.5107421875,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1679.0,
+      "completions/mean_length": 545.861328125,
+      "completions/mean_terminated_length": 542.9216918945312,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.13700470114170585,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 669442.2813001474,
-      "kl": 103936.03869628906,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03997537222476599,
+      "kl": 0.0093994140625,
       "learning_rate": 4.170685064908342e-06,
-      "loss": 1039.7333,
-      "num_tokens": 91353529.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0148,
+      "num_tokens": 90198135.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 153
     },
     {
@@ -4300,25 +4300,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1472.0,
-      "completions/max_terminated_length": 1472.0,
-      "completions/mean_length": 586.95703125,
-      "completions/mean_terminated_length": 585.880615234375,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_length": 1516.0,
+      "completions/max_terminated_length": 1516.0,
+      "completions/mean_length": 585.26953125,
+      "completions/mean_terminated_length": 584.1995849609375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.13790015670472353,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 122492.80685143314,
-      "kl": 38912.03790283203,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005841763677871051,
+      "kl": 0.0170135498046875,
       "learning_rate": 4.159353700157365e-06,
-      "loss": 387.657,
-      "num_tokens": 91944195.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 90787937.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 154
     },
     {
@@ -4328,19 +4328,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1201.0,
-      "completions/max_terminated_length": 1201.0,
-      "completions/mean_length": 548.65234375,
-      "completions/mean_terminated_length": 548.65234375,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/max_length": 1345.0,
+      "completions/max_terminated_length": 1345.0,
+      "completions/mean_length": 526.51171875,
+      "completions/mean_terminated_length": 526.51171875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.1387956122677412,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0060252282224972425,
-      "kl": 0.0513916015625,
+      "grad_norm": 0.004156360771568148,
+      "kl": 0.00970458984375,
       "learning_rate": 4.14796320174778e-06,
-      "loss": 0.0005,
-      "num_tokens": 92511985.0,
+      "loss": 0.0001,
+      "num_tokens": 91344391.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4355,26 +4355,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1338.0,
-      "completions/mean_length": 578.60546875,
-      "completions/mean_terminated_length": 575.7299194335938,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1665.0,
+      "completions/max_terminated_length": 1665.0,
+      "completions/mean_length": 545.390625,
+      "completions/mean_terminated_length": 545.390625,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.1396910678307589,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07969173074262088,
-      "kl": 0.05120849609375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0033243904947931058,
+      "kl": 0.0087432861328125,
       "learning_rate": 4.136514047602087e-06,
-      "loss": 0.0121,
-      "num_tokens": 93123255.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 91938655.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 156
     },
     {
@@ -4384,25 +4384,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1571.0,
-      "completions/max_terminated_length": 1571.0,
-      "completions/mean_length": 583.58203125,
-      "completions/mean_terminated_length": 583.58203125,
-      "completions/min_length": 236.0,
-      "completions/min_terminated_length": 236.0,
+      "completions/max_length": 1236.0,
+      "completions/max_terminated_length": 1236.0,
+      "completions/mean_length": 548.173828125,
+      "completions/mean_terminated_length": 548.173828125,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.14058652339377659,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052805707699068435,
-      "kl": 0.0496826171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0034809305022384003,
+      "kl": 0.0088653564453125,
       "learning_rate": 4.1250067181038635e-06,
-      "loss": -0.0005,
-      "num_tokens": 93739809.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 92537080.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 157
     },
     {
@@ -4411,20 +4411,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1360.0,
-      "completions/max_terminated_length": 1360.0,
-      "completions/mean_length": 596.07421875,
-      "completions/mean_terminated_length": 596.07421875,
-      "completions/min_length": 143.0,
-      "completions/min_terminated_length": 143.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1552.0,
+      "completions/max_terminated_length": 1552.0,
+      "completions/mean_length": 574.640625,
+      "completions/mean_terminated_length": 574.1820068359375,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.14148197895679426,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07305239980212441,
-      "kl": 0.05096435546875,
+      "grad_norm": 0.8063755669722333,
+      "kl": 0.0182952880859375,
       "learning_rate": 4.113441696077608e-06,
-      "loss": -0.0005,
-      "num_tokens": 94378567.0,
+      "loss": 0.0002,
+      "num_tokens": 93164864.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -4439,26 +4439,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1299.0,
-      "completions/max_terminated_length": 1299.0,
-      "completions/mean_length": 601.626953125,
-      "completions/mean_terminated_length": 601.626953125,
-      "completions/min_length": 182.0,
-      "completions/min_terminated_length": 182.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1657.0,
+      "completions/mean_length": 575.2890625,
+      "completions/mean_terminated_length": 572.4070434570312,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.14237743451981194,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005014221442121329,
-      "kl": 0.051513671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.050040387791137446,
+      "kl": 0.00885009765625,
       "learning_rate": 4.101819466768484e-06,
-      "loss": 0.0005,
-      "num_tokens": 94987320.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0164,
+      "num_tokens": 93760132.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 159
     },
     {
@@ -4468,19 +4468,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1367.0,
-      "completions/max_terminated_length": 1367.0,
-      "completions/mean_length": 585.607421875,
-      "completions/mean_terminated_length": 585.607421875,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/max_length": 1469.0,
+      "completions/max_terminated_length": 1469.0,
+      "completions/mean_length": 585.19921875,
+      "completions/mean_terminated_length": 585.19921875,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.14327289008282965,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004656472609840778,
-      "kl": 0.04998779296875,
+      "grad_norm": 0.0036055384423648723,
+      "kl": 0.0088653564453125,
       "learning_rate": 4.0901405178219535e-06,
-      "loss": 0.0005,
-      "num_tokens": 95588047.0,
+      "loss": 0.0001,
+      "num_tokens": 94360650.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4495,26 +4495,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1379.0,
-      "completions/max_terminated_length": 1379.0,
-      "completions/mean_length": 545.453125,
-      "completions/mean_terminated_length": 544.9236450195312,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1168.0,
+      "completions/max_terminated_length": 1168.0,
+      "completions/mean_length": 546.8515625,
+      "completions/mean_terminated_length": 546.8515625,
       "completions/min_length": 180.0,
       "completions/min_terminated_length": 180.0,
       "epoch": 0.14416834564584732,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.11889440151384241,
-      "kl": 0.06475830078125,
-      "learning_rate": 4.078405339263326e-06,
-      "loss": -0.0015,
-      "num_tokens": 96157031.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004011318075529169,
+      "kl": 0.0095977783203125,
+      "learning_rate": 4.078405339263326e-06,
+      "loss": 0.0001,
+      "num_tokens": 94930350.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 161
     },
     {
@@ -4523,20 +4523,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1741.0,
-      "completions/mean_length": 644.041015625,
-      "completions/mean_terminated_length": 641.2935180664062,
-      "completions/min_length": 243.0,
-      "completions/min_terminated_length": 243.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1495.0,
+      "completions/max_terminated_length": 1495.0,
+      "completions/mean_length": 628.458984375,
+      "completions/mean_terminated_length": 628.458984375,
+      "completions/min_length": 248.0,
+      "completions/min_terminated_length": 248.0,
       "epoch": 0.145063801208865,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005252156535586015,
-      "kl": 0.04974365234375,
+      "grad_norm": 0.005474106861111618,
+      "kl": 0.00885009765625,
       "learning_rate": 4.06661442347719e-06,
-      "loss": 0.0005,
-      "num_tokens": 96820956.0,
+      "loss": 0.0001,
+      "num_tokens": 95586297.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4551,20 +4551,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1574.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 598.158203125,
-      "completions/mean_terminated_length": 597.5126953125,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1564.0,
+      "completions/max_terminated_length": 1564.0,
+      "completions/mean_length": 605.478515625,
+      "completions/mean_terminated_length": 605.478515625,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.1459592567718827,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01181182986602236,
-      "kl": 0.04864501953125,
+      "grad_norm": 0.006117440922761299,
+      "kl": 0.0095062255859375,
       "learning_rate": 4.054768265186758e-06,
-      "loss": 0.0005,
-      "num_tokens": 97436333.0,
+      "loss": 0.0001,
+      "num_tokens": 96205422.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4580,19 +4580,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1857.0,
-      "completions/max_terminated_length": 1857.0,
-      "completions/mean_length": 583.1484375,
-      "completions/mean_terminated_length": 583.1484375,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1669.0,
+      "completions/max_terminated_length": 1669.0,
+      "completions/mean_length": 614.33203125,
+      "completions/mean_terminated_length": 614.33203125,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.14685471233490038,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0054554811693594725,
-      "kl": 0.04931640625,
+      "grad_norm": 0.007188095412002734,
+      "kl": 0.010223388671875,
       "learning_rate": 4.0428673614331036e-06,
-      "loss": 0.0005,
-      "num_tokens": 98023273.0,
+      "loss": 0.0001,
+      "num_tokens": 96808328.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4607,20 +4607,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1654.0,
-      "completions/max_terminated_length": 1654.0,
-      "completions/mean_length": 620.806640625,
-      "completions/mean_terminated_length": 619.1565551757812,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1823.0,
+      "completions/max_terminated_length": 1823.0,
+      "completions/mean_length": 652.791015625,
+      "completions/mean_terminated_length": 652.791015625,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.14775016789791806,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006482515932308206,
-      "kl": 0.05206298828125,
+      "grad_norm": 0.0068502617917810356,
+      "kl": 0.0096588134765625,
       "learning_rate": 4.030912211554316e-06,
-      "loss": 0.0005,
-      "num_tokens": 98676790.0,
+      "loss": 0.0001,
+      "num_tokens": 97478221.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4635,26 +4635,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1674.0,
-      "completions/max_terminated_length": 1674.0,
-      "completions/mean_length": 641.078125,
-      "completions/mean_terminated_length": 641.078125,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1845.0,
+      "completions/mean_length": 663.80078125,
+      "completions/mean_terminated_length": 661.0919799804688,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.14864562346093574,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005327353948433487,
-      "kl": 0.04791259765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.09300391631800778,
+      "kl": 0.0095672607421875,
       "learning_rate": 4.018903317164539e-06,
-      "loss": 0.0005,
-      "num_tokens": 99300030.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0123,
+      "num_tokens": 98113095.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 166
     },
     {
@@ -4663,26 +4663,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1534.0,
-      "completions/max_terminated_length": 1534.0,
-      "completions/mean_length": 614.861328125,
-      "completions/mean_terminated_length": 614.861328125,
-      "completions/min_length": 216.0,
-      "completions/min_terminated_length": 216.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1403.0,
+      "completions/mean_length": 624.248046875,
+      "completions/mean_terminated_length": 621.4618530273438,
+      "completions/min_length": 283.0,
+      "completions/min_terminated_length": 283.0,
       "epoch": 0.14954107902395344,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.09522432906751536,
-      "kl": 0.04833984375,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.11604895692678942,
+      "kl": 0.009368896484375,
       "learning_rate": 4.006841182132932e-06,
-      "loss": 0.0009,
-      "num_tokens": 99931063.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0143,
+      "num_tokens": 98748934.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 167
     },
     {
@@ -4691,20 +4691,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1574.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 608.65234375,
-      "completions/mean_terminated_length": 608.65234375,
-      "completions/min_length": 224.0,
-      "completions/min_terminated_length": 224.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1383.0,
+      "completions/max_terminated_length": 1383.0,
+      "completions/mean_length": 629.244140625,
+      "completions/mean_terminated_length": 628.5205688476562,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
       "epoch": 0.15043653458697112,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0054496948068973525,
-      "kl": 0.048583984375,
+      "grad_norm": 0.08576197401573622,
+      "kl": 0.0146331787109375,
       "learning_rate": 3.9947263125625195e-06,
-      "loss": 0.0005,
-      "num_tokens": 100523189.0,
+      "loss": 0.0001,
+      "num_tokens": 99351603.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4720,25 +4720,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1655.0,
-      "completions/max_terminated_length": 1655.0,
-      "completions/mean_length": 635.2734375,
-      "completions/mean_terminated_length": 635.2734375,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/max_length": 1867.0,
+      "completions/max_terminated_length": 1867.0,
+      "completions/mean_length": 656.34375,
+      "completions/mean_terminated_length": 656.34375,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.1513319901499888,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005337411111325454,
-      "kl": 0.0496826171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.052099211200067566,
+      "kl": 0.0095672607421875,
       "learning_rate": 3.982559216768967e-06,
-      "loss": 0.0005,
-      "num_tokens": 101165953.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.008,
+      "num_tokens": 100005155.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 169
     },
     {
@@ -4747,26 +4747,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1283.0,
-      "completions/mean_length": 585.845703125,
-      "completions/mean_terminated_length": 582.9843139648438,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1222.0,
+      "completions/max_terminated_length": 1222.0,
+      "completions/mean_length": 606.388671875,
+      "completions/mean_terminated_length": 606.388671875,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.1522274457130065,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06625166911677717,
-      "kl": 0.04742431640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005600844822173869,
+      "kl": 0.0096893310546875,
       "learning_rate": 3.970340405259245e-06,
-      "loss": 0.0127,
-      "num_tokens": 101758578.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 100608298.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 170
     },
     {
@@ -4775,26 +4775,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1243.0,
-      "completions/mean_length": 574.005859375,
-      "completions/mean_terminated_length": 569.803955078125,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1278.0,
+      "completions/max_terminated_length": 1278.0,
+      "completions/mean_length": 584.5703125,
+      "completions/mean_terminated_length": 584.5703125,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.15312290127602418,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.11944483374125982,
-      "kl": 0.10235595703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004428337264446385,
+      "kl": 0.009765625,
       "learning_rate": 3.958070390710214e-06,
-      "loss": 0.0142,
-      "num_tokens": 102337637.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 101192766.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 171
     },
     {
@@ -4804,19 +4804,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1406.0,
-      "completions/max_terminated_length": 1406.0,
-      "completions/mean_length": 577.810546875,
-      "completions/mean_terminated_length": 577.810546875,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_length": 1509.0,
+      "completions/max_terminated_length": 1509.0,
+      "completions/mean_length": 620.2578125,
+      "completions/mean_terminated_length": 620.2578125,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.15401835683904186,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007013422365760943,
-      "kl": 0.048095703125,
+      "grad_norm": 0.004033698260223417,
+      "kl": 0.0086822509765625,
       "learning_rate": 3.945749687947109e-06,
-      "loss": 0.0005,
-      "num_tokens": 102932356.0,
+      "loss": 0.0001,
+      "num_tokens": 101809218.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4831,26 +4831,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1270.0,
-      "completions/max_terminated_length": 1270.0,
-      "completions/mean_length": 557.375,
-      "completions/mean_terminated_length": 557.375,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1283.0,
+      "completions/max_terminated_length": 1283.0,
+      "completions/mean_length": 572.96875,
+      "completions/mean_terminated_length": 572.2837524414062,
       "completions/min_length": 218.0,
       "completions/min_terminated_length": 218.0,
       "epoch": 0.15491381240205954,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00570724158301358,
-      "kl": 0.0462646484375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 2.1602393924211474,
+      "kl": 0.0108795166015625,
       "learning_rate": 3.933378813921942e-06,
-      "loss": 0.0005,
-      "num_tokens": 103486132.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0032,
+      "num_tokens": 102370978.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 173
     },
     {
@@ -4859,26 +4859,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1897.0,
-      "completions/max_terminated_length": 1897.0,
-      "completions/mean_length": 623.728515625,
-      "completions/mean_terminated_length": 623.728515625,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1435.0,
+      "completions/mean_length": 636.423828125,
+      "completions/mean_terminated_length": 633.6614379882812,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
       "epoch": 0.15580926796507724,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005255713212212517,
-      "kl": 0.048095703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.057483002571070425,
+      "kl": 0.0085906982421875,
       "learning_rate": 3.920958287691811e-06,
-      "loss": 0.0005,
-      "num_tokens": 104111945.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0125,
+      "num_tokens": 103003291.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 174
     },
     {
@@ -4888,25 +4888,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1481.0,
-      "completions/max_terminated_length": 1481.0,
-      "completions/mean_length": 575.19140625,
-      "completions/mean_terminated_length": 575.19140625,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/max_length": 1548.0,
+      "completions/max_terminated_length": 1548.0,
+      "completions/mean_length": 591.62109375,
+      "completions/mean_terminated_length": 591.62109375,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.15670472352809492,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09350953931872348,
-      "kl": 0.04718017578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0036255201179309767,
+      "kl": 0.0087127685546875,
       "learning_rate": 3.908488630397121e-06,
-      "loss": 0.0081,
-      "num_tokens": 104716315.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 103616073.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 175
     },
     {
@@ -4915,20 +4915,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1441.0,
-      "completions/mean_length": 592.453125,
-      "completions/mean_terminated_length": 589.6046752929688,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/max_terminated_length": 1623.0,
+      "completions/mean_length": 577.505859375,
+      "completions/mean_terminated_length": 572.5725708007812,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.1576001790911126,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06667660244468013,
-      "kl": 0.049072265625,
+      "grad_norm": 47744.02884793233,
+      "kl": 1600.0066375732422,
       "learning_rate": 3.8959703652397175e-06,
-      "loss": 0.0117,
-      "num_tokens": 105326371.0,
+      "loss": 16.0277,
+      "num_tokens": 104218476.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -4943,26 +4943,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1544.0,
-      "completions/mean_length": 606.212890625,
-      "completions/mean_terminated_length": 603.391357421875,
-      "completions/min_length": 114.0,
-      "completions/min_terminated_length": 114.0,
+      "completions/max_terminated_length": 1608.0,
+      "completions/mean_length": 605.826171875,
+      "completions/mean_terminated_length": 597.326171875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.1584956346541303,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04243473871670977,
-      "kl": 0.0487060546875,
+      "grad_norm": 0.06412456396717171,
+      "kl": 0.0096893310546875,
       "learning_rate": 3.883404017460935e-06,
-      "loss": 0.0141,
-      "num_tokens": 105935984.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0128,
+      "num_tokens": 104827891.0,
+      "reward": 0.09941406548023224,
+      "reward_std": 0.0012597277527675033,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.994140625,
+      "rewards/format_reward/std": 0.07639661431312561,
       "step": 177
     },
     {
@@ -4971,20 +4971,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1227.0,
-      "completions/max_terminated_length": 1227.0,
-      "completions/mean_length": 572.67578125,
-      "completions/mean_terminated_length": 571.4833374023438,
-      "completions/min_length": 230.0,
-      "completions/min_terminated_length": 230.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1319.0,
+      "completions/max_terminated_length": 1319.0,
+      "completions/mean_length": 580.29296875,
+      "completions/mean_terminated_length": 580.29296875,
+      "completions/min_length": 263.0,
+      "completions/min_terminated_length": 263.0,
       "epoch": 0.15939109021714798,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006519149950539733,
-      "kl": 0.05511474609375,
+      "grad_norm": 0.004861320279381554,
+      "kl": 0.00921630859375,
       "learning_rate": 3.870790114319559e-06,
-      "loss": 0.0006,
-      "num_tokens": 106519466.0,
+      "loss": 0.0001,
+      "num_tokens": 105415273.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -4999,26 +4999,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1574.0,
-      "completions/max_terminated_length": 1574.0,
-      "completions/mean_length": 554.39453125,
-      "completions/mean_terminated_length": 552.3992309570312,
-      "completions/min_length": 148.0,
-      "completions/min_terminated_length": 148.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1593.0,
+      "completions/max_terminated_length": 1593.0,
+      "completions/mean_length": 550.037109375,
+      "completions/mean_terminated_length": 550.037109375,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.16028654578016566,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.023331785461308704,
-      "kl": 0.063720703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.061738880104298316,
+      "kl": 0.009918212890625,
       "learning_rate": 3.858129185069701e-06,
-      "loss": 0.0006,
-      "num_tokens": 107086260.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0,
+      "num_tokens": 105979836.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 179
     },
     {
@@ -5027,20 +5027,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1365.0,
-      "completions/max_terminated_length": 1365.0,
-      "completions/mean_length": 587.150390625,
-      "completions/mean_terminated_length": 586.5479125976562,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1259.0,
+      "completions/max_terminated_length": 1259.0,
+      "completions/mean_length": 570.638671875,
+      "completions/mean_terminated_length": 570.638671875,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.16118200134318333,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005382943100030063,
-      "kl": 0.0528564453125,
+      "grad_norm": 0.004637633663432918,
+      "kl": 0.009307861328125,
       "learning_rate": 3.845421760938597e-06,
-      "loss": 0.0005,
-      "num_tokens": 107711489.0,
+      "loss": 0.0001,
+      "num_tokens": 106596611.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5053,28 +5053,28 @@
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1169.0,
-      "completions/max_terminated_length": 1169.0,
-      "completions/mean_length": 538.271484375,
-      "completions/mean_terminated_length": 538.271484375,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1052.0,
+      "completions/mean_length": 532.470703125,
+      "completions/mean_terminated_length": 529.5048828125,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.16207745690620104,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005548822156078627,
-      "kl": 0.0509033203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08330239155652147,
+      "kl": 0.0096435546875,
       "learning_rate": 3.832668375104312e-06,
-      "loss": 0.0005,
-      "num_tokens": 108283516.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.016,
+      "num_tokens": 107165668.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 181
     },
     {
@@ -5084,19 +5084,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1659.0,
-      "completions/max_terminated_length": 1659.0,
-      "completions/mean_length": 586.451171875,
-      "completions/mean_terminated_length": 586.451171875,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_length": 1999.0,
+      "completions/max_terminated_length": 1999.0,
+      "completions/mean_length": 569.216796875,
+      "completions/mean_terminated_length": 569.216796875,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.16297291246921872,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00603788980728595,
-      "kl": 0.0499267578125,
+      "grad_norm": 0.005295883162438518,
+      "kl": 0.0092315673828125,
       "learning_rate": 3.8198695626733725e-06,
-      "loss": 0.0005,
-      "num_tokens": 108898835.0,
+      "loss": 0.0001,
+      "num_tokens": 107772163.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5112,19 +5112,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1504.0,
-      "completions/max_terminated_length": 1504.0,
-      "completions/mean_length": 585.361328125,
-      "completions/mean_terminated_length": 585.361328125,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/max_length": 1655.0,
+      "completions/max_terminated_length": 1655.0,
+      "completions/mean_length": 589.267578125,
+      "completions/mean_terminated_length": 589.267578125,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.1638683680322364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005262098905925186,
-      "kl": 0.0496826171875,
+      "grad_norm": 0.005391282000726491,
+      "kl": 0.0091400146484375,
       "learning_rate": 3.8070258606583156e-06,
-      "loss": 0.0005,
-      "num_tokens": 109496748.0,
+      "loss": 0.0001,
+      "num_tokens": 108372076.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5140,25 +5140,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1667.0,
-      "completions/max_terminated_length": 1667.0,
-      "completions/mean_length": 594.75,
-      "completions/mean_terminated_length": 594.75,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_length": 1423.0,
+      "completions/max_terminated_length": 1423.0,
+      "completions/mean_length": 595.279296875,
+      "completions/mean_terminated_length": 595.279296875,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.16476382359525407,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0048278578819136685,
-      "kl": 0.0506591796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0952696472933219,
+      "kl": 0.009857177734375,
       "learning_rate": 3.7941378079551544e-06,
-      "loss": 0.0005,
-      "num_tokens": 110117148.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0028,
+      "num_tokens": 108992747.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 184
     },
     {
@@ -5167,26 +5167,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1774.0,
-      "completions/max_terminated_length": 1774.0,
-      "completions/mean_length": 592.947265625,
-      "completions/mean_terminated_length": 588.956787109375,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1404.0,
+      "completions/max_terminated_length": 1404.0,
+      "completions/mean_length": 567.849609375,
+      "completions/mean_terminated_length": 567.849609375,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.16565927915827178,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008863827601325813,
-      "kl": 0.07080078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.058420288604048326,
+      "kl": 0.00994873046875,
       "learning_rate": 3.7812059453207677e-06,
-      "loss": 0.0007,
-      "num_tokens": 110735937.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0013,
+      "num_tokens": 109598686.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 185
     },
     {
@@ -5195,20 +5195,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1491.0,
-      "completions/max_terminated_length": 1491.0,
-      "completions/mean_length": 565.056640625,
-      "completions/mean_terminated_length": 562.4459838867188,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1777.0,
+      "completions/max_terminated_length": 1777.0,
+      "completions/mean_length": 547.923828125,
+      "completions/mean_terminated_length": 547.923828125,
+      "completions/min_length": 152.0,
+      "completions/min_terminated_length": 152.0,
       "epoch": 0.16655473472128945,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02719947360988251,
-      "kl": 0.0760498046875,
+      "grad_norm": 0.005346867400011268,
+      "kl": 0.009521484375,
       "learning_rate": 3.768230815350213e-06,
-      "loss": 0.0008,
-      "num_tokens": 111340030.0,
+      "loss": 0.0001,
+      "num_tokens": 110194007.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5223,26 +5223,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1201.0,
-      "completions/max_terminated_length": 1201.0,
-      "completions/mean_length": 552.83984375,
-      "completions/mean_terminated_length": 551.7005615234375,
-      "completions/min_length": 111.0,
-      "completions/min_terminated_length": 111.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1268.0,
+      "completions/max_terminated_length": 1268.0,
+      "completions/mean_length": 544.93359375,
+      "completions/mean_terminated_length": 544.93359375,
+      "completions/min_length": 106.0,
+      "completions/min_terminated_length": 106.0,
       "epoch": 0.16745019028430713,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.9236121462672159,
-      "kl": 0.057373046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.008310871893682886,
+      "kl": 0.0106201171875,
       "learning_rate": 3.7552129624539557e-06,
-      "loss": 0.0075,
-      "num_tokens": 111923644.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 110773573.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 187
     },
     {
@@ -5252,25 +5252,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1403.0,
-      "completions/mean_length": 558.384765625,
-      "completions/mean_terminated_length": 555.4696655273438,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_length": 1137.0,
+      "completions/max_terminated_length": 1137.0,
+      "completions/mean_length": 530.16796875,
+      "completions/mean_terminated_length": 529.2857055664062,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.16834564584732484,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.061955717355027674,
-      "kl": 0.050537109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 158.14659789913986,
+      "kl": 22.0072021484375,
       "learning_rate": 3.7421529328350316e-06,
-      "loss": -0.0028,
-      "num_tokens": 112512289.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.2275,
+      "num_tokens": 111347771.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 188
     },
     {
@@ -5279,20 +5279,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1771.0,
-      "completions/max_terminated_length": 1771.0,
-      "completions/mean_length": 601.98828125,
-      "completions/mean_terminated_length": 601.3092041015625,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1653.0,
+      "completions/max_terminated_length": 1653.0,
+      "completions/mean_length": 594.1875,
+      "completions/mean_terminated_length": 594.1875,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
       "epoch": 0.16924110141034251,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005052704969820576,
-      "kl": 0.0516357421875,
+      "grad_norm": 0.008808132081770864,
+      "kl": 0.0102081298828125,
       "learning_rate": 3.7290512744661274e-06,
-      "loss": 0.0005,
-      "num_tokens": 113142027.0,
+      "loss": 0.0001,
+      "num_tokens": 111973515.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5307,20 +5307,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1692.0,
-      "completions/mean_length": 589.88671875,
-      "completions/mean_terminated_length": 581.292724609375,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1485.0,
+      "completions/max_terminated_length": 1485.0,
+      "completions/mean_length": 583.603515625,
+      "completions/mean_terminated_length": 583.603515625,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.1701365569733602,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.2800674868147115,
-      "kl": 0.85662841796875,
+      "grad_norm": 0.010967235624627656,
+      "kl": 0.01129150390625,
       "learning_rate": 3.715908537066589e-06,
-      "loss": 0.0086,
-      "num_tokens": 113742433.0,
+      "loss": 0.0001,
+      "num_tokens": 112570704.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5335,20 +5335,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1471.0,
-      "completions/max_terminated_length": 1471.0,
-      "completions/mean_length": 553.888671875,
-      "completions/mean_terminated_length": 553.0528564453125,
-      "completions/min_length": 216.0,
-      "completions/min_terminated_length": 216.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1795.0,
+      "completions/max_terminated_length": 1795.0,
+      "completions/mean_length": 570.9375,
+      "completions/mean_terminated_length": 570.9375,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.17103201253637787,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006196419562863969,
-      "kl": 0.04998779296875,
+      "grad_norm": 0.013565646568363069,
+      "kl": 0.0124664306640625,
       "learning_rate": 3.7027252720793538e-06,
-      "loss": 0.0005,
-      "num_tokens": 114329720.0,
+      "loss": 0.0001,
+      "num_tokens": 113166720.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5364,25 +5364,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1544.0,
-      "completions/max_terminated_length": 1544.0,
-      "completions/mean_length": 557.12109375,
-      "completions/mean_terminated_length": 557.12109375,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/max_length": 1883.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 581.537109375,
+      "completions/mean_terminated_length": 581.537109375,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.17192746809939558,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06389277517732068,
-      "kl": 0.04864501953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.014691973713636838,
+      "kl": 0.013153076171875,
       "learning_rate": 3.689502032647817e-06,
-      "loss": 0.0013,
-      "num_tokens": 114920070.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 113769571.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 192
     },
     {
@@ -5392,25 +5392,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1318.0,
-      "completions/max_terminated_length": 1318.0,
-      "completions/mean_length": 540.427734375,
-      "completions/mean_terminated_length": 540.427734375,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/max_length": 1793.0,
+      "completions/max_terminated_length": 1793.0,
+      "completions/mean_length": 550.32421875,
+      "completions/mean_terminated_length": 550.32421875,
+      "completions/min_length": 104.0,
+      "completions/min_terminated_length": 104.0,
       "epoch": 0.17282292366241325,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0056093930735562555,
-      "kl": 0.0491943359375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06606192521106205,
+      "kl": 0.0179595947265625,
       "learning_rate": 3.6762393735926245e-06,
-      "loss": 0.0005,
-      "num_tokens": 115477249.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0033,
+      "num_tokens": 114331817.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 193
     },
     {
@@ -5419,26 +5419,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1444.0,
-      "completions/max_terminated_length": 1444.0,
-      "completions/mean_length": 549.9765625,
-      "completions/mean_terminated_length": 549.9765625,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2043.0,
+      "completions/mean_length": 587.3203125,
+      "completions/mean_terminated_length": 584.4618530273438,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.17371837922543093,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0797524710726536,
-      "kl": 0.05047607421875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0182360788370408,
+      "kl": 0.0151214599609375,
       "learning_rate": 3.6629378513883852e-06,
-      "loss": 0.0031,
-      "num_tokens": 116066757.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 114940445.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 194
     },
     {
@@ -5448,19 +5448,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1387.0,
-      "completions/max_terminated_length": 1387.0,
-      "completions/mean_length": 541.453125,
-      "completions/mean_terminated_length": 541.453125,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_length": 1629.0,
+      "completions/max_terminated_length": 1629.0,
+      "completions/mean_length": 559.173828125,
+      "completions/mean_terminated_length": 559.173828125,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.17461383478844864,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004492696441640513,
-      "kl": 0.0484619140625,
+      "grad_norm": 0.015121651645506792,
+      "kl": 0.01385498046875,
       "learning_rate": 3.6495980241403307e-06,
-      "loss": 0.0005,
-      "num_tokens": 116622141.0,
+      "loss": 0.0001,
+      "num_tokens": 115504902.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5475,26 +5475,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1729.0,
-      "completions/max_terminated_length": 1729.0,
-      "completions/mean_length": 606.66015625,
-      "completions/mean_terminated_length": 606.66015625,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1650.0,
+      "completions/mean_length": 644.818359375,
+      "completions/mean_terminated_length": 642.0723876953125,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.1755092903514663,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06668454810927989,
-      "kl": 0.0494384765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.10159970928736292,
+      "kl": 0.012451171875,
       "learning_rate": 3.636220451560896e-06,
-      "loss": 0.0012,
-      "num_tokens": 117240367.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0108,
+      "num_tokens": 116142665.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 196
     },
     {
@@ -5503,26 +5503,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1500.0,
-      "completions/max_terminated_length": 1500.0,
-      "completions/mean_length": 589.2578125,
-      "completions/mean_terminated_length": 589.2578125,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1878.0,
+      "completions/mean_length": 619.048828125,
+      "completions/mean_terminated_length": 613.4451293945312,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.176404745914484,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004513425242810063,
-      "kl": 0.05010986328125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.0956666312311249,
+      "kl": 0.0129852294921875,
       "learning_rate": 3.622805694946235e-06,
-      "loss": 0.0005,
-      "num_tokens": 117866035.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.011,
+      "num_tokens": 116783586.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 197
     },
     {
@@ -5531,20 +5531,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1554.0,
-      "completions/max_terminated_length": 1554.0,
-      "completions/mean_length": 572.9609375,
-      "completions/mean_terminated_length": 571.9628295898438,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1488.0,
+      "completions/max_terminated_length": 1488.0,
+      "completions/mean_length": 592.234375,
+      "completions/mean_terminated_length": 592.234375,
+      "completions/min_length": 284.0,
+      "completions/min_terminated_length": 284.0,
       "epoch": 0.17730020147750167,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005737804995862872,
-      "kl": 0.0511474609375,
+      "grad_norm": 0.011625600847511287,
+      "kl": 0.0123138427734375,
       "learning_rate": 3.609354317152667e-06,
-      "loss": 0.0005,
-      "num_tokens": 118461087.0,
+      "loss": 0.0001,
+      "num_tokens": 117388506.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5559,20 +5559,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1371.0,
-      "completions/max_terminated_length": 1371.0,
-      "completions/mean_length": 579.970703125,
-      "completions/mean_terminated_length": 579.365966796875,
-      "completions/min_length": 174.0,
-      "completions/min_terminated_length": 174.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1462.0,
+      "completions/max_terminated_length": 1462.0,
+      "completions/mean_length": 586.54296875,
+      "completions/mean_terminated_length": 586.54296875,
+      "completions/min_length": 231.0,
+      "completions/min_terminated_length": 231.0,
       "epoch": 0.17819565704051937,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011689236262334025,
-      "kl": 0.05712890625,
+      "grad_norm": 0.010127412065854468,
+      "kl": 0.011474609375,
       "learning_rate": 3.595866882573063e-06,
-      "loss": 0.0006,
-      "num_tokens": 119055360.0,
+      "loss": 0.0001,
+      "num_tokens": 117986144.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5587,20 +5587,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1640.0,
-      "completions/mean_length": 555.169921875,
-      "completions/mean_terminated_length": 552.24853515625,
-      "completions/min_length": 148.0,
-      "completions/min_terminated_length": 148.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1648.0,
+      "completions/max_terminated_length": 1648.0,
+      "completions/mean_length": 549.142578125,
+      "completions/mean_terminated_length": 549.142578125,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
       "epoch": 0.17909111260353705,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005120448524080564,
-      "kl": 0.0484619140625,
+      "grad_norm": 0.009383976587188618,
+      "kl": 0.0116729736328125,
       "learning_rate": 3.5823439571131675e-06,
-      "loss": 0.0005,
-      "num_tokens": 119650455.0,
+      "loss": 0.0001,
+      "num_tokens": 118578153.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5615,20 +5615,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1604.0,
-      "completions/max_terminated_length": 1604.0,
-      "completions/mean_length": 583.546875,
-      "completions/mean_terminated_length": 579.5451049804688,
-      "completions/min_length": 125.0,
-      "completions/min_terminated_length": 125.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1422.0,
+      "completions/max_terminated_length": 1422.0,
+      "completions/mean_length": 582.240234375,
+      "completions/mean_terminated_length": 582.240234375,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.17998656816655473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02771302136152841,
-      "kl": 0.06964111328125,
+      "grad_norm": 0.005973744897078448,
+      "kl": 0.0115814208984375,
       "learning_rate": 3.5687861081678477e-06,
-      "loss": 0.0007,
-      "num_tokens": 120246799.0,
+      "loss": 0.0001,
+      "num_tokens": 119173828.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5644,19 +5644,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1386.0,
-      "completions/max_terminated_length": 1386.0,
-      "completions/mean_length": 593.412109375,
-      "completions/mean_terminated_length": 593.412109375,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/max_length": 1376.0,
+      "completions/max_terminated_length": 1376.0,
+      "completions/mean_length": 579.224609375,
+      "completions/mean_terminated_length": 579.224609375,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.18088202372957243,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007212915094271475,
-      "kl": 0.05010986328125,
+      "grad_norm": 0.004596046170425412,
+      "kl": 0.010162353515625,
       "learning_rate": 3.555193904597291e-06,
-      "loss": 0.0005,
-      "num_tokens": 120858322.0,
+      "loss": 0.0001,
+      "num_tokens": 119778087.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5671,20 +5671,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1592.0,
-      "completions/mean_length": 590.431640625,
-      "completions/mean_terminated_length": 587.5792236328125,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1391.0,
+      "completions/max_terminated_length": 1391.0,
+      "completions/mean_length": 574.671875,
+      "completions/mean_terminated_length": 574.671875,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.1817774792925901,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005532916229287632,
-      "kl": 0.04833984375,
+      "grad_norm": 0.004380723205360899,
+      "kl": 0.009918212890625,
       "learning_rate": 3.541567916703138e-06,
-      "loss": 0.0005,
-      "num_tokens": 121442351.0,
+      "loss": 0.0001,
+      "num_tokens": 120354047.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5700,19 +5700,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2002.0,
-      "completions/mean_length": 629.58984375,
-      "completions/mean_terminated_length": 626.8140869140625,
-      "completions/min_length": 199.0,
-      "completions/min_terminated_length": 199.0,
+      "completions/max_length": 1886.0,
+      "completions/max_terminated_length": 1886.0,
+      "completions/mean_length": 626.44140625,
+      "completions/mean_terminated_length": 625.624267578125,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.1826729348556078,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07012025346138279,
-      "kl": 0.04815673828125,
+      "grad_norm": 46.46835422177109,
+      "kl": 1.1946563720703125,
       "learning_rate": 3.5279087162045517e-06,
-      "loss": 0.0031,
-      "num_tokens": 122078173.0,
+      "loss": 0.019,
+      "num_tokens": 120988257.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -5728,19 +5728,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1894.0,
-      "completions/max_terminated_length": 1894.0,
-      "completions/mean_length": 618.482421875,
-      "completions/mean_terminated_length": 618.482421875,
-      "completions/min_length": 294.0,
-      "completions/min_terminated_length": 294.0,
+      "completions/max_length": 1310.0,
+      "completions/max_terminated_length": 1310.0,
+      "completions/mean_length": 588.21484375,
+      "completions/mean_terminated_length": 588.21484375,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.18356839041862547,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005303183352547322,
-      "kl": 0.04888916015625,
+      "grad_norm": 0.006442269227273499,
+      "kl": 0.009613037109375,
       "learning_rate": 3.5142168762142265e-06,
-      "loss": 0.0005,
-      "num_tokens": 122732068.0,
+      "loss": 0.0001,
+      "num_tokens": 121626655.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5756,25 +5756,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1630.0,
-      "completions/max_terminated_length": 1630.0,
-      "completions/mean_length": 612.640625,
-      "completions/mean_terminated_length": 612.640625,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/max_length": 1938.0,
+      "completions/max_terminated_length": 1938.0,
+      "completions/mean_length": 617.9921875,
+      "completions/mean_terminated_length": 617.9921875,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.18446384598164317,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06296171173023327,
-      "kl": 0.05108642578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.007299388832908803,
+      "kl": 0.0101470947265625,
       "learning_rate": 3.500492971214347e-06,
-      "loss": -0.0047,
-      "num_tokens": 123357500.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 122254827.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 206
     },
     {
@@ -5783,20 +5783,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1385.0,
-      "completions/mean_length": 565.35546875,
-      "completions/mean_terminated_length": 561.558837890625,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_terminated_length": 1213.0,
+      "completions/mean_length": 549.857421875,
+      "completions/mean_terminated_length": 546.9256591796875,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.18535930154466085,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008285648181235224,
-      "kl": 0.050537109375,
+      "grad_norm": 0.007711548389239973,
+      "kl": 0.0094757080078125,
       "learning_rate": 3.48673757703248e-06,
-      "loss": 0.0005,
-      "num_tokens": 123947746.0,
+      "loss": 0.0001,
+      "num_tokens": 122837138.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5811,20 +5811,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1469.0,
-      "completions/mean_length": 582.73828125,
-      "completions/mean_terminated_length": 572.6154174804688,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1328.0,
+      "completions/max_terminated_length": 1328.0,
+      "completions/mean_length": 559.216796875,
+      "completions/mean_terminated_length": 559.216796875,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.18625475710767853,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.059277726369991776,
-      "kl": 0.06085205078125,
+      "grad_norm": 0.007709733879977267,
+      "kl": 0.0093841552734375,
       "learning_rate": 3.472951270817418e-06,
-      "loss": 0.0006,
-      "num_tokens": 124560364.0,
+      "loss": 0.0001,
+      "num_tokens": 123437713.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5839,20 +5839,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1881.0,
-      "completions/max_terminated_length": 1881.0,
-      "completions/mean_length": 578.189453125,
-      "completions/mean_terminated_length": 578.189453125,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1805.0,
+      "completions/mean_length": 551.783203125,
+      "completions/mean_terminated_length": 548.8551635742188,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
       "epoch": 0.1871502126706962,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004773000588147992,
-      "kl": 0.0477294921875,
+      "grad_norm": 0.006901104494651673,
+      "kl": 0.0103912353515625,
       "learning_rate": 3.4591346310149578e-06,
-      "loss": 0.0005,
-      "num_tokens": 125158765.0,
+      "loss": 0.0001,
+      "num_tokens": 124022594.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5867,20 +5867,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2012.0,
-      "completions/mean_length": 632.3125,
-      "completions/mean_terminated_length": 624.06103515625,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/max_terminated_length": 1738.0,
+      "completions/mean_length": 577.857421875,
+      "completions/mean_terminated_length": 574.9804077148438,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.1880456682337139,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02963143641657182,
-      "kl": 0.0645751953125,
+      "grad_norm": 0.005911630324218355,
+      "kl": 0.0094451904296875,
       "learning_rate": 3.445288237343632e-06,
-      "loss": 0.0006,
-      "num_tokens": 125815789.0,
+      "loss": 0.0001,
+      "num_tokens": 124651737.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5895,26 +5895,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1522.0,
-      "completions/mean_length": 668.24609375,
-      "completions/mean_terminated_length": 646.3452758789062,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/max_terminated_length": 1783.0,
+      "completions/mean_length": 631.33203125,
+      "completions/mean_terminated_length": 628.5596923828125,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.18894112379673159,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00816300375427543,
-      "kl": 0.05059814453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08768246283609199,
+      "kl": 0.009124755859375,
       "learning_rate": 3.4314126707703895e-06,
-      "loss": 0.0005,
-      "num_tokens": 126491035.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0086,
+      "num_tokens": 125308083.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 211
     },
     {
@@ -5923,20 +5923,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1748.0,
-      "completions/mean_length": 605.109375,
-      "completions/mean_terminated_length": 600.2549438476562,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1173.0,
+      "completions/max_terminated_length": 1173.0,
+      "completions/mean_length": 552.49609375,
+      "completions/mean_terminated_length": 552.49609375,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.18983657935974926,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00596263048445519,
-      "kl": 0.05322265625,
+      "grad_norm": 0.00454190795624133,
+      "kl": 0.0095672607421875,
       "learning_rate": 3.4175085134862128e-06,
-      "loss": 0.0005,
-      "num_tokens": 127117859.0,
+      "loss": 0.0001,
+      "num_tokens": 125907969.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5951,20 +5951,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2028.0,
-      "completions/mean_length": 681.240234375,
-      "completions/mean_terminated_length": 654.0139770507812,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1566.0,
+      "completions/max_terminated_length": 1566.0,
+      "completions/mean_length": 611.28125,
+      "completions/mean_terminated_length": 611.28125,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
       "epoch": 0.19073203492276697,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005312087159167155,
-      "kl": 0.04901123046875,
+      "grad_norm": 0.003633879019486841,
+      "kl": 0.008270263671875,
       "learning_rate": 3.4035763488816953e-06,
-      "loss": 0.0005,
-      "num_tokens": 127788046.0,
+      "loss": 0.0001,
+      "num_tokens": 126542337.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -5979,26 +5979,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1576.0,
-      "completions/mean_length": 613.779296875,
-      "completions/mean_terminated_length": 588.962158203125,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1747.0,
+      "completions/max_terminated_length": 1747.0,
+      "completions/mean_length": 565.1015625,
+      "completions/mean_terminated_length": 565.1015625,
+      "completions/min_length": 233.0,
+      "completions/min_terminated_length": 233.0,
       "epoch": 0.19162749048578465,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1042.0108001699423,
-      "kl": 1.84454345703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003575289557022859,
+      "kl": 0.0095367431640625,
       "learning_rate": 3.3896167615225594e-06,
-      "loss": 0.0181,
-      "num_tokens": 128423037.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 127152405.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 214
     },
     {
@@ -6007,20 +6007,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1859.0,
-      "completions/mean_length": 549.240234375,
-      "completions/mean_terminated_length": 532.712890625,
-      "completions/min_length": 159.0,
-      "completions/min_terminated_length": 159.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1061.0,
+      "completions/max_terminated_length": 1061.0,
+      "completions/mean_length": 512.28515625,
+      "completions/mean_terminated_length": 512.28515625,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.19252294604880232,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.015176587381927465,
-      "kl": 0.06927490234375,
+      "grad_norm": 0.0037462711645953797,
+      "kl": 0.0099639892578125,
       "learning_rate": 3.375630337125133e-06,
-      "loss": 0.0007,
-      "num_tokens": 128981160.0,
+      "loss": 0.0001,
+      "num_tokens": 127691607.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6035,26 +6035,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1923.0,
-      "completions/mean_length": 634.53515625,
-      "completions/mean_terminated_length": 616.1663818359375,
-      "completions/min_length": 205.0,
-      "completions/min_terminated_length": 205.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1562.0,
+      "completions/max_terminated_length": 1562.0,
+      "completions/mean_length": 579.341796875,
+      "completions/mean_terminated_length": 579.341796875,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.19341840161182,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011838339376112336,
-      "kl": 0.05859375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.037437587504057374,
+      "kl": 0.0086669921875,
       "learning_rate": 3.361617662531772e-06,
-      "loss": 0.0006,
-      "num_tokens": 129585882.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "num_tokens": 128268070.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 216
     },
     {
@@ -6063,26 +6063,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1946.0,
-      "completions/mean_length": 626.982421875,
-      "completions/mean_terminated_length": 610.1324462890625,
-      "completions/min_length": 149.0,
-      "completions/min_terminated_length": 149.0,
+      "completions/max_terminated_length": 1805.0,
+      "completions/mean_length": 589.671875,
+      "completions/mean_terminated_length": 583.9530029296875,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.1943138571748377,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004774713168803346,
-      "kl": 0.04888916015625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.055260322848602106,
+      "kl": 0.009307861328125,
       "learning_rate": 3.347579325686237e-06,
-      "loss": 0.0005,
-      "num_tokens": 130210065.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0096,
+      "num_tokens": 128873150.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 217
     },
     {
@@ -6091,20 +6091,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1875.0,
-      "completions/mean_length": 628.736328125,
-      "completions/mean_terminated_length": 594.6740112304688,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1452.0,
+      "completions/max_terminated_length": 1452.0,
+      "completions/mean_length": 550.720703125,
+      "completions/mean_terminated_length": 550.720703125,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.19520931273785538,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009277070324678074,
-      "kl": 0.05230712890625,
+      "grad_norm": 0.004407824970276491,
+      "kl": 0.0103302001953125,
       "learning_rate": 3.333515915609027e-06,
-      "loss": 0.0005,
-      "num_tokens": 130821866.0,
+      "loss": 0.0001,
+      "num_tokens": 129445007.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6119,26 +6119,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1792.0,
-      "completions/mean_length": 618.80078125,
-      "completions/mean_terminated_length": 606.6588134765625,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/max_terminated_length": 1731.0,
+      "completions/mean_length": 581.884765625,
+      "completions/mean_terminated_length": 579.015625,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.19610476830087306,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.056877402143134144,
-      "kl": 0.0538330078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006748768231765914,
+      "kl": 0.0100860595703125,
       "learning_rate": 3.3194280223726616e-06,
-      "loss": 0.0095,
-      "num_tokens": 131481876.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 130086116.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 219
     },
     {
@@ -6147,20 +6147,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1790.0,
-      "completions/mean_length": 641.310546875,
-      "completions/mean_terminated_length": 628.4141845703125,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1772.0,
+      "completions/max_terminated_length": 1772.0,
+      "completions/mean_length": 589.65625,
+      "completions/mean_terminated_length": 589.65625,
+      "completions/min_length": 86.0,
+      "completions/min_terminated_length": 86.0,
       "epoch": 0.19700022386389077,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00940584007592381,
-      "kl": 0.05828857421875,
+      "grad_norm": 0.003539717009736075,
+      "kl": 0.0095062255859375,
       "learning_rate": 3.305316237076927e-06,
-      "loss": 0.0006,
-      "num_tokens": 132144915.0,
+      "loss": 0.0001,
+      "num_tokens": 130722708.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6175,20 +6175,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1961.0,
-      "completions/mean_length": 562.9921875,
-      "completions/mean_terminated_length": 551.6476440429688,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1886.0,
+      "completions/max_terminated_length": 1886.0,
+      "completions/mean_length": 540.95703125,
+      "completions/mean_terminated_length": 540.2034912109375,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.19789567942690844,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006685311606671522,
-      "kl": 0.0562744140625,
+      "grad_norm": 0.00650263702708007,
+      "kl": 0.0131988525390625,
       "learning_rate": 3.291181151824071e-06,
-      "loss": 0.0006,
-      "num_tokens": 132707855.0,
+      "loss": 0.0001,
+      "num_tokens": 131274366.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6203,20 +6203,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1332.0,
-      "completions/mean_length": 571.44921875,
-      "completions/mean_terminated_length": 562.74658203125,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1143.0,
+      "completions/max_terminated_length": 1143.0,
+      "completions/mean_length": 545.39453125,
+      "completions/mean_terminated_length": 545.39453125,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.19879113498992612,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00478893835093211,
-      "kl": 0.04815673828125,
+      "grad_norm": 0.0033779834664063595,
+      "kl": 0.00946044921875,
       "learning_rate": 3.27702335969396e-06,
-      "loss": 0.0005,
-      "num_tokens": 133284101.0,
+      "loss": 0.0001,
+      "num_tokens": 131837272.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6231,26 +6231,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1737.0,
-      "completions/mean_length": 577.771484375,
-      "completions/mean_terminated_length": 568.0078735351562,
-      "completions/min_length": 109.0,
-      "completions/min_terminated_length": 109.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1666.0,
+      "completions/max_terminated_length": 1666.0,
+      "completions/mean_length": 565.56640625,
+      "completions/mean_terminated_length": 565.56640625,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.1996865905529438,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006428223247827128,
-      "kl": 0.0548095703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.055179447231091354,
+      "kl": 0.00946044921875,
       "learning_rate": 3.2628434547191985e-06,
-      "loss": 0.0005,
-      "num_tokens": 133876064.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0,
+      "num_tokens": 132422986.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 223
     },
     {
@@ -6259,26 +6259,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1768.0,
-      "completions/mean_length": 591.037109375,
-      "completions/mean_terminated_length": 578.5680541992188,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1929.0,
+      "completions/max_terminated_length": 1929.0,
+      "completions/mean_length": 565.845703125,
+      "completions/mean_terminated_length": 565.2308959960938,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.2005820461159615,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06631243326090758,
-      "kl": 0.06134033203125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.21010312450686547,
+      "kl": 0.0242919921875,
       "learning_rate": 3.2486420318601973e-06,
-      "loss": 0.0131,
-      "num_tokens": 134478851.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 133012875.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 224
     },
     {
@@ -6287,20 +6287,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1559.0,
-      "completions/mean_length": 597.66015625,
-      "completions/mean_terminated_length": 592.931396484375,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/max_terminated_length": 1093.0,
+      "completions/mean_length": 578.96484375,
+      "completions/mean_terminated_length": 576.0900268554688,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.20147750167897918,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008008668550094192,
-      "kl": 0.05316162109375,
+      "grad_norm": 0.013242127221665219,
+      "kl": 0.009429931640625,
       "learning_rate": 3.2344196869802187e-06,
-      "loss": 0.0005,
-      "num_tokens": 135111445.0,
+      "loss": 0.0001,
+      "num_tokens": 133635897.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6315,20 +6315,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1460.0,
-      "completions/max_terminated_length": 1460.0,
-      "completions/mean_length": 582.05859375,
-      "completions/mean_terminated_length": 580.8982543945312,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2034.0,
+      "completions/max_terminated_length": 2034.0,
+      "completions/mean_length": 566.775390625,
+      "completions/mean_terminated_length": 566.775390625,
+      "completions/min_length": 259.0,
+      "completions/min_terminated_length": 259.0,
       "epoch": 0.20237295724199686,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006453994968670067,
-      "kl": 0.05267333984375,
+      "grad_norm": 0.003305902510981695,
+      "kl": 0.008819580078125,
       "learning_rate": 3.2201770168203694e-06,
-      "loss": 0.0005,
-      "num_tokens": 135674979.0,
+      "loss": 0.0001,
+      "num_tokens": 134191606.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6343,20 +6343,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1670.0,
-      "completions/max_terminated_length": 1670.0,
-      "completions/mean_length": 626.3046875,
-      "completions/mean_terminated_length": 626.3046875,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1829.0,
+      "completions/mean_length": 614.041015625,
+      "completions/mean_terminated_length": 611.2348022460938,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.20326841280501456,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0049162524656169515,
-      "kl": 0.04791259765625,
+      "grad_norm": 0.003785127098280972,
+      "kl": 0.008941650390625,
       "learning_rate": 3.205914618974563e-06,
-      "loss": 0.0005,
-      "num_tokens": 136314863.0,
+      "loss": 0.0001,
+      "num_tokens": 134825211.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6371,26 +6371,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1508.0,
-      "completions/mean_length": 588.765625,
-      "completions/mean_terminated_length": 583.0431518554688,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1418.0,
+      "completions/max_terminated_length": 1418.0,
+      "completions/mean_length": 573.529296875,
+      "completions/mean_terminated_length": 573.529296875,
+      "completions/min_length": 209.0,
+      "completions/min_terminated_length": 209.0,
       "epoch": 0.20416386836803224,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08725654752259214,
-      "kl": 0.0467529296875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003247968244119725,
+      "kl": 0.0087890625,
       "learning_rate": 3.1916330918644496e-06,
-      "loss": 0.0138,
-      "num_tokens": 136916679.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 135419226.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 228
     },
     {
@@ -6400,19 +6400,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1188.0,
-      "completions/max_terminated_length": 1188.0,
-      "completions/mean_length": 552.72265625,
-      "completions/mean_terminated_length": 552.72265625,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/max_length": 1325.0,
+      "completions/max_terminated_length": 1325.0,
+      "completions/mean_length": 560.087890625,
+      "completions/mean_terminated_length": 560.087890625,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.20505932393104992,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005918437311628653,
-      "kl": 0.04791259765625,
+      "grad_norm": 0.0032627582108987214,
+      "kl": 0.0091094970703125,
       "learning_rate": 3.177333034714303e-06,
-      "loss": 0.0005,
-      "num_tokens": 137493209.0,
+      "loss": 0.0001,
+      "num_tokens": 135999527.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6427,26 +6427,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1678.0,
-      "completions/max_terminated_length": 1678.0,
-      "completions/mean_length": 581.326171875,
-      "completions/mean_terminated_length": 581.326171875,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1949.0,
+      "completions/mean_length": 586.2265625,
+      "completions/mean_terminated_length": 583.365966796875,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.2059547794940676,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00490406522911546,
-      "kl": 0.0498046875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09080945802381934,
+      "kl": 0.0089111328125,
       "learning_rate": 3.1630150475258813e-06,
-      "loss": 0.0005,
-      "num_tokens": 138111952.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0121,
+      "num_tokens": 136620779.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 230
     },
     {
@@ -6455,20 +6455,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1587.0,
-      "completions/mean_length": 570.556640625,
-      "completions/mean_terminated_length": 566.9745483398438,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1531.0,
+      "completions/max_terminated_length": 1531.0,
+      "completions/mean_length": 538.849609375,
+      "completions/mean_terminated_length": 538.849609375,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.2068502350570853,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005622069756397748,
-      "kl": 0.052001953125,
+      "grad_norm": 0.003138487381120811,
+      "kl": 0.008453369140625,
       "learning_rate": 3.148679731053252e-06,
-      "loss": 0.0005,
-      "num_tokens": 138669325.0,
+      "loss": 0.0001,
+      "num_tokens": 137161918.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6483,20 +6483,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1477.0,
-      "completions/max_terminated_length": 1477.0,
-      "completions/mean_length": 561.345703125,
-      "completions/mean_terminated_length": 560.127197265625,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1240.0,
+      "completions/max_terminated_length": 1240.0,
+      "completions/mean_length": 542.169921875,
+      "completions/mean_terminated_length": 542.169921875,
+      "completions/min_length": 102.0,
+      "completions/min_terminated_length": 102.0,
       "epoch": 0.20774569062010298,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0061702858371463455,
-      "kl": 0.05303955078125,
+      "grad_norm": 0.009515406157466605,
+      "kl": 0.008758544921875,
       "learning_rate": 3.1343276867775805e-06,
-      "loss": 0.0005,
-      "num_tokens": 139245246.0,
+      "loss": 0.0001,
+      "num_tokens": 137728021.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6511,26 +6511,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1300.0,
-      "completions/max_terminated_length": 1300.0,
-      "completions/mean_length": 554.18359375,
-      "completions/mean_terminated_length": 553.1956787109375,
-      "completions/min_length": 161.0,
-      "completions/min_terminated_length": 161.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2033.0,
+      "completions/max_terminated_length": 2033.0,
+      "completions/mean_length": 542.384765625,
+      "completions/mean_terminated_length": 542.384765625,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.20864114618312066,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0076579380750235754,
-      "kl": 0.0537109375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0803866907591395,
+      "kl": 0.0088348388671875,
       "learning_rate": 3.1199595168819043e-06,
-      "loss": 0.0005,
-      "num_tokens": 139843420.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0006,
+      "num_tokens": 138320154.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 233
     },
     {
@@ -6539,26 +6539,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1842.0,
-      "completions/max_terminated_length": 1842.0,
-      "completions/mean_length": 567.6640625,
-      "completions/mean_terminated_length": 566.75537109375,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1453.0,
+      "completions/max_terminated_length": 1453.0,
+      "completions/mean_length": 555.943359375,
+      "completions/mean_terminated_length": 555.943359375,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.20953660174613833,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06004552744678529,
-      "kl": 0.0496826171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004165672775291565,
+      "kl": 0.009033203125,
       "learning_rate": 3.105575824225852e-06,
-      "loss": 0.005,
-      "num_tokens": 140447648.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 138918381.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 234
     },
     {
@@ -6569,24 +6569,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1984.0,
-      "completions/mean_length": 606.52734375,
-      "completions/mean_terminated_length": 600.8745727539062,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/max_terminated_length": 1522.0,
+      "completions/mean_length": 602.44921875,
+      "completions/mean_terminated_length": 596.7804565429688,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.21043205730915604,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08033506958705695,
-      "kl": 0.04827880859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003358640291167207,
+      "kl": 0.0087738037109375,
       "learning_rate": 3.091177212320363e-06,
-      "loss": 0.0086,
-      "num_tokens": 141079454.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 139548099.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 235
     },
     {
@@ -6596,25 +6596,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1724.0,
-      "completions/max_terminated_length": 1724.0,
-      "completions/mean_length": 544.689453125,
-      "completions/mean_terminated_length": 544.689453125,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/max_length": 1849.0,
+      "completions/max_terminated_length": 1849.0,
+      "completions/mean_length": 524.318359375,
+      "completions/mean_terminated_length": 524.318359375,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.21132751287217372,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07990215927891466,
-      "kl": 0.046142578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003609540305049046,
+      "kl": 0.0086212158203125,
       "learning_rate": 3.0767642853023538e-06,
-      "loss": 0.0005,
-      "num_tokens": 141634255.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 140092470.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 236
     },
     {
@@ -6624,19 +6624,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1361.0,
-      "completions/max_terminated_length": 1361.0,
-      "completions/mean_length": 563.033203125,
-      "completions/mean_terminated_length": 561.4774780273438,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/max_length": 1873.0,
+      "completions/max_terminated_length": 1873.0,
+      "completions/mean_length": 556.666015625,
+      "completions/mean_terminated_length": 554.7005615234375,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.2122229684351914,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006933071789708101,
-      "kl": 0.05389404296875,
+      "grad_norm": 316.8174705618427,
+      "kl": 65.50691223144531,
       "learning_rate": 3.062337647909376e-06,
-      "loss": 0.0005,
-      "num_tokens": 142211648.0,
+      "loss": 0.6582,
+      "num_tokens": 140666603.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6651,20 +6651,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1348.0,
-      "completions/max_terminated_length": 1348.0,
-      "completions/mean_length": 552.03515625,
-      "completions/mean_terminated_length": 552.03515625,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1543.0,
+      "completions/mean_length": 533.71875,
+      "completions/mean_terminated_length": 530.75537109375,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.2131184239982091,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005791270028062369,
-      "kl": 0.04583740234375,
+      "grad_norm": 0.004126053408821417,
+      "kl": 0.0089111328125,
       "learning_rate": 3.04789790545424e-06,
-      "loss": 0.0005,
-      "num_tokens": 142782642.0,
+      "loss": 0.0001,
+      "num_tokens": 141228219.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6679,20 +6679,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1458.0,
-      "completions/max_terminated_length": 1458.0,
-      "completions/mean_length": 572.533203125,
-      "completions/mean_terminated_length": 572.533203125,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1307.0,
+      "completions/mean_length": 571.94140625,
+      "completions/mean_terminated_length": 569.0528564453125,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.21401387956122678,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008004719272026003,
-      "kl": 0.0469970703125,
+      "grad_norm": 0.0031841456494166574,
+      "kl": 0.00885009765625,
       "learning_rate": 3.033445663799621e-06,
-      "loss": 0.0005,
-      "num_tokens": 143375619.0,
+      "loss": 0.0001,
+      "num_tokens": 141820893.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6707,20 +6707,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1575.0,
-      "completions/mean_length": 583.48046875,
-      "completions/mean_terminated_length": 580.614501953125,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1528.0,
+      "completions/max_terminated_length": 1528.0,
+      "completions/mean_length": 556.794921875,
+      "completions/mean_terminated_length": 556.794921875,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.21490933512424445,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004638129698711235,
-      "kl": 0.04461669921875,
+      "grad_norm": 0.005177262843237681,
+      "kl": 0.009063720703125,
       "learning_rate": 3.018981529332633e-06,
-      "loss": 0.0004,
-      "num_tokens": 143961561.0,
+      "loss": 0.0001,
+      "num_tokens": 142393172.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6736,19 +6736,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1391.0,
-      "completions/max_terminated_length": 1391.0,
-      "completions/mean_length": 585.458984375,
-      "completions/mean_terminated_length": 585.458984375,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/max_length": 1442.0,
+      "completions/max_terminated_length": 1442.0,
+      "completions/mean_length": 572.935546875,
+      "completions/mean_terminated_length": 572.935546875,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.21580479068726213,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.018662130861021326,
-      "kl": 0.04779052734375,
+      "grad_norm": 0.0034165096991114357,
+      "kl": 0.0080718994140625,
       "learning_rate": 3.00450610893939e-06,
-      "loss": 0.0005,
-      "num_tokens": 144583124.0,
+      "loss": 0.0001,
+      "num_tokens": 143008323.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6763,20 +6763,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1732.0,
-      "completions/max_terminated_length": 1732.0,
-      "completions/mean_length": 571.75390625,
-      "completions/mean_terminated_length": 571.75390625,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1551.0,
+      "completions/mean_length": 556.876953125,
+      "completions/mean_terminated_length": 553.9589233398438,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.21670024625027984,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00517530359345756,
-      "kl": 0.04498291015625,
+      "grad_norm": 0.0029792538892433463,
+      "kl": 0.0089874267578125,
       "learning_rate": 2.9900200099795396e-06,
-      "loss": 0.0005,
-      "num_tokens": 145184774.0,
+      "loss": 0.0001,
+      "num_tokens": 143602356.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6792,19 +6792,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1316.0,
-      "completions/max_terminated_length": 1316.0,
-      "completions/mean_length": 532.478515625,
-      "completions/mean_terminated_length": 532.478515625,
-      "completions/min_length": 145.0,
-      "completions/min_terminated_length": 145.0,
+      "completions/max_length": 1276.0,
+      "completions/max_terminated_length": 1276.0,
+      "completions/mean_length": 508.087890625,
+      "completions/mean_terminated_length": 508.087890625,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.21759570181329752,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005194283804810911,
-      "kl": 0.0467529296875,
+      "grad_norm": 0.0032021746534605607,
+      "kl": 0.008880615234375,
       "learning_rate": 2.9755238402607826e-06,
-      "loss": 0.0005,
-      "num_tokens": 145765275.0,
+      "loss": 0.0001,
+      "num_tokens": 144170369.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6819,20 +6819,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1867.0,
-      "completions/mean_length": 597.060546875,
-      "completions/mean_terminated_length": 594.2211303710938,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1538.0,
+      "completions/max_terminated_length": 1538.0,
+      "completions/mean_length": 586.673828125,
+      "completions/mean_terminated_length": 586.673828125,
+      "completions/min_length": 226.0,
+      "completions/min_terminated_length": 226.0,
       "epoch": 0.2184911573763152,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0654952222523936,
-      "kl": 0.0472412109375,
+      "grad_norm": 0.04216403891799333,
+      "kl": 0.0085296630859375,
       "learning_rate": 2.961018208013367e-06,
-      "loss": 0.0096,
-      "num_tokens": 146409306.0,
+      "loss": -0.0005,
+      "num_tokens": 144809082.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -6847,20 +6847,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1451.0,
-      "completions/mean_length": 598.9921875,
-      "completions/mean_terminated_length": 593.8074951171875,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/max_terminated_length": 1335.0,
+      "completions/mean_length": 601.25,
+      "completions/mean_terminated_length": 598.4187622070312,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.2193866129393329,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006328652943732272,
-      "kl": 0.059326171875,
+      "grad_norm": 0.004208175970862356,
+      "kl": 0.0084991455078125,
       "learning_rate": 2.9465037218645694e-06,
-      "loss": 0.0006,
-      "num_tokens": 147054854.0,
+      "loss": 0.0001,
+      "num_tokens": 145455786.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6876,19 +6876,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1251.0,
-      "completions/max_terminated_length": 1251.0,
-      "completions/mean_length": 579.97265625,
-      "completions/mean_terminated_length": 579.97265625,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/max_length": 1122.0,
+      "completions/max_terminated_length": 1122.0,
+      "completions/mean_length": 569.181640625,
+      "completions/mean_terminated_length": 569.181640625,
+      "completions/min_length": 134.0,
+      "completions/min_terminated_length": 134.0,
       "epoch": 0.22028206850235058,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005146548055592316,
-      "kl": 0.047607421875,
+      "grad_norm": 0.00319025018151349,
+      "kl": 0.0084075927734375,
       "learning_rate": 2.9319809908131604e-06,
-      "loss": 0.0005,
-      "num_tokens": 147642312.0,
+      "loss": 0.0001,
+      "num_tokens": 146037719.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6903,26 +6903,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1445.0,
-      "completions/max_terminated_length": 1445.0,
-      "completions/mean_length": 584.04296875,
-      "completions/mean_terminated_length": 583.6438598632812,
-      "completions/min_length": 254.0,
-      "completions/min_terminated_length": 254.0,
+      "completions/clipped_ratio": -6.9375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1360.0,
+      "completions/mean_length": 593.521484375,
+      "completions/mean_terminated_length": 584.3070678710938,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.22117752406536825,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005701587911010035,
-      "kl": 0.05059814453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.12997437289473063,
+      "kl": 0.015045166015625,
       "learning_rate": 2.917450624203847e-06,
-      "loss": 0.0005,
-      "num_tokens": 148250750.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0128,
+      "num_tokens": 146651010.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 247
     },
     {
@@ -6931,20 +6931,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1826.0,
-      "completions/mean_length": 580.466796875,
-      "completions/mean_terminated_length": 574.662109375,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1196.0,
+      "completions/max_terminated_length": 1196.0,
+      "completions/mean_length": 568.05859375,
+      "completions/mean_terminated_length": 568.05859375,
+      "completions/min_length": 224.0,
+      "completions/min_terminated_length": 224.0,
       "epoch": 0.22207297962838593,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005432057373877413,
-      "kl": 0.05230712890625,
+      "grad_norm": 0.002920667478560272,
+      "kl": 0.0084991455078125,
       "learning_rate": 2.9029132317017118e-06,
-      "loss": 0.0005,
-      "num_tokens": 148858237.0,
+      "loss": 0.0001,
+      "num_tokens": 147252144.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6959,20 +6959,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1624.0,
-      "completions/mean_length": 544.392578125,
-      "completions/mean_terminated_length": 541.4500732421875,
-      "completions/min_length": 158.0,
-      "completions/min_terminated_length": 158.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1365.0,
+      "completions/max_terminated_length": 1365.0,
+      "completions/mean_length": 551.31640625,
+      "completions/mean_terminated_length": 551.31640625,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.22296843519140364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004372778590201045,
-      "kl": 0.04522705078125,
+      "grad_norm": 0.0027478217190436602,
+      "kl": 0.00798797607421875,
       "learning_rate": 2.888369423266629e-06,
-      "loss": 0.0005,
-      "num_tokens": 149419590.0,
+      "loss": 0.0001,
+      "num_tokens": 147817042.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -6987,20 +6987,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1348.0,
-      "completions/max_terminated_length": 1348.0,
-      "completions/mean_length": 559.509765625,
-      "completions/mean_terminated_length": 558.8160400390625,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2025.0,
+      "completions/max_terminated_length": 2025.0,
+      "completions/mean_length": 561.966796875,
+      "completions/mean_terminated_length": 561.966796875,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.2238638907544213,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007646236721940514,
-      "kl": 0.05322265625,
+      "grad_norm": 0.005290429562560041,
+      "kl": 0.008819580078125,
       "learning_rate": 2.8738198091276712e-06,
-      "loss": 0.0005,
-      "num_tokens": 150017483.0,
+      "loss": 0.0001,
+      "num_tokens": 148416193.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7015,20 +7015,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1453.0,
-      "completions/mean_length": 543.091796875,
-      "completions/mean_terminated_length": 538.3988647460938,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1354.0,
+      "completions/max_terminated_length": 1354.0,
+      "completions/mean_length": 545.177734375,
+      "completions/mean_terminated_length": 545.177734375,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.224759346317439,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008486197284828401,
-      "kl": 0.05853271484375,
+      "grad_norm": 0.00344700409889045,
+      "kl": 0.008514404296875,
       "learning_rate": 2.859264999757509e-06,
-      "loss": 0.0006,
-      "num_tokens": 150561082.0,
+      "loss": 0.0001,
+      "num_tokens": 148960860.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7043,20 +7043,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2048.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1325.0,
       "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 582.26171875,
-      "completions/mean_terminated_length": 574.6023559570312,
-      "completions/min_length": 127.0,
-      "completions/min_terminated_length": 127.0,
+      "completions/mean_length": 575.69921875,
+      "completions/mean_terminated_length": 574.74755859375,
+      "completions/min_length": 164.0,
+      "completions/min_terminated_length": 164.0,
       "epoch": 0.2256548018804567,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07379730684287858,
-      "kl": 0.0567626953125,
+      "grad_norm": 0.09620477170433032,
+      "kl": 0.010467529296875,
       "learning_rate": 2.8447056058467928e-06,
-      "loss": 0.0042,
-      "num_tokens": 151185920.0,
+      "loss": 0.0059,
+      "num_tokens": 149582338.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -7071,26 +7071,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1381.0,
-      "completions/mean_length": 609.107421875,
-      "completions/mean_terminated_length": 598.6134033203125,
-      "completions/min_length": 152.0,
-      "completions/min_terminated_length": 152.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1466.0,
+      "completions/max_terminated_length": 1466.0,
+      "completions/mean_length": 609.81640625,
+      "completions/mean_terminated_length": 609.81640625,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.22655025744347437,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008824669605331369,
-      "kl": 0.0582275390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05824071665078531,
+      "kl": 0.0098114013671875,
       "learning_rate": 2.830142238278531e-06,
-      "loss": 0.0006,
-      "num_tokens": 151836199.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0021,
+      "num_tokens": 150232980.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 253
     },
     {
@@ -7101,18 +7101,18 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1344.0,
-      "completions/mean_length": 596.0546875,
-      "completions/mean_terminated_length": 593.2133178710938,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/max_terminated_length": 1494.0,
+      "completions/mean_length": 614.978515625,
+      "completions/mean_terminated_length": 612.1741333007812,
+      "completions/min_length": 255.0,
+      "completions/min_terminated_length": 255.0,
       "epoch": 0.22744571300649205,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00403469657058676,
-      "kl": 0.0469970703125,
+      "grad_norm": 0.002749795848165891,
+      "kl": 0.008148193359375,
       "learning_rate": 2.81557550810246e-06,
-      "loss": 0.0005,
-      "num_tokens": 152429587.0,
+      "loss": 0.0001,
+      "num_tokens": 150836057.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7127,26 +7127,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1476.0,
-      "completions/mean_length": 585.66015625,
-      "completions/mean_terminated_length": 580.1296997070312,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_terminated_length": 1787.0,
+      "completions/mean_length": 583.310546875,
+      "completions/mean_terminated_length": 580.4442138671875,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.22834116856950973,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010358310622908547,
-      "kl": 0.05926513671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05832611040542345,
+      "kl": 0.0085906982421875,
       "learning_rate": 2.8010060265094026e-06,
-      "loss": 0.0006,
-      "num_tokens": 153022197.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0012,
+      "num_tokens": 151427464.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 255
     },
     {
@@ -7155,20 +7155,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1802.0,
-      "completions/mean_length": 569.18359375,
-      "completions/mean_terminated_length": 563.3843383789062,
-      "completions/min_length": 150.0,
-      "completions/min_terminated_length": 150.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1373.0,
+      "completions/max_terminated_length": 1373.0,
+      "completions/mean_length": 559.775390625,
+      "completions/mean_terminated_length": 559.775390625,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.22923662413252743,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0050837400422739315,
-      "kl": 0.0462646484375,
+      "grad_norm": 0.0025223599801953577,
+      "kl": 0.00812530517578125,
       "learning_rate": 2.786434404805629e-06,
-      "loss": 0.0005,
-      "num_tokens": 153618579.0,
+      "loss": 0.0001,
+      "num_tokens": 152019029.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7183,20 +7183,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1237.0,
-      "completions/mean_length": 612.283203125,
-      "completions/mean_terminated_length": 608.6686401367188,
-      "completions/min_length": 247.0,
-      "completions/min_terminated_length": 247.0,
+      "completions/max_terminated_length": 1523.0,
+      "completions/mean_length": 614.580078125,
+      "completions/mean_terminated_length": 611.7749633789062,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
       "epoch": 0.2301320796955451,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004218948614674557,
-      "kl": 0.04693603515625,
+      "grad_norm": 0.0030536557032659947,
+      "kl": 0.0078582763671875,
       "learning_rate": 2.771861254387199e-06,
-      "loss": 0.0005,
-      "num_tokens": 154276452.0,
+      "loss": 0.0001,
+      "num_tokens": 152678078.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7211,20 +7211,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1304.0,
-      "completions/mean_length": 623.123046875,
-      "completions/mean_terminated_length": 620.3346557617188,
-      "completions/min_length": 262.0,
-      "completions/min_terminated_length": 262.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1291.0,
+      "completions/max_terminated_length": 1291.0,
+      "completions/mean_length": 615.45703125,
+      "completions/mean_terminated_length": 615.45703125,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
       "epoch": 0.2310275352585628,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0038371490056063186,
-      "kl": 0.044921875,
+      "grad_norm": 0.003043134978341531,
+      "kl": 0.0074005126953125,
       "learning_rate": 2.7572871867143204e-06,
-      "loss": 0.0004,
-      "num_tokens": 154894739.0,
+      "loss": 0.0001,
+      "num_tokens": 153292440.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7239,20 +7239,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1241.0,
-      "completions/mean_length": 578.615234375,
-      "completions/mean_terminated_length": 571.785888671875,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1704.0,
+      "completions/max_terminated_length": 1704.0,
+      "completions/mean_length": 576.3359375,
+      "completions/mean_terminated_length": 576.3359375,
+      "completions/min_length": 163.0,
+      "completions/min_terminated_length": 163.0,
       "epoch": 0.23192299082158047,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008145436520239484,
-      "kl": 0.06011962890625,
+      "grad_norm": 0.0028982682805773425,
+      "kl": 0.00792694091796875,
       "learning_rate": 2.742712813285681e-06,
-      "loss": 0.0006,
-      "num_tokens": 155517646.0,
+      "loss": 0.0001,
+      "num_tokens": 153914180.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7267,26 +7267,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1960.0,
-      "completions/mean_length": 626.619140625,
-      "completions/mean_terminated_length": 618.24169921875,
-      "completions/min_length": 221.0,
-      "completions/min_terminated_length": 221.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1893.0,
+      "completions/max_terminated_length": 1893.0,
+      "completions/mean_length": 612.654296875,
+      "completions/mean_terminated_length": 611.6085815429688,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.23281844638459817,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0689990067886915,
-      "kl": 0.055908203125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00965268256928715,
+      "kl": 0.0164031982421875,
       "learning_rate": 2.7281387456128017e-06,
-      "loss": 0.0071,
-      "num_tokens": 156187803.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 154577187.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 260
     },
     {
@@ -7295,20 +7295,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1619.0,
-      "completions/mean_length": 612.373046875,
-      "completions/mean_terminated_length": 603.91162109375,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_terminated_length": 1499.0,
+      "completions/mean_length": 597.28515625,
+      "completions/mean_terminated_length": 594.4461669921875,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.23371390194761585,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.1313821813786047,
-      "kl": 0.098388671875,
+      "grad_norm": 0.004089560262463787,
+      "kl": 0.0096588134765625,
       "learning_rate": 2.7135655951943716e-06,
-      "loss": 0.001,
-      "num_tokens": 156841610.0,
+      "loss": 0.0001,
+      "num_tokens": 155223269.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7323,20 +7323,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1338.0,
-      "completions/mean_length": 560.53515625,
-      "completions/mean_terminated_length": 547.2662963867188,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_terminated_length": 1608.0,
+      "completions/mean_length": 564.794921875,
+      "completions/mean_terminated_length": 561.892333984375,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.23460935751063353,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09702522439017101,
-      "kl": 0.05462646484375,
+      "grad_norm": 0.10228660877556683,
+      "kl": 0.0084991455078125,
       "learning_rate": 2.698993973490598e-06,
-      "loss": 0.015,
-      "num_tokens": 157469980.0,
+      "loss": 0.0131,
+      "num_tokens": 155853820.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -7351,20 +7351,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1909.0,
-      "completions/mean_length": 535.12109375,
-      "completions/mean_terminated_length": 522.0059204101562,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1835.0,
+      "completions/max_terminated_length": 1835.0,
+      "completions/mean_length": 549.693359375,
+      "completions/mean_terminated_length": 548.9823608398438,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.23550481307365123,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010862493459670133,
-      "kl": 0.06060791015625,
+      "grad_norm": 0.01584764131757789,
+      "kl": 0.0160675048828125,
       "learning_rate": 2.6844244918975416e-06,
-      "loss": 0.0006,
-      "num_tokens": 158024346.0,
+      "loss": 0.0002,
+      "num_tokens": 156415647.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7379,26 +7379,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1640.0,
-      "completions/mean_length": 558.44140625,
-      "completions/mean_terminated_length": 543.75146484375,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1293.0,
+      "completions/max_terminated_length": 1293.0,
+      "completions/mean_length": 568.513671875,
+      "completions/mean_terminated_length": 567.7240600585938,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.2364002686366689,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01007907413544423,
-      "kl": 0.0545654296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09690673980335059,
+      "kl": 0.022705078125,
       "learning_rate": 2.66985776172147e-06,
-      "loss": 0.0005,
-      "num_tokens": 158597404.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0004,
+      "num_tokens": 156993862.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 264
     },
     {
@@ -7407,20 +7407,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.859375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1474.0,
-      "completions/mean_length": 573.693359375,
-      "completions/mean_terminated_length": 548.4691772460938,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1643.0,
+      "completions/max_terminated_length": 1643.0,
+      "completions/mean_length": 576.32421875,
+      "completions/mean_terminated_length": 576.32421875,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
       "epoch": 0.2372957241996866,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009575151333470916,
-      "kl": 0.05621337890625,
+      "grad_norm": 0.004439510741313418,
+      "kl": 0.008209228515625,
       "learning_rate": 2.6552943941532088e-06,
-      "loss": 0.0006,
-      "num_tokens": 159207039.0,
+      "loss": 0.0001,
+      "num_tokens": 157604844.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7435,20 +7435,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.75,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1998.0,
-      "completions/mean_length": 573.662109375,
-      "completions/mean_terminated_length": 526.102783203125,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/max_terminated_length": 1738.0,
+      "completions/mean_length": 580.025390625,
+      "completions/mean_terminated_length": 577.1526489257812,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
       "epoch": 0.23819117976270426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.013645668908412388,
-      "kl": 0.06549072265625,
+      "grad_norm": 0.005051127249402352,
+      "kl": 0.00849151611328125,
       "learning_rate": 2.6407350002424927e-06,
-      "loss": 0.0007,
-      "num_tokens": 159824162.0,
+      "loss": 0.0001,
+      "num_tokens": 158225225.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7463,20 +7463,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.734375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1562.0,
-      "completions/mean_length": 574.169921875,
-      "completions/mean_terminated_length": 523.5535888671875,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_terminated_length": 1434.0,
+      "completions/mean_length": 570.8515625,
+      "completions/mean_terminated_length": 567.9608764648438,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.23908663532572197,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014855709612759045,
-      "kl": 0.0626220703125,
+      "grad_norm": 0.009859382079514532,
+      "kl": 0.0097503662109375,
       "learning_rate": 2.626180190872329e-06,
-      "loss": 0.0006,
-      "num_tokens": 160402873.0,
+      "loss": 0.0001,
+      "num_tokens": 158802237.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7491,26 +7491,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.828125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1247.0,
-      "completions/mean_length": 545.73828125,
-      "completions/mean_terminated_length": 512.7545166015625,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1336.0,
+      "completions/max_terminated_length": 1336.0,
+      "completions/mean_length": 566.79296875,
+      "completions/mean_terminated_length": 566.79296875,
+      "completions/min_length": 231.0,
+      "completions/min_terminated_length": 231.0,
       "epoch": 0.23998209088873965,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.052896967313257874,
-      "kl": 0.06146240234375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004277627243588492,
+      "kl": 0.00750732421875,
       "learning_rate": 2.611630576733372e-06,
-      "loss": -0.0025,
-      "num_tokens": 160994419.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 159404563.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 268
     },
     {
@@ -7519,20 +7519,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1549.0,
-      "completions/mean_length": 539.318359375,
-      "completions/mean_terminated_length": 505.3060302734375,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1528.0,
+      "completions/max_terminated_length": 1528.0,
+      "completions/mean_length": 564.849609375,
+      "completions/mean_terminated_length": 564.849609375,
+      "completions/min_length": 200.0,
+      "completions/min_terminated_length": 200.0,
       "epoch": 0.24087754645175732,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011827177223144848,
-      "kl": 0.0672607421875,
+      "grad_norm": 0.004866099932917146,
+      "kl": 0.0077972412109375,
       "learning_rate": 2.5970867682982885e-06,
-      "loss": 0.0007,
-      "num_tokens": 161569638.0,
+      "loss": 0.0001,
+      "num_tokens": 159992854.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7547,26 +7547,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.78125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1363.0,
-      "completions/mean_length": 550.271484375,
-      "completions/mean_terminated_length": 508.1666564941406,
-      "completions/min_length": 102.0,
-      "completions/min_terminated_length": 102.0,
+      "completions/max_terminated_length": 1802.0,
+      "completions/mean_length": 570.490234375,
+      "completions/mean_terminated_length": 567.5988159179688,
+      "completions/min_length": 84.0,
+      "completions/min_terminated_length": 84.0,
       "epoch": 0.24177300201477503,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009620558992138992,
-      "kl": 0.06439208984375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05839995382013476,
+      "kl": 0.008270263671875,
       "learning_rate": 2.582549375796154e-06,
-      "loss": 0.0006,
-      "num_tokens": 162189217.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0129,
+      "num_tokens": 160622785.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 270
     },
     {
@@ -7575,20 +7575,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1735.0,
-      "completions/mean_length": 564.26953125,
-      "completions/mean_terminated_length": 528.6600341796875,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/max_terminated_length": 1279.0,
+      "completions/mean_length": 582.490234375,
+      "completions/mean_terminated_length": 579.622314453125,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.2426684575777927,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011339660484190167,
-      "kl": 0.06573486328125,
+      "grad_norm": 0.0035678064526539688,
+      "kl": 0.008392333984375,
       "learning_rate": 2.568019009186841e-06,
-      "loss": 0.0007,
-      "num_tokens": 162740027.0,
+      "loss": 0.0001,
+      "num_tokens": 161182924.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7603,20 +7603,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1700.0,
-      "completions/mean_length": 513.61328125,
-      "completions/mean_terminated_length": 483.0478210449219,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1361.0,
+      "completions/max_terminated_length": 1361.0,
+      "completions/mean_length": 525.4921875,
+      "completions/mean_terminated_length": 525.4921875,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.24356391314081038,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009385702347993711,
-      "kl": 0.0584716796875,
+      "grad_norm": 0.0031233149161186808,
+      "kl": 0.008209228515625,
       "learning_rate": 2.5534962781354317e-06,
-      "loss": 0.0006,
-      "num_tokens": 163306197.0,
+      "loss": 0.0001,
+      "num_tokens": 161755176.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7631,26 +7631,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.859375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1686.0,
-      "completions/mean_length": 552.63671875,
-      "completions/mean_terminated_length": 525.8806762695312,
-      "completions/min_length": 139.0,
-      "completions/min_terminated_length": 139.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1826.0,
+      "completions/max_terminated_length": 1826.0,
+      "completions/mean_length": 568.16796875,
+      "completions/mean_terminated_length": 568.16796875,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.24445936870382806,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08002624358034882,
-      "kl": 0.0601806640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0034764954757281943,
+      "kl": 0.0081634521484375,
       "learning_rate": 2.538981791986634e-06,
-      "loss": 0.0008,
-      "num_tokens": 163885371.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 162342302.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 273
     },
     {
@@ -7659,26 +7659,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.8125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1331.0,
-      "completions/mean_length": 552.349609375,
-      "completions/mean_terminated_length": 516.4540405273438,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1273.0,
+      "completions/max_terminated_length": 1273.0,
+      "completions/mean_length": 580.52734375,
+      "completions/mean_terminated_length": 580.52734375,
+      "completions/min_length": 141.0,
+      "completions/min_terminated_length": 141.0,
       "epoch": 0.24535482426684577,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06166515634300674,
-      "kl": 0.073974609375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0028758465684256365,
+      "kl": 0.008209228515625,
       "learning_rate": 2.524476159739218e-06,
-      "loss": 0.0006,
-      "num_tokens": 164493006.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 162964364.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 274
     },
     {
@@ -7687,20 +7687,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1314.0,
-      "completions/mean_length": 547.078125,
-      "completions/mean_terminated_length": 526.2732543945312,
-      "completions/min_length": 124.0,
-      "completions/min_terminated_length": 124.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1399.0,
+      "completions/max_terminated_length": 1399.0,
+      "completions/mean_length": 575.3125,
+      "completions/mean_terminated_length": 575.3125,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
       "epoch": 0.24625027982986344,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01257928535443048,
-      "kl": 0.0628662109375,
+      "grad_norm": 0.0039951419743716335,
+      "kl": 0.0081787109375,
       "learning_rate": 2.5099799900204607e-06,
-      "loss": 0.0006,
-      "num_tokens": 165077334.0,
+      "loss": 0.0001,
+      "num_tokens": 163563148.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7715,20 +7715,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1098.0,
-      "completions/mean_length": 537.591796875,
-      "completions/mean_terminated_length": 531.6686401367188,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1598.0,
+      "completions/max_terminated_length": 1598.0,
+      "completions/mean_length": 605.78125,
+      "completions/mean_terminated_length": 605.78125,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.24714573539288112,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008330623143216287,
-      "kl": 0.0562744140625,
+      "grad_norm": 0.0027137761723457256,
+      "kl": 0.00742340087890625,
       "learning_rate": 2.4954938910606108e-06,
-      "loss": 0.0006,
-      "num_tokens": 165627301.0,
+      "loss": 0.0001,
+      "num_tokens": 164148028.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7743,26 +7743,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1601.0,
-      "completions/mean_length": 513.478515625,
-      "completions/mean_terminated_length": 507.4608154296875,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1356.0,
+      "completions/max_terminated_length": 1356.0,
+      "completions/mean_length": 540.462890625,
+      "completions/mean_terminated_length": 540.462890625,
+      "completions/min_length": 103.0,
+      "completions/min_terminated_length": 103.0,
       "epoch": 0.24804119095589883,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08070272275220955,
-      "kl": 0.0557861328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0036590336323787287,
+      "kl": 0.008209228515625,
       "learning_rate": 2.481018470667368e-06,
-      "loss": 0.0006,
-      "num_tokens": 166173162.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 164707705.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 277
     },
     {
@@ -7771,20 +7771,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1408.0,
-      "completions/mean_length": 528.388671875,
-      "completions/mean_terminated_length": 522.429443359375,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1390.0,
+      "completions/max_terminated_length": 1390.0,
+      "completions/mean_length": 572.0390625,
+      "completions/mean_terminated_length": 572.0390625,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.2489366465189165,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006969320341412188,
-      "kl": 0.05340576171875,
+      "grad_norm": 0.002865287351534278,
+      "kl": 0.00798797607421875,
       "learning_rate": 2.4665543362003802e-06,
-      "loss": 0.0005,
-      "num_tokens": 166727345.0,
+      "loss": 0.0001,
+      "num_tokens": 165284237.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7799,20 +7799,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1269.0,
-      "completions/max_terminated_length": 1269.0,
-      "completions/mean_length": 540.447265625,
-      "completions/mean_terminated_length": 539.8199462890625,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1581.0,
+      "completions/max_terminated_length": 1581.0,
+      "completions/mean_length": 576.732421875,
+      "completions/mean_terminated_length": 576.732421875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.24983210208193418,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007011045285617038,
-      "kl": 0.05621337890625,
+      "grad_norm": 0.0026953753539202627,
+      "kl": 0.0083465576171875,
       "learning_rate": 2.4521020945457615e-06,
-      "loss": 0.0006,
-      "num_tokens": 167316710.0,
+      "loss": 0.0001,
+      "num_tokens": 165892180.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7827,26 +7827,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1439.0,
-      "completions/mean_length": 534.302734375,
-      "completions/mean_terminated_length": 531.3405151367188,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1806.0,
+      "completions/max_terminated_length": 1806.0,
+      "completions/mean_length": 572.642578125,
+      "completions/mean_terminated_length": 572.642578125,
+      "completions/min_length": 166.0,
+      "completions/min_terminated_length": 166.0,
       "epoch": 0.25072755764495186,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07799875756079883,
-      "kl": 0.0548095703125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06153457584936372,
+      "kl": 0.007965087890625,
       "learning_rate": 2.4376623520906255e-06,
-      "loss": -0.0067,
-      "num_tokens": 167907169.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": -0.0001,
+      "num_tokens": 166502269.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 280
     },
     {
@@ -7856,25 +7856,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1438.0,
-      "completions/max_terminated_length": 1438.0,
-      "completions/mean_length": 576.05078125,
-      "completions/mean_terminated_length": 576.05078125,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/max_length": 1791.0,
+      "completions/max_terminated_length": 1791.0,
+      "completions/mean_length": 618.21875,
+      "completions/mean_terminated_length": 618.21875,
+      "completions/min_length": 155.0,
+      "completions/min_terminated_length": 155.0,
       "epoch": 0.25162301320796954,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04411135965167851,
-      "kl": 0.05206298828125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0024592397394463637,
+      "kl": 0.0079193115234375,
       "learning_rate": 2.4232357146976478e-06,
-      "loss": -0.0027,
-      "num_tokens": 168517275.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 167133965.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 281
     },
     {
@@ -7884,19 +7884,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1842.0,
-      "completions/max_terminated_length": 1842.0,
-      "completions/mean_length": 582.052734375,
-      "completions/mean_terminated_length": 582.052734375,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/max_length": 1651.0,
+      "completions/max_terminated_length": 1651.0,
+      "completions/mean_length": 600.8828125,
+      "completions/mean_terminated_length": 600.8828125,
+      "completions/min_length": 219.0,
+      "completions/min_terminated_length": 219.0,
       "epoch": 0.2525184687709872,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006361764340800594,
-      "kl": 0.0521240234375,
+      "grad_norm": 0.003365096936430135,
+      "kl": 0.0074310302734375,
       "learning_rate": 2.408822787679637e-06,
-      "loss": 0.0005,
-      "num_tokens": 169103398.0,
+      "loss": 0.0001,
+      "num_tokens": 167729729.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7911,20 +7911,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1212.0,
-      "completions/mean_length": 554.568359375,
-      "completions/mean_terminated_length": 548.7117919921875,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1282.0,
+      "completions/max_terminated_length": 1282.0,
+      "completions/mean_length": 551.373046875,
+      "completions/mean_terminated_length": 551.373046875,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.25341392433400495,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.09731232801458174,
-      "kl": 0.0823974609375,
+      "grad_norm": 0.0031736404722537124,
+      "kl": 0.0082550048828125,
       "learning_rate": 2.3944241757741475e-06,
-      "loss": 0.0008,
-      "num_tokens": 169684473.0,
+      "loss": 0.0001,
+      "num_tokens": 168309168.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -7940,19 +7940,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1220.0,
-      "completions/max_terminated_length": 1220.0,
-      "completions/mean_length": 586.662109375,
-      "completions/mean_terminated_length": 586.662109375,
-      "completions/min_length": 162.0,
-      "completions/min_terminated_length": 162.0,
+      "completions/max_length": 1599.0,
+      "completions/max_terminated_length": 1599.0,
+      "completions/mean_length": 590.669921875,
+      "completions/mean_terminated_length": 590.669921875,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.2543093798970226,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06020667119192139,
-      "kl": 0.05108642578125,
+      "grad_norm": 0.0567955043559705,
+      "kl": 0.0079345703125,
       "learning_rate": 2.380040483118097e-06,
-      "loss": -0.0002,
-      "num_tokens": 170264460.0,
+      "loss": 0.0004,
+      "num_tokens": 168891207.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -7967,26 +7967,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1498.0,
-      "completions/mean_length": 568.12890625,
-      "completions/mean_terminated_length": 565.2328491210938,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1773.0,
+      "completions/max_terminated_length": 1773.0,
+      "completions/mean_length": 569.77734375,
+      "completions/mean_terminated_length": 569.77734375,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.2552048354600403,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09146996211829635,
-      "kl": 0.05084228515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0030194281648096032,
+      "kl": 0.00792694091796875,
       "learning_rate": 2.365672313222419e-06,
-      "loss": 0.0006,
-      "num_tokens": 170859550.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 169487141.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 285
     },
     {
@@ -7995,26 +7995,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1467.0,
-      "completions/mean_length": 622.955078125,
-      "completions/mean_terminated_length": 617.36669921875,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1905.0,
+      "completions/max_terminated_length": 1905.0,
+      "completions/mean_length": 615.369140625,
+      "completions/mean_terminated_length": 615.369140625,
+      "completions/min_length": 240.0,
+      "completions/min_terminated_length": 240.0,
       "epoch": 0.256100291023058,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.048868905799068094,
-      "kl": 0.0537109375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003087403099165956,
+      "kl": 0.008270263671875,
       "learning_rate": 2.351320268946749e-06,
-      "loss": -0.0049,
-      "num_tokens": 171513463.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 170137170.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 286
     },
     {
@@ -8023,26 +8023,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1393.0,
-      "completions/mean_length": 545.109375,
-      "completions/mean_terminated_length": 538.8231811523438,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1735.0,
+      "completions/max_terminated_length": 1735.0,
+      "completions/mean_length": 505.8828125,
+      "completions/mean_terminated_length": 505.8828125,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.25699574658607566,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0656092790599532,
-      "kl": 0.0565185546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00622764953467058,
+      "kl": 0.009307861328125,
       "learning_rate": 2.336984952474119e-06,
-      "loss": 0.0027,
-      "num_tokens": 172059935.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 170663558.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 287
     },
     {
@@ -8051,20 +8051,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1208.0,
-      "completions/mean_length": 561.330078125,
-      "completions/mean_terminated_length": 555.5000610351562,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/max_terminated_length": 1442.0,
+      "completions/mean_length": 529.994140625,
+      "completions/mean_terminated_length": 527.0234985351562,
+      "completions/min_length": 157.0,
+      "completions/min_terminated_length": 157.0,
       "epoch": 0.25789120214909333,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007045410253853639,
-      "kl": 0.053955078125,
+      "grad_norm": 0.0029162163258119552,
+      "kl": 0.00823974609375,
       "learning_rate": 2.322666965285697e-06,
-      "loss": 0.0005,
-      "num_tokens": 172613208.0,
+      "loss": 0.0001,
+      "num_tokens": 171200787.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8079,20 +8079,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1763.0,
-      "completions/mean_length": 595.86328125,
-      "completions/mean_terminated_length": 590.1686401367188,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1782.0,
+      "completions/max_terminated_length": 1782.0,
+      "completions/mean_length": 551.056640625,
+      "completions/mean_terminated_length": 551.056640625,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.258786657712111,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005338478822253961,
-      "kl": 0.0498046875,
+      "grad_norm": 0.002784819528636807,
+      "kl": 0.008056640625,
       "learning_rate": 2.3083669081355507e-06,
-      "loss": 0.0005,
-      "num_tokens": 173226114.0,
+      "loss": 0.0001,
+      "num_tokens": 171790752.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8107,20 +8107,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1233.0,
-      "completions/mean_length": 578.037109375,
-      "completions/mean_terminated_length": 562.0119018554688,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/max_terminated_length": 1225.0,
+      "completions/mean_length": 544.69140625,
+      "completions/mean_terminated_length": 541.74951171875,
+      "completions/min_length": 139.0,
+      "completions/min_terminated_length": 139.0,
       "epoch": 0.25968211327512875,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.014065273768047462,
-      "kl": 0.06695556640625,
+      "grad_norm": 0.0033671102355632226,
+      "kl": 0.0086822509765625,
       "learning_rate": 2.2940853810254377e-06,
-      "loss": 0.0007,
-      "num_tokens": 173819477.0,
+      "loss": 0.0001,
+      "num_tokens": 172367042.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8135,20 +8135,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1943.0,
-      "completions/mean_length": 612.31640625,
-      "completions/mean_terminated_length": 603.8546142578125,
-      "completions/min_length": 244.0,
-      "completions/min_terminated_length": 244.0,
+      "completions/max_terminated_length": 1565.0,
+      "completions/mean_length": 566.361328125,
+      "completions/mean_terminated_length": 563.4618530273438,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.2605775688381464,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005526260451771069,
-      "kl": 0.049072265625,
+      "grad_norm": 0.0026251951857379574,
+      "kl": 0.0079498291015625,
       "learning_rate": 2.2798229831796313e-06,
-      "loss": 0.0005,
-      "num_tokens": 174392967.0,
+      "loss": 0.0001,
+      "num_tokens": 172917003.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8163,20 +8163,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1816.0,
-      "completions/mean_length": 638.041015625,
-      "completions/mean_terminated_length": 621.7806396484375,
-      "completions/min_length": 224.0,
-      "completions/min_terminated_length": 224.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2025.0,
+      "completions/max_terminated_length": 2025.0,
+      "completions/mean_length": 613.81640625,
+      "completions/mean_terminated_length": 613.81640625,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.2614730244011641,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006603994997144677,
-      "kl": 0.05377197265625,
+      "grad_norm": 0.0027979065409168315,
+      "kl": 0.00760650634765625,
       "learning_rate": 2.2655803130197816e-06,
-      "loss": 0.0005,
-      "num_tokens": 175013868.0,
+      "loss": 0.0001,
+      "num_tokens": 173525501.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8191,26 +8191,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1967.0,
-      "completions/mean_length": 608.197265625,
-      "completions/mean_terminated_length": 579.5159301757812,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_terminated_length": 1287.0,
+      "completions/mean_length": 543.486328125,
+      "completions/mean_terminated_length": 540.5420532226562,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.2623684799641818,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07767449708337147,
-      "kl": 0.06494140625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002850882049670849,
+      "kl": 0.00788116455078125,
       "learning_rate": 2.2513579681398034e-06,
-      "loss": 0.0111,
-      "num_tokens": 175615297.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 174093798.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 293
     },
     {
@@ -8219,20 +8219,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.796875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1844.0,
-      "completions/mean_length": 590.447265625,
-      "completions/mean_terminated_length": 552.4749755859375,
-      "completions/min_length": 173.0,
-      "completions/min_terminated_length": 173.0,
+      "completions/max_terminated_length": 1779.0,
+      "completions/mean_length": 519.80859375,
+      "completions/mean_terminated_length": 516.8179931640625,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.26326393552719946,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007266034489754845,
-      "kl": 0.05670166015625,
+      "grad_norm": 0.002895038223420061,
+      "kl": 0.00860595703125,
       "learning_rate": 2.237156545280803e-06,
-      "loss": 0.0006,
-      "num_tokens": 176180822.0,
+      "loss": 0.0001,
+      "num_tokens": 174623156.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8247,20 +8247,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1821.0,
-      "completions/mean_length": 599.375,
-      "completions/mean_terminated_length": 593.6941528320312,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1346.0,
+      "completions/max_terminated_length": 1346.0,
+      "completions/mean_length": 556.666015625,
+      "completions/mean_terminated_length": 556.666015625,
+      "completions/min_length": 174.0,
+      "completions/min_terminated_length": 174.0,
       "epoch": 0.26415939109021713,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005244228890614103,
-      "kl": 0.05059814453125,
+      "grad_norm": 0.0031545665495167205,
+      "kl": 0.0075531005859375,
       "learning_rate": 2.2229766403060403e-06,
-      "loss": 0.0005,
-      "num_tokens": 176775718.0,
+      "loss": 0.0001,
+      "num_tokens": 175196185.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8275,20 +8275,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1598.0,
-      "completions/mean_length": 619.388671875,
-      "completions/mean_terminated_length": 603.662109375,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1574.0,
+      "completions/max_terminated_length": 1574.0,
+      "completions/mean_length": 580.091796875,
+      "completions/mean_terminated_length": 580.091796875,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.2650548466532348,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0067364873550132195,
-      "kl": 0.05413818359375,
+      "grad_norm": 0.0025604974218733523,
+      "kl": 0.00716400146484375,
       "learning_rate": 2.2088188481759305e-06,
-      "loss": 0.0005,
-      "num_tokens": 177382541.0,
+      "loss": 0.0001,
+      "num_tokens": 175782888.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8303,26 +8303,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.84375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1749.0,
-      "completions/mean_length": 633.080078125,
-      "completions/mean_terminated_length": 604.8944702148438,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_terminated_length": 1464.0,
+      "completions/mean_length": 581.8515625,
+      "completions/mean_terminated_length": 578.9823608398438,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.26595030221625254,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08346497425582332,
-      "kl": 0.05059814453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002912348412355826,
+      "kl": 0.0078277587890625,
       "learning_rate": 2.194683762923073e-06,
-      "loss": 0.0129,
-      "num_tokens": 178022646.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 176396764.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 297
     },
     {
@@ -8331,20 +8331,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1680.0,
-      "completions/mean_length": 578.904296875,
-      "completions/mean_terminated_length": 567.3366088867188,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/max_terminated_length": 1329.0,
+      "completions/mean_length": 543.87109375,
+      "completions/mean_terminated_length": 540.9276123046875,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.2668457577792702,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0606077670201604,
-      "kl": 0.0494384765625,
+      "grad_norm": 0.07647397750323286,
+      "kl": 0.0075836181640625,
       "learning_rate": 2.1805719776273387e-06,
-      "loss": 0.0148,
-      "num_tokens": 178601237.0,
+      "loss": 0.0137,
+      "num_tokens": 176957418.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -8359,26 +8359,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2019.0,
-      "completions/mean_length": 631.0078125,
-      "completions/mean_terminated_length": 614.20556640625,
-      "completions/min_length": 200.0,
-      "completions/min_terminated_length": 200.0,
+      "completions/max_terminated_length": 1864.0,
+      "completions/mean_length": 584.349609375,
+      "completions/mean_terminated_length": 581.4852905273438,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.2677412133422879,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005602309576249166,
-      "kl": 0.049072265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0832525767775582,
+      "kl": 0.0071868896484375,
       "learning_rate": 2.166484084390974e-06,
-      "loss": 0.0005,
-      "num_tokens": 179219561.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0082,
+      "num_tokens": 177551853.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 299
     },
     {
@@ -8387,20 +8387,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2040.0,
-      "completions/mean_length": 597.46875,
-      "completions/mean_terminated_length": 577.3782348632812,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1175.0,
+      "completions/max_terminated_length": 1175.0,
+      "completions/mean_length": 572.251953125,
+      "completions/mean_terminated_length": 572.251953125,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.2686366689053056,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007717796395967091,
-      "kl": 0.05401611328125,
+      "grad_norm": 0.0024824890095177894,
+      "kl": 0.0075225830078125,
       "learning_rate": 2.1524206743137636e-06,
-      "loss": 0.0005,
-      "num_tokens": 179815625.0,
+      "loss": 0.0001,
+      "num_tokens": 178135006.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8415,26 +8415,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1439.0,
-      "completions/mean_length": 618.8203125,
-      "completions/mean_terminated_length": 601.87353515625,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1303.0,
+      "completions/max_terminated_length": 1303.0,
+      "completions/mean_length": 590.498046875,
+      "completions/mean_terminated_length": 590.498046875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.26953212446832325,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00549947178848661,
-      "kl": 0.04949951171875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06156528604399248,
+      "kl": 0.00756072998046875,
       "learning_rate": 2.1383823374682287e-06,
-      "loss": 0.0005,
-      "num_tokens": 180447117.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0006,
+      "num_tokens": 178751997.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 301
     },
     {
@@ -8443,26 +8443,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1992.0,
-      "completions/mean_length": 599.6796875,
-      "completions/mean_terminated_length": 581.9069213867188,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1633.0,
+      "completions/max_terminated_length": 1633.0,
+      "completions/mean_length": 574.0390625,
+      "completions/mean_terminated_length": 574.0390625,
+      "completions/min_length": 144.0,
+      "completions/min_terminated_length": 144.0,
       "epoch": 0.27042758003134093,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07205405556316538,
-      "kl": 0.05169677734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.009728130461388736,
+      "kl": 0.00785064697265625,
       "learning_rate": 2.124369662874868e-06,
-      "loss": 0.0029,
-      "num_tokens": 181059241.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 179350993.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 302
     },
     {
@@ -8471,20 +8471,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1763.0,
-      "completions/mean_length": 581.98828125,
-      "completions/mean_terminated_length": 570.4448852539062,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1401.0,
+      "completions/max_terminated_length": 1401.0,
+      "completions/mean_length": 558.1328125,
+      "completions/mean_terminated_length": 558.1328125,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.2713230355943586,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005610020156050893,
-      "kl": 0.04766845703125,
+      "grad_norm": 0.00216036159896381,
+      "kl": 0.00812530517578125,
       "learning_rate": 2.110383238477441e-06,
-      "loss": 0.0005,
-      "num_tokens": 181622851.0,
+      "loss": 0.0001,
+      "num_tokens": 179902389.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8499,20 +8499,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.859375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1266.0,
-      "completions/mean_length": 643.609375,
-      "completions/mean_terminated_length": 620.0357666015625,
-      "completions/min_length": 206.0,
-      "completions/min_terminated_length": 206.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1786.0,
+      "completions/max_terminated_length": 1786.0,
+      "completions/mean_length": 616.208984375,
+      "completions/mean_terminated_length": 616.208984375,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.27221849115737634,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0068354809798462845,
-      "kl": 0.0518798828125,
+      "grad_norm": 0.0030452568956129706,
+      "kl": 0.0074005126953125,
       "learning_rate": 2.096423651118305e-06,
-      "loss": 0.0005,
-      "num_tokens": 182268715.0,
+      "loss": 0.0001,
+      "num_tokens": 180534224.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8527,20 +8527,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1517.0,
-      "completions/mean_length": 582.166015625,
-      "completions/mean_terminated_length": 575.3182983398438,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1252.0,
+      "completions/max_terminated_length": 1252.0,
+      "completions/mean_length": 563.119140625,
+      "completions/mean_terminated_length": 563.119140625,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.273113946720394,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007411219035362688,
-      "kl": 0.05291748046875,
+      "grad_norm": 0.003442268008121624,
+      "kl": 0.00763702392578125,
       "learning_rate": 2.082491486513788e-06,
-      "loss": 0.0005,
-      "num_tokens": 182850160.0,
+      "loss": 0.0001,
+      "num_tokens": 181105917.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8555,26 +8555,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1792.0,
-      "completions/mean_length": 579.1015625,
-      "completions/mean_terminated_length": 563.873291015625,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1777.0,
+      "completions/max_terminated_length": 1777.0,
+      "completions/mean_length": 569.068359375,
+      "completions/mean_terminated_length": 569.068359375,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
       "epoch": 0.2740094022834117,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009215146072812768,
-      "kl": 0.05816650390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05941448759219727,
+      "kl": 0.008209228515625,
       "learning_rate": 2.0685873292296116e-06,
-      "loss": 0.0006,
-      "num_tokens": 183423268.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0006,
+      "num_tokens": 181673888.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 306
     },
     {
@@ -8583,20 +8583,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1401.0,
-      "completions/mean_length": 587.12890625,
-      "completions/mean_terminated_length": 569.2713012695312,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/max_terminated_length": 1471.0,
+      "completions/mean_length": 573.65625,
+      "completions/mean_terminated_length": 570.7710571289062,
+      "completions/min_length": 160.0,
+      "completions/min_terminated_length": 160.0,
       "epoch": 0.2749048578464294,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.016584960558114703,
-      "kl": 0.0501708984375,
+      "grad_norm": 0.005439544093753847,
+      "kl": 0.00799560546875,
       "learning_rate": 2.054711762656369e-06,
-      "loss": 0.0005,
-      "num_tokens": 184034310.0,
+      "loss": 0.0001,
+      "num_tokens": 182278032.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8611,26 +8611,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.921875,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1832.0,
-      "completions/mean_length": 593.46484375,
-      "completions/mean_terminated_length": 581.0433959960938,
-      "completions/min_length": 248.0,
-      "completions/min_terminated_length": 248.0,
+      "completions/max_terminated_length": 1634.0,
+      "completions/mean_length": 588.927734375,
+      "completions/mean_terminated_length": 586.0723876953125,
+      "completions/min_length": 199.0,
+      "completions/min_terminated_length": 199.0,
       "epoch": 0.27580031340944705,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011959138514659325,
-      "kl": 0.0601806640625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07732112935482617,
+      "kl": 0.0078582763671875,
       "learning_rate": 2.040865368985044e-06,
-      "loss": 0.0006,
-      "num_tokens": 184622052.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0149,
+      "num_tokens": 182863451.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 308
     },
     {
@@ -8639,20 +8639,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1662.0,
-      "completions/mean_length": 651.04296875,
-      "completions/mean_terminated_length": 640.0433349609375,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1832.0,
+      "completions/max_terminated_length": 1832.0,
+      "completions/mean_length": 638.77734375,
+      "completions/mean_terminated_length": 638.77734375,
+      "completions/min_length": 159.0,
+      "completions/min_terminated_length": 159.0,
       "epoch": 0.27669576897246473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004220974278526902,
-      "kl": 0.04766845703125,
+      "grad_norm": 0.0027031381335453617,
+      "kl": 0.00719451904296875,
       "learning_rate": 2.027048729182583e-06,
-      "loss": 0.0005,
-      "num_tokens": 185266378.0,
+      "loss": 0.0001,
+      "num_tokens": 183501497.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8667,26 +8667,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1602.0,
-      "completions/mean_length": 609.353515625,
-      "completions/mean_terminated_length": 600.874267578125,
-      "completions/min_length": 239.0,
-      "completions/min_terminated_length": 239.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1680.0,
+      "completions/max_terminated_length": 1680.0,
+      "completions/mean_length": 607.361328125,
+      "completions/mean_terminated_length": 607.361328125,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.2775912245354824,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05299649935908705,
-      "kl": 0.05035400390625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00369268777417457,
+      "kl": 0.0084075927734375,
       "learning_rate": 2.0132624229675205e-06,
-      "loss": -0.0005,
-      "num_tokens": 185915055.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 184149154.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 310
     },
     {
@@ -8695,54 +8695,54 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.890625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 2016.0,
-      "completions/mean_length": 600.80859375,
-      "completions/mean_terminated_length": 587.051513671875,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1241.0,
+      "completions/max_terminated_length": 1241.0,
+      "completions/mean_length": 571.37109375,
+      "completions/mean_terminated_length": 571.37109375,
+      "completions/min_length": 217.0,
+      "completions/min_terminated_length": 217.0,
       "epoch": 0.27848668009850014,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008478364104235272,
-      "kl": 0.0601806640625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08066837486158972,
+      "kl": 0.0079193115234375,
       "learning_rate": 1.9995070287856546e-06,
-      "loss": 0.0006,
-      "num_tokens": 186510029.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0064,
+      "num_tokens": 184729056.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 311
     },
     {
       "clip_ratio/high_max": 0.0,
       "clip_ratio/high_mean": 0.0,
       "clip_ratio/low_mean": 0.0,
-      "clip_ratio/low_min": 0.0,
-      "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.765625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1390.0,
-      "completions/mean_length": 614.634765625,
-      "completions/mean_terminated_length": 571.3742065429688,
-      "completions/min_length": 234.0,
-      "completions/min_terminated_length": 234.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1238.0,
+      "completions/max_terminated_length": 1238.0,
+      "completions/mean_length": 571.208984375,
+      "completions/mean_terminated_length": 571.208984375,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.2793821356615178,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.14229366676561211,
-      "kl": 0.052978515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002942664647734796,
+      "kl": 0.0076904296875,
       "learning_rate": 1.985783123785774e-06,
-      "loss": 0.0146,
-      "num_tokens": 187108514.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 185305307.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 312
     },
     {
@@ -8751,26 +8751,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1280.0,
-      "completions/mean_length": 570.029296875,
-      "completions/mean_terminated_length": 565.2745361328125,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1427.0,
+      "completions/max_terminated_length": 1427.0,
+      "completions/mean_length": 561.5,
+      "completions/mean_terminated_length": 560.5322875976562,
+      "completions/min_length": 239.0,
+      "completions/min_terminated_length": 239.0,
       "epoch": 0.2802775912245355,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07031878190856904,
-      "kl": 0.048095703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.008108481289585232,
+      "kl": 0.0139617919921875,
       "learning_rate": 1.9720912837954486e-06,
-      "loss": 0.0002,
-      "num_tokens": 187701185.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 185893611.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 313
     },
     {
@@ -8779,26 +8779,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1490.0,
-      "completions/mean_length": 554.68359375,
-      "completions/mean_terminated_length": 551.76123046875,
-      "completions/min_length": 222.0,
-      "completions/min_terminated_length": 222.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1551.0,
+      "completions/max_terminated_length": 1551.0,
+      "completions/mean_length": 560.962890625,
+      "completions/mean_terminated_length": 560.962890625,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.28117304678755317,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05260379600232649,
-      "kl": 0.04962158203125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.005140104045070806,
+      "kl": 0.00801849365234375,
       "learning_rate": 1.958432083296862e-06,
-      "loss": 0.0008,
-      "num_tokens": 188279167.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 186474808.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 314
     },
     {
@@ -8808,25 +8808,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1435.0,
-      "completions/max_terminated_length": 1435.0,
-      "completions/mean_length": 538.119140625,
-      "completions/mean_terminated_length": 537.2328491210938,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/max_length": 1372.0,
+      "completions/max_terminated_length": 1372.0,
+      "completions/mean_length": 572.884765625,
+      "completions/mean_terminated_length": 572.131103515625,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.28206850235057085,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.03687044756246853,
-      "kl": 0.05096435546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 1.1897760386031404,
+      "kl": 0.13326263427734375,
       "learning_rate": 1.9448060954027093e-06,
-      "loss": 0.0008,
-      "num_tokens": 188828012.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0013,
+      "num_tokens": 187041453.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 315
     },
     {
@@ -8836,19 +8836,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1454.0,
-      "completions/max_terminated_length": 1454.0,
-      "completions/mean_length": 561.173828125,
-      "completions/mean_terminated_length": 561.173828125,
-      "completions/min_length": 122.0,
-      "completions/min_terminated_length": 122.0,
+      "completions/max_length": 1512.0,
+      "completions/max_terminated_length": 1512.0,
+      "completions/mean_length": 567.240234375,
+      "completions/mean_terminated_length": 567.240234375,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.2829639579135885,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004801073747051007,
-      "kl": 0.04974365234375,
+      "grad_norm": 0.004133586004795131,
+      "kl": 0.008209228515625,
       "learning_rate": 1.931213891832153e-06,
-      "loss": 0.0005,
-      "num_tokens": 189415253.0,
+      "loss": 0.0001,
+      "num_tokens": 187631800.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8863,20 +8863,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1515.0,
-      "completions/max_terminated_length": 1515.0,
-      "completions/mean_length": 548.810546875,
-      "completions/mean_terminated_length": 548.810546875,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1612.0,
+      "completions/mean_length": 584.296875,
+      "completions/mean_terminated_length": 578.556884765625,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.2838594134766062,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004862757981370567,
-      "kl": 0.046875,
+      "grad_norm": 0.003874494807536967,
+      "kl": 0.007781982421875,
       "learning_rate": 1.9176560428868336e-06,
-      "loss": 0.0005,
-      "num_tokens": 189986548.0,
+      "loss": 0.0001,
+      "num_tokens": 188221264.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8892,19 +8892,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1594.0,
-      "completions/max_terminated_length": 1594.0,
-      "completions/mean_length": 591.748046875,
-      "completions/mean_terminated_length": 591.748046875,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/max_length": 1663.0,
+      "completions/max_terminated_length": 1663.0,
+      "completions/mean_length": 614.064453125,
+      "completions/mean_terminated_length": 614.064453125,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.2847548690396239,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004813244125939766,
-      "kl": 0.049072265625,
+      "grad_norm": 0.0040355129943725355,
+      "kl": 0.007965087890625,
       "learning_rate": 1.9041331174269373e-06,
-      "loss": 0.0005,
-      "num_tokens": 190630643.0,
+      "loss": 0.0001,
+      "num_tokens": 188876785.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -8919,26 +8919,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1462.0,
-      "completions/max_terminated_length": 1462.0,
-      "completions/mean_length": 549.74609375,
-      "completions/mean_terminated_length": 547.9608764648438,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1833.0,
+      "completions/max_terminated_length": 1833.0,
+      "completions/mean_length": 561.494140625,
+      "completions/mean_terminated_length": 561.494140625,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.2856503246026416,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 2.327604947622168,
-      "kl": 0.0538330078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0029625994636393927,
+      "kl": 0.00824737548828125,
       "learning_rate": 1.8906456828473341e-06,
-      "loss": 0.011,
-      "num_tokens": 191228609.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 189480766.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 319
     },
     {
@@ -8947,26 +8947,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1705.0,
-      "completions/mean_length": 550.708984375,
-      "completions/mean_terminated_length": 547.7788696289062,
-      "completions/min_length": 225.0,
-      "completions/min_terminated_length": 225.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1279.0,
+      "completions/max_terminated_length": 1279.0,
+      "completions/mean_length": 589.814453125,
+      "completions/mean_terminated_length": 589.814453125,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.2865457801656593,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07457068940555031,
-      "kl": 0.049560546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0029549040436407316,
+      "kl": 0.0076751708984375,
       "learning_rate": 1.8771943050537656e-06,
-      "loss": 0.0119,
-      "num_tokens": 191800700.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 190072879.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 320
     },
     {
@@ -8976,19 +8976,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1793.0,
-      "completions/max_terminated_length": 1793.0,
-      "completions/mean_length": 542.2109375,
-      "completions/mean_terminated_length": 542.2109375,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/max_length": 1840.0,
+      "completions/max_terminated_length": 1840.0,
+      "completions/mean_length": 582.072265625,
+      "completions/mean_terminated_length": 582.072265625,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.28744123572867697,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0051614526596112445,
-      "kl": 0.04888916015625,
+      "grad_norm": 0.004613611713105247,
+      "kl": 0.00787353515625,
       "learning_rate": 1.8637795484391046e-06,
-      "loss": 0.0005,
-      "num_tokens": 192402488.0,
+      "loss": 0.0001,
+      "num_tokens": 190695076.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9004,25 +9004,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1628.0,
-      "completions/max_terminated_length": 1628.0,
-      "completions/mean_length": 546.103515625,
-      "completions/mean_terminated_length": 544.9647827148438,
-      "completions/min_length": 223.0,
-      "completions/min_terminated_length": 223.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1186.0,
+      "completions/mean_length": 563.12890625,
+      "completions/mean_terminated_length": 560.2230834960938,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.28833669129169465,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.1883992636807388,
-      "kl": 0.2384033203125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.04918106982067727,
+      "kl": 0.007476806640625,
       "learning_rate": 1.8504019758596698e-06,
-      "loss": 0.0024,
-      "num_tokens": 193005661.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0137,
+      "num_tokens": 191306966.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 322
     },
     {
@@ -9031,20 +9031,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1251.0,
-      "completions/max_terminated_length": 1251.0,
-      "completions/mean_length": 537.029296875,
-      "completions/mean_terminated_length": 537.029296875,
-      "completions/min_length": 121.0,
-      "completions/min_terminated_length": 121.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1462.0,
+      "completions/max_terminated_length": 1462.0,
+      "completions/mean_length": 568.73828125,
+      "completions/mean_terminated_length": 567.2211303710938,
+      "completions/min_length": 120.0,
+      "completions/min_terminated_length": 120.0,
       "epoch": 0.2892321468547123,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006500542025853673,
-      "kl": 0.0484619140625,
+      "grad_norm": 0.005913204495923806,
+      "kl": 0.0136260986328125,
       "learning_rate": 1.8370621486116163e-06,
-      "loss": 0.0005,
-      "num_tokens": 193581292.0,
+      "loss": 0.0001,
+      "num_tokens": 191898832.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9059,26 +9059,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1362.0,
-      "completions/max_terminated_length": 1362.0,
-      "completions/mean_length": 576.76171875,
-      "completions/mean_terminated_length": 576.76171875,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1475.0,
+      "completions/mean_length": 604.5859375,
+      "completions/mean_terminated_length": 601.76123046875,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.29012760241773,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06971423792820254,
-      "kl": 0.04791259765625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.029507018255369777,
+      "kl": 0.0104827880859375,
       "learning_rate": 1.823760626407377e-06,
-      "loss": 0.0039,
-      "num_tokens": 194215410.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 192547196.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 324
     },
     {
@@ -9087,20 +9087,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1765.0,
-      "completions/max_terminated_length": 1765.0,
-      "completions/mean_length": 592.740234375,
-      "completions/mean_terminated_length": 592.740234375,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1531.0,
+      "completions/mean_length": 630.859375,
+      "completions/mean_terminated_length": 625.302001953125,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.2910230579807477,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004864534599712507,
-      "kl": 0.04632568359375,
+      "grad_norm": 0.0024468077068406937,
+      "kl": 0.00716400146484375,
       "learning_rate": 1.8104979673521838e-06,
-      "loss": 0.0005,
-      "num_tokens": 194846109.0,
+      "loss": 0.0001,
+      "num_tokens": 193197412.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9116,19 +9116,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1486.0,
-      "completions/max_terminated_length": 1486.0,
-      "completions/mean_length": 552.826171875,
-      "completions/mean_terminated_length": 552.826171875,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/max_length": 1583.0,
+      "completions/max_terminated_length": 1583.0,
+      "completions/mean_length": 577.087890625,
+      "completions/mean_terminated_length": 577.087890625,
+      "completions/min_length": 125.0,
+      "completions/min_terminated_length": 125.0,
       "epoch": 0.2919185135437654,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007089949043229102,
-      "kl": 0.04876708984375,
+      "grad_norm": 0.003442353077357445,
+      "kl": 0.0084228515625,
       "learning_rate": 1.7972747279206482e-06,
-      "loss": 0.0005,
-      "num_tokens": 195429956.0,
+      "loss": 0.0001,
+      "num_tokens": 193793681.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9143,26 +9143,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1858.0,
-      "completions/mean_length": 556.896484375,
-      "completions/mean_terminated_length": 553.9784545898438,
-      "completions/min_length": 218.0,
-      "completions/min_terminated_length": 218.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1713.0,
+      "completions/max_terminated_length": 1713.0,
+      "completions/mean_length": 577.369140625,
+      "completions/mean_terminated_length": 577.369140625,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.2928139691067831,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.0776498558557657,
-      "kl": 0.0474853515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0032150616476302094,
+      "kl": 0.00772857666015625,
       "learning_rate": 1.7840914629334122e-06,
-      "loss": 0.0082,
-      "num_tokens": 196038223.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 194412430.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 327
     },
     {
@@ -9171,20 +9171,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1416.0,
-      "completions/mean_length": 571.099609375,
-      "completions/mean_terminated_length": 568.2094116210938,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1558.0,
+      "completions/max_terminated_length": 1558.0,
+      "completions/mean_length": 582.974609375,
+      "completions/mean_terminated_length": 582.974609375,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.29370942466980077,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006748427087785765,
-      "kl": 0.04718017578125,
+      "grad_norm": 0.0031326415879365894,
+      "kl": 0.0074615478515625,
       "learning_rate": 1.7709487255338731e-06,
-      "loss": 0.0005,
-      "num_tokens": 196659266.0,
+      "loss": 0.0001,
+      "num_tokens": 195039553.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9199,26 +9199,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1758.0,
-      "completions/max_terminated_length": 1758.0,
-      "completions/mean_length": 575.4453125,
-      "completions/mean_terminated_length": 575.4453125,
-      "completions/min_length": 155.0,
-      "completions/min_terminated_length": 155.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1566.0,
+      "completions/mean_length": 612.119140625,
+      "completions/mean_terminated_length": 606.48828125,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.29460488023281844,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006442099438506587,
-      "kl": 0.05078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0986403330371718,
+      "kl": 0.00791168212890625,
       "learning_rate": 1.7578470671649684e-06,
-      "loss": 0.0005,
-      "num_tokens": 197297718.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0137,
+      "num_tokens": 195696782.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 329
     },
     {
@@ -9228,19 +9228,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1384.0,
-      "completions/max_terminated_length": 1384.0,
-      "completions/mean_length": 544.8359375,
-      "completions/mean_terminated_length": 544.8359375,
-      "completions/min_length": 141.0,
-      "completions/min_terminated_length": 141.0,
+      "completions/max_length": 1522.0,
+      "completions/max_terminated_length": 1522.0,
+      "completions/mean_length": 565.794921875,
+      "completions/mean_terminated_length": 565.794921875,
+      "completions/min_length": 115.0,
+      "completions/min_terminated_length": 115.0,
       "epoch": 0.2955003357958361,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007338475781825208,
-      "kl": 0.049072265625,
+      "grad_norm": 0.0034720570770899726,
+      "kl": 0.00804901123046875,
       "learning_rate": 1.744787037546045e-06,
-      "loss": 0.0005,
-      "num_tokens": 197901570.0,
+      "loss": 0.0001,
+      "num_tokens": 196311365.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9256,19 +9256,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1936.0,
-      "completions/max_terminated_length": 1936.0,
-      "completions/mean_length": 573.591796875,
-      "completions/mean_terminated_length": 572.878662109375,
-      "completions/min_length": 215.0,
-      "completions/min_terminated_length": 215.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1588.0,
+      "completions/mean_length": 608.689453125,
+      "completions/mean_terminated_length": 605.872802734375,
+      "completions/min_length": 236.0,
+      "completions/min_terminated_length": 236.0,
       "epoch": 0.2963957913588538,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.07468248860939418,
-      "kl": 0.06597900390625,
+      "grad_norm": 0.007254504464050369,
+      "kl": 0.00806427001953125,
       "learning_rate": 1.731769184649788e-06,
-      "loss": 0.0007,
-      "num_tokens": 198541121.0,
+      "loss": 0.0001,
+      "num_tokens": 196968886.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9284,19 +9284,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1480.0,
-      "completions/max_terminated_length": 1480.0,
-      "completions/mean_length": 557.41796875,
-      "completions/mean_terminated_length": 556.6849365234375,
-      "completions/min_length": 121.0,
-      "completions/min_terminated_length": 121.0,
+      "completions/max_length": 1772.0,
+      "completions/max_terminated_length": 1772.0,
+      "completions/mean_length": 571.40234375,
+      "completions/mean_terminated_length": 570.3307495117188,
+      "completions/min_length": 137.0,
+      "completions/min_terminated_length": 137.0,
       "epoch": 0.2972912469218715,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.035730325862680115,
-      "kl": 0.05535888671875,
+      "grad_norm": 0.01230074303508439,
+      "kl": 0.0143280029296875,
       "learning_rate": 1.7187940546792325e-06,
-      "loss": 0.0006,
-      "num_tokens": 199119687.0,
+      "loss": 0.0001,
+      "num_tokens": 197554612.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9312,19 +9312,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1388.0,
-      "completions/max_terminated_length": 1388.0,
-      "completions/mean_length": 569.91015625,
-      "completions/mean_terminated_length": 569.91015625,
-      "completions/min_length": 131.0,
-      "completions/min_terminated_length": 131.0,
+      "completions/max_length": 1974.0,
+      "completions/max_terminated_length": 1974.0,
+      "completions/mean_length": 594.96875,
+      "completions/mean_terminated_length": 594.96875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.2981867024848892,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0064265463541584265,
-      "kl": 0.0482177734375,
+      "grad_norm": 0.0027166854804102003,
+      "kl": 0.0079498291015625,
       "learning_rate": 1.7058621920448465e-06,
-      "loss": 0.0005,
-      "num_tokens": 199718585.0,
+      "loss": 0.0001,
+      "num_tokens": 198166340.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9340,19 +9340,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1220.0,
-      "completions/max_terminated_length": 1220.0,
-      "completions/mean_length": 569.783203125,
-      "completions/mean_terminated_length": 569.783203125,
-      "completions/min_length": 260.0,
-      "completions/min_terminated_length": 260.0,
+      "completions/max_length": 1840.0,
+      "completions/max_terminated_length": 1840.0,
+      "completions/mean_length": 597.056640625,
+      "completions/mean_terminated_length": 597.056640625,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.2990821580479069,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00432845573715703,
-      "kl": 0.04718017578125,
+      "grad_norm": 0.0025843510081473833,
+      "kl": 0.00785064697265625,
       "learning_rate": 1.6929741393416855e-06,
-      "loss": 0.0005,
-      "num_tokens": 200322890.0,
+      "loss": 0.0001,
+      "num_tokens": 198784609.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9367,26 +9367,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1511.0,
-      "completions/mean_length": 548.86328125,
-      "completions/mean_terminated_length": 543.6817626953125,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1316.0,
+      "completions/max_terminated_length": 1316.0,
+      "completions/mean_length": 555.533203125,
+      "completions/mean_terminated_length": 555.533203125,
+      "completions/min_length": 192.0,
+      "completions/min_terminated_length": 192.0,
       "epoch": 0.29997761361092457,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.10020521096041221,
-      "kl": 0.06817626953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002875308277545933,
+      "kl": 0.0073699951171875,
       "learning_rate": 1.6801304373266286e-06,
-      "loss": 0.0131,
-      "num_tokens": 200884212.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 199349346.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 335
     },
     {
@@ -9396,25 +9396,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1516.0,
-      "completions/max_terminated_length": 1516.0,
-      "completions/mean_length": 551.220703125,
-      "completions/mean_terminated_length": 551.220703125,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/max_length": 1431.0,
+      "completions/max_terminated_length": 1431.0,
+      "completions/mean_length": 555.279296875,
+      "completions/mean_terminated_length": 555.279296875,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.30087306917394224,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.055243589570167055,
-      "kl": 0.04827880859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0026065604510884808,
+      "kl": 0.00782012939453125,
       "learning_rate": 1.667331624895689e-06,
-      "loss": 0.001,
-      "num_tokens": 201479509.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 199946721.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 336
     },
     {
@@ -9424,19 +9424,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1496.0,
-      "completions/mean_length": 572.970703125,
-      "completions/mean_terminated_length": 570.0841674804688,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_length": 1670.0,
+      "completions/max_terminated_length": 1622.0,
+      "completions/mean_length": 597.9375,
+      "completions/mean_terminated_length": 595.8395385742188,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.3017685247369599,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05189289377144097,
-      "kl": 0.05181884765625,
+      "grad_norm": 0.10141160919841502,
+      "kl": 0.009429931640625,
       "learning_rate": 1.6545782390614037e-06,
-      "loss": 0.0093,
-      "num_tokens": 202068726.0,
+      "loss": 0.0086,
+      "num_tokens": 200548721.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -9452,19 +9452,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1363.0,
-      "completions/max_terminated_length": 1363.0,
-      "completions/mean_length": 532.65234375,
-      "completions/mean_terminated_length": 532.65234375,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/max_length": 1428.0,
+      "completions/max_terminated_length": 1428.0,
+      "completions/mean_length": 546.87109375,
+      "completions/mean_terminated_length": 546.87109375,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.3026639802999776,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004464966063305006,
-      "kl": 0.04925537109375,
+      "grad_norm": 0.0024956441500498133,
+      "kl": 0.0073699951171875,
       "learning_rate": 1.6418708149302992e-06,
-      "loss": 0.0005,
-      "num_tokens": 202608612.0,
+      "loss": 0.0001,
+      "num_tokens": 201095887.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9480,19 +9480,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1407.0,
-      "completions/max_terminated_length": 1407.0,
-      "completions/mean_length": 561.587890625,
-      "completions/mean_terminated_length": 561.587890625,
-      "completions/min_length": 172.0,
-      "completions/min_terminated_length": 172.0,
+      "completions/max_length": 1574.0,
+      "completions/max_terminated_length": 1574.0,
+      "completions/mean_length": 557.408203125,
+      "completions/mean_terminated_length": 557.408203125,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.3035594358629953,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004036873268911823,
-      "kl": 0.04779052734375,
+      "grad_norm": 0.002926922407824277,
+      "kl": 0.0082550048828125,
       "learning_rate": 1.6292098856804423e-06,
-      "loss": 0.0005,
-      "num_tokens": 203193185.0,
+      "loss": 0.0001,
+      "num_tokens": 201678320.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9508,19 +9508,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1599.0,
-      "completions/max_terminated_length": 1599.0,
-      "completions/mean_length": 632.673828125,
-      "completions/mean_terminated_length": 632.673828125,
-      "completions/min_length": 130.0,
-      "completions/min_terminated_length": 130.0,
+      "completions/max_length": 1695.0,
+      "completions/max_terminated_length": 1695.0,
+      "completions/mean_length": 647.646484375,
+      "completions/mean_terminated_length": 647.646484375,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.304454891426013,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005087884865379063,
-      "kl": 0.04986572265625,
+      "grad_norm": 0.0026743176977703474,
+      "kl": 0.00763702392578125,
       "learning_rate": 1.6165959825390661e-06,
-      "loss": 0.0005,
-      "num_tokens": 203897802.0,
+      "loss": 0.0001,
+      "num_tokens": 202390603.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9536,25 +9536,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1579.0,
-      "completions/max_terminated_length": 1579.0,
-      "completions/mean_length": 591.111328125,
-      "completions/mean_terminated_length": 590.142822265625,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2007.0,
+      "completions/mean_length": 597.0546875,
+      "completions/mean_terminated_length": 594.2152709960938,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.3053503469890307,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005377231692265041,
-      "kl": 0.051513671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0664857311197945,
+      "kl": 0.00785064697265625,
       "learning_rate": 1.604029634760284e-06,
-      "loss": 0.0005,
-      "num_tokens": 204505315.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0131,
+      "num_tokens": 203001159.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 341
     },
     {
@@ -9563,26 +9563,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1411.0,
-      "completions/max_terminated_length": 1411.0,
-      "completions/mean_length": 587.439453125,
-      "completions/mean_terminated_length": 586.878662109375,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1380.0,
+      "completions/max_terminated_length": 1380.0,
+      "completions/mean_length": 582.21875,
+      "completions/mean_terminated_length": 582.21875,
+      "completions/min_length": 115.0,
+      "completions/min_terminated_length": 115.0,
       "epoch": 0.30624580255204836,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07341969235932985,
-      "kl": 0.04986572265625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0022938907257129934,
+      "kl": 0.00775146484375,
       "learning_rate": 1.59151136960288e-06,
-      "loss": 0.0029,
-      "num_tokens": 205135572.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 203628743.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 342
     },
     {
@@ -9592,19 +9592,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1459.0,
-      "completions/max_terminated_length": 1459.0,
-      "completions/mean_length": 574.189453125,
-      "completions/mean_terminated_length": 574.189453125,
-      "completions/min_length": 222.0,
-      "completions/min_terminated_length": 222.0,
+      "completions/max_length": 1719.0,
+      "completions/max_terminated_length": 1719.0,
+      "completions/mean_length": 580.033203125,
+      "completions/mean_terminated_length": 580.033203125,
+      "completions/min_length": 133.0,
+      "completions/min_terminated_length": 133.0,
       "epoch": 0.30714125811506604,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004649311531668215,
-      "kl": 0.048828125,
+      "grad_norm": 0.002789323837930816,
+      "kl": 0.0079498291015625,
       "learning_rate": 1.5790417123081903e-06,
-      "loss": 0.0005,
-      "num_tokens": 205708453.0,
+      "loss": 0.0001,
+      "num_tokens": 204204616.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9619,26 +9619,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1230.0,
-      "completions/max_terminated_length": 1230.0,
-      "completions/mean_length": 555.431640625,
-      "completions/mean_terminated_length": 555.431640625,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1919.0,
+      "completions/mean_length": 554.44140625,
+      "completions/mean_terminated_length": 548.8372802734375,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.3080367136780837,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09669618323590906,
-      "kl": 0.0479736328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 20.33983786834859,
+      "kl": 1.365081787109375,
       "learning_rate": 1.5666211860780583e-06,
-      "loss": 0.0021,
-      "num_tokens": 206264770.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.027,
+      "num_tokens": 204760426.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 344
     },
     {
@@ -9647,20 +9647,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1325.0,
-      "completions/max_terminated_length": 1325.0,
-      "completions/mean_length": 608.814453125,
-      "completions/mean_terminated_length": 607.5107421875,
-      "completions/min_length": 261.0,
-      "completions/min_terminated_length": 261.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1682.0,
+      "completions/max_terminated_length": 1682.0,
+      "completions/mean_length": 594.248046875,
+      "completions/mean_terminated_length": 594.248046875,
+      "completions/min_length": 235.0,
+      "completions/min_terminated_length": 235.0,
       "epoch": 0.3089321692411014,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006432639585104695,
-      "kl": 0.0496826171875,
+      "grad_norm": 0.0026992151848093727,
+      "kl": 0.0073699951171875,
       "learning_rate": 1.5542503120528918e-06,
-      "loss": 0.0005,
-      "num_tokens": 206890307.0,
+      "loss": 0.0001,
+      "num_tokens": 205378505.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9676,19 +9676,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1633.0,
-      "completions/max_terminated_length": 1633.0,
-      "completions/mean_length": 597.21484375,
-      "completions/mean_terminated_length": 597.21484375,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1530.0,
+      "completions/max_terminated_length": 1530.0,
+      "completions/mean_length": 588.36328125,
+      "completions/mean_terminated_length": 588.36328125,
+      "completions/min_length": 156.0,
+      "completions/min_terminated_length": 156.0,
       "epoch": 0.3098276248041191,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0054548326889689094,
-      "kl": 0.04541015625,
+      "grad_norm": 0.002201032636526166,
+      "kl": 0.00725555419921875,
       "learning_rate": 1.5419296092897866e-06,
-      "loss": 0.0005,
-      "num_tokens": 207469153.0,
+      "loss": 0.0001,
+      "num_tokens": 205952819.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9704,19 +9704,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1477.0,
-      "completions/max_terminated_length": 1477.0,
-      "completions/mean_length": 542.3125,
-      "completions/mean_terminated_length": 542.3125,
-      "completions/min_length": 158.0,
-      "completions/min_terminated_length": 158.0,
+      "completions/max_length": 1891.0,
+      "completions/max_terminated_length": 1891.0,
+      "completions/mean_length": 532.150390625,
+      "completions/mean_terminated_length": 532.150390625,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.3107230803671368,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004968550676186407,
-      "kl": 0.04718017578125,
+      "grad_norm": 0.002508504389666509,
+      "kl": 0.00791168212890625,
       "learning_rate": 1.529659594740755e-06,
-      "loss": 0.0005,
-      "num_tokens": 208023537.0,
+      "loss": 0.0001,
+      "num_tokens": 206502000.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9731,26 +9731,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1501.0,
-      "completions/mean_length": 609.939453125,
-      "completions/mean_terminated_length": 604.9157104492188,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 1734.0,
+      "completions/max_terminated_length": 1583.0,
+      "completions/mean_length": 599.701171875,
+      "completions/mean_terminated_length": 593.0157470703125,
+      "completions/min_length": 129.0,
+      "completions/min_terminated_length": 129.0,
       "epoch": 0.3116185359301545,
-      "frac_reward_zero_std": 0.90625,
-      "grad_norm": 0.12041308563154017,
-      "kl": 0.05133056640625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09161593323878525,
+      "kl": 0.00812530517578125,
       "learning_rate": 1.5174407832310338e-06,
-      "loss": 0.0176,
-      "num_tokens": 208666306.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0023437500931322575,
+      "loss": 0.0083,
+      "num_tokens": 207139527.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 348
     },
     {
@@ -9760,25 +9760,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1665.0,
-      "completions/max_terminated_length": 1665.0,
-      "completions/mean_length": 606.841796875,
-      "completions/mean_terminated_length": 605.76708984375,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1606.0,
+      "completions/mean_length": 601.6015625,
+      "completions/mean_terminated_length": 598.7710571289062,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.31251399149317216,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 1.0907557345293637,
-      "kl": 0.0474853515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.004103618000259221,
+      "kl": 0.0073089599609375,
       "learning_rate": 1.5052736874374815e-06,
-      "loss": 0.0043,
-      "num_tokens": 209293617.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 207764155.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 349
     },
     {
@@ -9787,26 +9787,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1548.0,
-      "completions/max_terminated_length": 1548.0,
-      "completions/mean_length": 568.509765625,
-      "completions/mean_terminated_length": 567.4539794921875,
-      "completions/min_length": 231.0,
-      "completions/min_terminated_length": 231.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1784.0,
+      "completions/max_terminated_length": 1784.0,
+      "completions/mean_length": 548.10546875,
+      "completions/mean_terminated_length": 548.10546875,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.31340944705618984,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06738432092092399,
-      "kl": 0.05059814453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00228408374001061,
+      "kl": 0.0070343017578125,
       "learning_rate": 1.4931588178670695e-06,
-      "loss": 0.0038,
-      "num_tokens": 209870166.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 208330257.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 350
     },
     {
@@ -9816,25 +9816,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1246.0,
-      "completions/max_terminated_length": 1246.0,
-      "completions/mean_length": 587.666015625,
-      "completions/mean_terminated_length": 587.666015625,
-      "completions/min_length": 217.0,
-      "completions/min_terminated_length": 217.0,
+      "completions/max_length": 1369.0,
+      "completions/max_terminated_length": 1369.0,
+      "completions/mean_length": 571.197265625,
+      "completions/mean_terminated_length": 571.197265625,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.3143049026192075,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0667752478450315,
-      "kl": 0.04486083984375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0020964969256138085,
+      "kl": 0.00688934326171875,
       "learning_rate": 1.4810966828354605e-06,
-      "loss": 0.0002,
-      "num_tokens": 210524939.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 208976598.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 351
     },
     {
@@ -9843,26 +9843,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1489.0,
-      "completions/mean_length": 585.203125,
-      "completions/mean_terminated_length": 582.3405151367188,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1443.0,
+      "completions/max_terminated_length": 1443.0,
+      "completions/mean_length": 561.779296875,
+      "completions/mean_terminated_length": 561.779296875,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.3152003581822252,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0747582857471383,
-      "kl": 0.048828125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003302003902452213,
+      "kl": 0.0074615478515625,
       "learning_rate": 1.469087788445684e-06,
-      "loss": 0.0083,
-      "num_tokens": 211128531.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 209568197.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 352
     },
     {
@@ -9872,19 +9872,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1240.0,
-      "completions/max_terminated_length": 1240.0,
-      "completions/mean_length": 571.16015625,
-      "completions/mean_terminated_length": 571.16015625,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/max_length": 1252.0,
+      "completions/max_terminated_length": 1252.0,
+      "completions/mean_length": 548.845703125,
+      "completions/mean_terminated_length": 548.845703125,
+      "completions/min_length": 191.0,
+      "completions/min_terminated_length": 191.0,
       "epoch": 0.31609581374524287,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0041276490396841056,
-      "kl": 0.047607421875,
+      "grad_norm": 0.002776716601889407,
+      "kl": 0.00762176513671875,
       "learning_rate": 1.4571326385668965e-06,
-      "loss": 0.0005,
-      "num_tokens": 211749589.0,
+      "loss": 0.0001,
+      "num_tokens": 210177830.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9899,20 +9899,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1388.0,
-      "completions/max_terminated_length": 1388.0,
-      "completions/mean_length": 576.314453125,
-      "completions/mean_terminated_length": 575.5655517578125,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1176.0,
+      "completions/max_terminated_length": 1176.0,
+      "completions/mean_length": 563.796875,
+      "completions/mean_terminated_length": 563.796875,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.3169912693082606,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.8926123024299515,
-      "kl": 0.26904296875,
+      "grad_norm": 0.0022599035175075058,
+      "kl": 0.007415771484375,
       "learning_rate": 1.4452317348132434e-06,
-      "loss": 0.0027,
-      "num_tokens": 212360054.0,
+      "loss": 0.0001,
+      "num_tokens": 210781886.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9927,20 +9927,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1667.0,
-      "completions/max_terminated_length": 1667.0,
-      "completions/mean_length": 583.8984375,
-      "completions/mean_terminated_length": 582.5910034179688,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1258.0,
+      "completions/max_terminated_length": 1258.0,
+      "completions/mean_length": 558.203125,
+      "completions/mean_terminated_length": 558.203125,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
       "epoch": 0.3178867248712783,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.45238995582403174,
-      "kl": 0.14990234375,
+      "grad_norm": 0.0020322560028473275,
+      "kl": 0.00682830810546875,
       "learning_rate": 1.4333855765228104e-06,
-      "loss": 0.0015,
-      "num_tokens": 212965298.0,
+      "loss": 0.0001,
+      "num_tokens": 211373974.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -9955,26 +9955,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1934.0,
-      "completions/mean_length": 625.888671875,
-      "completions/mean_terminated_length": 623.1056518554688,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1791.0,
+      "completions/max_terminated_length": 1791.0,
+      "completions/mean_length": 622.130859375,
+      "completions/mean_terminated_length": 622.130859375,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.31878218043429596,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06558967056937225,
-      "kl": 0.04473876953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0021667727258320112,
+      "kl": 0.0065765380859375,
       "learning_rate": 1.421594660736675e-06,
-      "loss": 0.0103,
-      "num_tokens": 213597257.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 212004009.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 356
     },
     {
@@ -9984,19 +9984,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1395.0,
-      "completions/max_terminated_length": 1395.0,
-      "completions/mean_length": 560.33203125,
-      "completions/mean_terminated_length": 560.33203125,
-      "completions/min_length": 193.0,
-      "completions/min_terminated_length": 193.0,
+      "completions/max_length": 1612.0,
+      "completions/max_terminated_length": 1612.0,
+      "completions/mean_length": 534.94140625,
+      "completions/mean_terminated_length": 534.94140625,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.31967763599731364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004939443011093821,
-      "kl": 0.04925537109375,
+      "grad_norm": 0.002663417489634053,
+      "kl": 0.00749969482421875,
       "learning_rate": 1.4098594821780476e-06,
-      "loss": 0.0005,
-      "num_tokens": 214158339.0,
+      "loss": 0.0001,
+      "num_tokens": 212552091.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10012,19 +10012,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1255.0,
-      "completions/max_terminated_length": 1255.0,
-      "completions/mean_length": 563.447265625,
-      "completions/mean_terminated_length": 563.447265625,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_length": 1600.0,
+      "completions/max_terminated_length": 1600.0,
+      "completions/mean_length": 564.744140625,
+      "completions/mean_terminated_length": 564.744140625,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.3205730915603313,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004402935055334641,
-      "kl": 0.04644775390625,
+      "grad_norm": 0.0031061900289884826,
+      "kl": 0.00829315185546875,
       "learning_rate": 1.3981805332315174e-06,
-      "loss": 0.0005,
-      "num_tokens": 214760824.0,
+      "loss": 0.0001,
+      "num_tokens": 213155240.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10040,19 +10040,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1443.0,
-      "completions/max_terminated_length": 1443.0,
-      "completions/mean_length": 575.783203125,
-      "completions/mean_terminated_length": 575.783203125,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_length": 1436.0,
+      "completions/max_terminated_length": 1436.0,
+      "completions/mean_length": 571.017578125,
+      "completions/mean_terminated_length": 571.017578125,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.321468547123349,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004501749654718466,
-      "kl": 0.0462646484375,
+      "grad_norm": 0.0021493978916938975,
+      "kl": 0.006927490234375,
       "learning_rate": 1.3865583039223929e-06,
-      "loss": 0.0005,
-      "num_tokens": 215352073.0,
+      "loss": 0.0001,
+      "num_tokens": 213744049.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10068,19 +10068,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1129.0,
-      "completions/max_terminated_length": 1129.0,
-      "completions/mean_length": 541.359375,
-      "completions/mean_terminated_length": 541.359375,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/max_length": 1491.0,
+      "completions/max_terminated_length": 1491.0,
+      "completions/mean_length": 547.46875,
+      "completions/mean_terminated_length": 547.46875,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.32236400268636667,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004656261138922308,
-      "kl": 0.04718017578125,
+      "grad_norm": 0.002390722572560558,
+      "kl": 0.00682830810546875,
       "learning_rate": 1.374993281896137e-06,
-      "loss": 0.0005,
-      "num_tokens": 215928289.0,
+      "loss": 0.0001,
+      "num_tokens": 214323393.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10095,20 +10095,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1584.0,
-      "completions/max_terminated_length": 1584.0,
-      "completions/mean_length": 542.564453125,
-      "completions/mean_terminated_length": 542.564453125,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1235.0,
+      "completions/mean_length": 530.275390625,
+      "completions/mean_terminated_length": 524.3235473632812,
+      "completions/min_length": 145.0,
+      "completions/min_terminated_length": 145.0,
       "epoch": 0.3232594582493844,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004741245952791287,
-      "kl": 0.04730224609375,
+      "grad_norm": 0.002902768186402617,
+      "kl": 0.00717926025390625,
       "learning_rate": 1.3634859523979134e-06,
-      "loss": 0.0005,
-      "num_tokens": 216489026.0,
+      "loss": 0.0001,
+      "num_tokens": 214877838.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10124,19 +10124,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1602.0,
-      "completions/max_terminated_length": 1602.0,
-      "completions/mean_length": 566.744140625,
-      "completions/mean_terminated_length": 566.744140625,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_length": 1317.0,
+      "completions/max_terminated_length": 1317.0,
+      "completions/mean_length": 566.4140625,
+      "completions/mean_terminated_length": 566.4140625,
+      "completions/min_length": 176.0,
+      "completions/min_terminated_length": 176.0,
       "epoch": 0.3241549138124021,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004922462540278737,
-      "kl": 0.04833984375,
+      "grad_norm": 0.002136628038833396,
+      "kl": 0.00792694091796875,
       "learning_rate": 1.3520367982522208e-06,
-      "loss": 0.0005,
-      "num_tokens": 217065039.0,
+      "loss": 0.0001,
+      "num_tokens": 215453682.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10151,20 +10151,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1338.0,
-      "completions/max_terminated_length": 1338.0,
-      "completions/mean_length": 547.189453125,
-      "completions/mean_terminated_length": 547.189453125,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1374.0,
+      "completions/mean_length": 557.8984375,
+      "completions/mean_terminated_length": 554.9823608398438,
+      "completions/min_length": 229.0,
+      "completions/min_terminated_length": 229.0,
       "epoch": 0.32505036937541976,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004862974742261159,
-      "kl": 0.04705810546875,
+      "grad_norm": 0.002198479192435782,
+      "kl": 0.00743865966796875,
       "learning_rate": 1.3406462998426358e-06,
-      "loss": 0.0005,
-      "num_tokens": 217628032.0,
+      "loss": 0.0001,
+      "num_tokens": 216022158.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10179,26 +10179,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1460.0,
-      "completions/max_terminated_length": 1460.0,
-      "completions/mean_length": 609.470703125,
-      "completions/mean_terminated_length": 607.2078857421875,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1581.0,
+      "completions/max_terminated_length": 1581.0,
+      "completions/mean_length": 619.244140625,
+      "completions/mean_terminated_length": 619.244140625,
+      "completions/min_length": 243.0,
+      "completions/min_terminated_length": 243.0,
       "epoch": 0.32594582493843743,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2.0311803594047575,
-      "kl": 0.0655517578125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002120261143082215,
+      "kl": 0.0067596435546875,
       "learning_rate": 1.3293149350916595e-06,
-      "loss": 0.0072,
-      "num_tokens": 218257233.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 216656363.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 364
     },
     {
@@ -10207,20 +10207,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1472.0,
-      "completions/max_terminated_length": 1472.0,
-      "completions/mean_length": 584.74609375,
-      "completions/mean_terminated_length": 584.74609375,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 2000.0,
+      "completions/mean_length": 580.7578125,
+      "completions/mean_terminated_length": 577.886474609375,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.3268412805014551,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004975922076955785,
-      "kl": 0.04620361328125,
+      "grad_norm": 0.008873138088250711,
+      "kl": 0.00872802734375,
       "learning_rate": 1.3180431794406623e-06,
-      "loss": 0.0005,
-      "num_tokens": 218856031.0,
+      "loss": 0.0001,
+      "num_tokens": 217253119.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10235,20 +10235,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1364.0,
-      "completions/max_terminated_length": 1364.0,
-      "completions/mean_length": 577.599609375,
-      "completions/mean_terminated_length": 576.2211303710938,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1617.0,
+      "completions/max_terminated_length": 1617.0,
+      "completions/mean_length": 579.234375,
+      "completions/mean_terminated_length": 579.234375,
+      "completions/min_length": 124.0,
+      "completions/min_terminated_length": 124.0,
       "epoch": 0.3277367360644728,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.048253010828955015,
-      "kl": 0.057373046875,
+      "grad_norm": 0.0021260611537859343,
+      "kl": 0.00704193115234375,
       "learning_rate": 1.3068315058299358e-06,
-      "loss": 0.0006,
-      "num_tokens": 219451442.0,
+      "loss": 0.0001,
+      "num_tokens": 217849367.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10263,20 +10263,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1321.0,
-      "completions/max_terminated_length": 1321.0,
-      "completions/mean_length": 594.275390625,
-      "completions/mean_terminated_length": 592.8532104492188,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1519.0,
+      "completions/max_terminated_length": 1519.0,
+      "completions/mean_length": 589.966796875,
+      "completions/mean_terminated_length": 589.966796875,
+      "completions/min_length": 246.0,
+      "completions/min_terminated_length": 246.0,
       "epoch": 0.32863219162749047,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0489489060192923,
-      "kl": 0.05865478515625,
+      "grad_norm": 0.002204999600811044,
+      "kl": 0.00702667236328125,
       "learning_rate": 1.2956803846788503e-06,
-      "loss": 0.0006,
-      "num_tokens": 220057999.0,
+      "loss": 0.0001,
+      "num_tokens": 218453718.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10291,20 +10291,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1641.0,
-      "completions/max_terminated_length": 1641.0,
-      "completions/mean_length": 571.3046875,
-      "completions/mean_terminated_length": 571.3046875,
-      "completions/min_length": 163.0,
-      "completions/min_terminated_length": 163.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1940.0,
+      "completions/mean_length": 576.056640625,
+      "completions/mean_terminated_length": 573.1761474609375,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.32952764719050814,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004236000922588133,
-      "kl": 0.0469970703125,
+      "grad_norm": 0.00232950684589546,
+      "kl": 0.00756072998046875,
       "learning_rate": 1.284590283866116e-06,
-      "loss": 0.0005,
-      "num_tokens": 220667243.0,
+      "loss": 0.0001,
+      "num_tokens": 219065395.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10320,19 +10320,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1322.0,
-      "completions/max_terminated_length": 1322.0,
-      "completions/mean_length": 646.376953125,
-      "completions/mean_terminated_length": 646.376953125,
-      "completions/min_length": 233.0,
-      "completions/min_terminated_length": 233.0,
+      "completions/max_length": 1794.0,
+      "completions/max_terminated_length": 1794.0,
+      "completions/mean_length": 629.060546875,
+      "completions/mean_terminated_length": 629.060546875,
+      "completions/min_length": 216.0,
+      "completions/min_terminated_length": 216.0,
       "epoch": 0.3304231027535259,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004577538862500448,
-      "kl": 0.046875,
+      "grad_norm": 0.0021195870410575485,
+      "kl": 0.006866455078125,
       "learning_rate": 1.2735616687101518e-06,
-      "loss": 0.0005,
-      "num_tokens": 221374060.0,
+      "loss": 0.0001,
+      "num_tokens": 219763346.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10347,26 +10347,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1172.0,
-      "completions/max_terminated_length": 1172.0,
-      "completions/mean_length": 565.533203125,
-      "completions/mean_terminated_length": 564.5929565429688,
-      "completions/min_length": 219.0,
-      "completions/min_terminated_length": 219.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1389.0,
+      "completions/max_terminated_length": 1389.0,
+      "completions/mean_length": 570.484375,
+      "completions/mean_terminated_length": 570.484375,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.33131855831654355,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09863928059149049,
-      "kl": 0.0606689453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0024615791853039603,
+      "kl": 0.00725555419921875,
       "learning_rate": 1.2625950019495614e-06,
-      "loss": 0.0009,
-      "num_tokens": 221970813.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 220362634.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 370
     },
     {
@@ -10376,19 +10376,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1147.0,
-      "completions/max_terminated_length": 1147.0,
-      "completions/mean_length": 564.126953125,
-      "completions/mean_terminated_length": 564.126953125,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/max_length": 1190.0,
+      "completions/max_terminated_length": 1190.0,
+      "completions/mean_length": 573.626953125,
+      "completions/mean_terminated_length": 573.626953125,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.33221401387956123,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005033013139063627,
-      "kl": 0.047607421875,
+      "grad_norm": 0.002016962022433684,
+      "kl": 0.00699615478515625,
       "learning_rate": 1.251690743723718e-06,
-      "loss": 0.0005,
-      "num_tokens": 222567662.0,
+      "loss": 0.0001,
+      "num_tokens": 220964347.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10404,19 +10404,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1359.0,
-      "completions/max_terminated_length": 1359.0,
-      "completions/mean_length": 571.357421875,
-      "completions/mean_terminated_length": 571.357421875,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1322.0,
+      "completions/max_terminated_length": 1322.0,
+      "completions/mean_length": 588.921875,
+      "completions/mean_terminated_length": 588.921875,
+      "completions/min_length": 168.0,
+      "completions/min_terminated_length": 168.0,
       "epoch": 0.3331094694425789,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004486081414495797,
-      "kl": 0.0452880859375,
+      "grad_norm": 0.0019903531026202316,
+      "kl": 0.00666046142578125,
       "learning_rate": 1.2408493515534581e-06,
-      "loss": 0.0005,
-      "num_tokens": 223146389.0,
+      "loss": 0.0001,
+      "num_tokens": 221552067.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10431,20 +10431,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1588.0,
-      "completions/max_terminated_length": 1588.0,
-      "completions/mean_length": 593.37109375,
-      "completions/mean_terminated_length": 592.3013916015625,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1343.0,
+      "completions/max_terminated_length": 1343.0,
+      "completions/mean_length": 589.775390625,
+      "completions/mean_terminated_length": 589.775390625,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.3340049250055966,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.6676414497649207,
-      "kl": 0.23779296875,
+      "grad_norm": 0.002242842785657926,
+      "kl": 0.00731658935546875,
       "learning_rate": 1.2300712803218834e-06,
-      "loss": 0.0024,
-      "num_tokens": 223739315.0,
+      "loss": 0.0001,
+      "num_tokens": 222143152.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10459,20 +10459,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1376.0,
-      "completions/max_terminated_length": 1376.0,
-      "completions/mean_length": 561.052734375,
-      "completions/mean_terminated_length": 558.0452270507812,
-      "completions/min_length": 136.0,
-      "completions/min_terminated_length": 136.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1534.0,
+      "completions/max_terminated_length": 1534.0,
+      "completions/mean_length": 558.314453125,
+      "completions/mean_terminated_length": 558.314453125,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.33490038056861426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 6.352807716641198,
-      "kl": 1.9359130859375,
+      "grad_norm": 0.003060731538706847,
+      "kl": 0.007781982421875,
       "learning_rate": 1.2193569822552772e-06,
-      "loss": 0.0193,
-      "num_tokens": 224340462.0,
+      "loss": 0.0001,
+      "num_tokens": 222742897.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10487,26 +10487,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1526.0,
-      "completions/max_terminated_length": 1526.0,
-      "completions/mean_length": 559.923828125,
-      "completions/mean_terminated_length": 559.0567626953125,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1411.0,
+      "completions/max_terminated_length": 1411.0,
+      "completions/mean_length": 582.775390625,
+      "completions/mean_terminated_length": 582.775390625,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.33579583613163194,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07663783762398667,
-      "kl": 0.0635986328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0017875770315972821,
+      "kl": 0.0068359375,
       "learning_rate": 1.2087069069041268e-06,
-      "loss": -0.0018,
-      "num_tokens": 224924535.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 223338670.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 375
     },
     {
@@ -10515,26 +10515,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1792.0,
-      "completions/mean_length": 594.92578125,
-      "completions/mean_terminated_length": 592.0822143554688,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1358.0,
+      "completions/max_terminated_length": 1358.0,
+      "completions/mean_length": 617.2421875,
+      "completions/mean_terminated_length": 617.2421875,
+      "completions/min_length": 142.0,
+      "completions/min_terminated_length": 142.0,
       "epoch": 0.3366912916946497,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.059619881056307,
-      "kl": 0.04766845703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.010428174741634343,
+      "kl": 0.00791168212890625,
       "learning_rate": 1.1981215011242654e-06,
-      "loss": 0.0094,
-      "num_tokens": 225539377.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 223964938.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 376
     },
     {
@@ -10543,26 +10543,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1601.0,
-      "completions/max_terminated_length": 1601.0,
-      "completions/mean_length": 569.505859375,
-      "completions/mean_terminated_length": 568.4813842773438,
-      "completions/min_length": 147.0,
-      "completions/min_terminated_length": 147.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1244.0,
+      "completions/max_terminated_length": 1244.0,
+      "completions/mean_length": 585.546875,
+      "completions/mean_terminated_length": 585.546875,
+      "completions/min_length": 108.0,
+      "completions/min_terminated_length": 108.0,
       "epoch": 0.33758674725766735,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.09423306788440591,
-      "kl": 0.0728759765625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0020485851533229055,
+      "kl": 0.00693511962890625,
       "learning_rate": 1.1876012090581184e-06,
-      "loss": 0.0034,
-      "num_tokens": 226144916.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 224578690.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 377
     },
     {
@@ -10571,20 +10571,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1744.0,
-      "completions/max_terminated_length": 1744.0,
-      "completions/mean_length": 585.783203125,
-      "completions/mean_terminated_length": 584.75537109375,
-      "completions/min_length": 126.0,
-      "completions/min_terminated_length": 126.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1708.0,
+      "completions/max_terminated_length": 1708.0,
+      "completions/mean_length": 586.498046875,
+      "completions/mean_terminated_length": 586.498046875,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.33848220282068503,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.015044538467711114,
-      "kl": 0.053955078125,
+      "grad_norm": 0.0023864026164809745,
+      "kl": 0.00688934326171875,
       "learning_rate": 1.177146472116071e-06,
-      "loss": 0.0005,
-      "num_tokens": 226759605.0,
+      "loss": 0.0001,
+      "num_tokens": 225193745.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10599,20 +10599,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1211.0,
-      "completions/max_terminated_length": 1211.0,
-      "completions/mean_length": 563.88671875,
-      "completions/mean_terminated_length": 563.88671875,
-      "completions/min_length": 151.0,
-      "completions/min_terminated_length": 151.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1357.0,
+      "completions/mean_length": 572.3984375,
+      "completions/mean_terminated_length": 569.5107421875,
+      "completions/min_length": 126.0,
+      "completions/min_terminated_length": 126.0,
       "epoch": 0.3393776583837027,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00500054399657516,
-      "kl": 0.04815673828125,
+      "grad_norm": 0.002935718694507386,
+      "kl": 0.00739288330078125,
       "learning_rate": 1.1667577289579462e-06,
-      "loss": 0.0005,
-      "num_tokens": 227361851.0,
+      "loss": 0.0001,
+      "num_tokens": 225800349.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10627,20 +10627,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1342.0,
-      "completions/max_terminated_length": 1342.0,
-      "completions/mean_length": 572.224609375,
-      "completions/mean_terminated_length": 572.224609375,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1268.0,
+      "completions/mean_length": 584.03515625,
+      "completions/mean_terminated_length": 581.1702270507812,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.3402731139467204,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005051424307733384,
-      "kl": 0.0479736328125,
+      "grad_norm": 0.0020063036346906656,
+      "kl": 0.00696563720703125,
       "learning_rate": 1.1564354154746007e-06,
-      "loss": 0.0005,
-      "num_tokens": 227963246.0,
+      "loss": 0.0001,
+      "num_tokens": 226407791.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10656,19 +10656,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1781.0,
-      "completions/max_terminated_length": 1781.0,
-      "completions/mean_length": 543.689453125,
-      "completions/mean_terminated_length": 541.678466796875,
-      "completions/min_length": 162.0,
-      "completions/min_terminated_length": 162.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1583.0,
+      "completions/mean_length": 567.478515625,
+      "completions/mean_terminated_length": 561.672607421875,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.34116856950973806,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.2170304837306363,
-      "kl": 0.05426025390625,
+      "grad_norm": 0.05304798394475104,
+      "kl": 0.00707244873046875,
       "learning_rate": 1.146179964769635e-06,
-      "loss": 0.0025,
-      "num_tokens": 228535055.0,
+      "loss": 0.0112,
+      "num_tokens": 226991780.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -10684,19 +10684,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1515.0,
-      "completions/max_terminated_length": 1515.0,
-      "completions/mean_length": 574.69921875,
-      "completions/mean_terminated_length": 574.69921875,
-      "completions/min_length": 242.0,
-      "completions/min_terminated_length": 242.0,
+      "completions/max_length": 1395.0,
+      "completions/max_terminated_length": 1395.0,
+      "completions/mean_length": 587.37890625,
+      "completions/mean_terminated_length": 587.37890625,
+      "completions/min_length": 227.0,
+      "completions/min_terminated_length": 227.0,
       "epoch": 0.34206402507275574,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004686795250678084,
-      "kl": 0.04437255859375,
+      "grad_norm": 0.0017700420725100794,
+      "kl": 0.0064697265625,
       "learning_rate": 1.1359918071412195e-06,
-      "loss": 0.0004,
-      "num_tokens": 229117413.0,
+      "loss": 0.0001,
+      "num_tokens": 227580630.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10711,26 +10711,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1461.0,
-      "completions/mean_length": 581.048828125,
-      "completions/mean_terminated_length": 576.4617309570312,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1330.0,
+      "completions/max_terminated_length": 1330.0,
+      "completions/mean_length": 574.92578125,
+      "completions/mean_terminated_length": 574.92578125,
+      "completions/min_length": 207.0,
+      "completions/min_terminated_length": 207.0,
       "epoch": 0.3429594806357735,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 1.1783978547689393,
-      "kl": 0.05865478515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002522387931367739,
+      "kl": 0.00714111328125,
       "learning_rate": 1.1258713700640456e-06,
-      "loss": 0.014,
-      "num_tokens": 229698558.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 228158640.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 383
     },
     {
@@ -10740,19 +10740,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1769.0,
-      "completions/max_terminated_length": 1769.0,
-      "completions/mean_length": 570.1640625,
-      "completions/mean_terminated_length": 569.1859130859375,
-      "completions/min_length": 138.0,
-      "completions/min_terminated_length": 138.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1440.0,
+      "completions/mean_length": 576.966796875,
+      "completions/mean_terminated_length": 574.0880737304688,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.34385493619879115,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01024322824980028,
-      "kl": 0.0523681640625,
+      "grad_norm": 0.0018472409757711492,
+      "kl": 0.00701141357421875,
       "learning_rate": 1.115819078171383e-06,
-      "loss": 0.0005,
-      "num_tokens": 230277586.0,
+      "loss": 0.0001,
+      "num_tokens": 228741151.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10768,19 +10768,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1455.0,
-      "completions/max_terminated_length": 1455.0,
-      "completions/mean_length": 583.599609375,
-      "completions/mean_terminated_length": 583.599609375,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_length": 1622.0,
+      "completions/max_terminated_length": 1622.0,
+      "completions/mean_length": 585.87109375,
+      "completions/mean_terminated_length": 585.87109375,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.3447503917618088,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00435591430534266,
-      "kl": 0.04876708984375,
+      "grad_norm": 0.0019917553098233345,
+      "kl": 0.00733184814453125,
       "learning_rate": 1.1058353532372667e-06,
-      "loss": 0.0005,
-      "num_tokens": 230868677.0,
+      "loss": 0.0001,
+      "num_tokens": 229333405.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10795,26 +10795,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1716.0,
-      "completions/max_terminated_length": 1716.0,
-      "completions/mean_length": 563.646484375,
-      "completions/mean_terminated_length": 559.092529296875,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1603.0,
+      "completions/max_terminated_length": 1603.0,
+      "completions/mean_length": 579.091796875,
+      "completions/mean_terminated_length": 579.091796875,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.3456458473248265,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0644270242005122,
-      "kl": 0.04962158203125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.02362205706260728,
+      "kl": 0.0114593505859375,
       "learning_rate": 1.0959206141587998e-06,
-      "loss": 0.0125,
-      "num_tokens": 231472384.0,
-      "reward": 0.09921875596046448,
-      "reward_std": 0.0013975425390526652,
+      "loss": 0.0001,
+      "num_tokens": 229945020.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.9921875,
-      "rewards/format_reward/std": 0.08812850713729858,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 386
     },
     {
@@ -10824,19 +10824,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1587.0,
-      "completions/max_terminated_length": 1587.0,
-      "completions/mean_length": 573.7109375,
-      "completions/mean_terminated_length": 573.7109375,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 1386.0,
+      "completions/max_terminated_length": 1386.0,
+      "completions/mean_length": 579.376953125,
+      "completions/mean_terminated_length": 579.376953125,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.3465413028878442,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0043769719826980726,
-      "kl": 0.0465087890625,
+      "grad_norm": 0.00198843994240326,
+      "kl": 0.0070648193359375,
       "learning_rate": 1.0860752769385766e-06,
-      "loss": 0.0005,
-      "num_tokens": 232054892.0,
+      "loss": 0.0001,
+      "num_tokens": 230530429.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10851,26 +10851,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1421.0,
-      "completions/max_terminated_length": 1421.0,
-      "completions/mean_length": 625.736328125,
-      "completions/mean_terminated_length": 625.0215454101562,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1863.0,
+      "completions/max_terminated_length": 1863.0,
+      "completions/mean_length": 632.87109375,
+      "completions/mean_terminated_length": 632.87109375,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.34743675845086186,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004764854587022044,
-      "kl": 0.04669189453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.06881464441737437,
+      "kl": 0.00665283203125,
       "learning_rate": 1.0762997546672279e-06,
-      "loss": 0.0005,
-      "num_tokens": 232676021.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.01,
+      "num_tokens": 231155211.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 388
     },
     {
@@ -10879,20 +10879,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1553.0,
-      "completions/max_terminated_length": 1553.0,
-      "completions/mean_length": 556.107421875,
-      "completions/mean_terminated_length": 554.7964477539062,
-      "completions/min_length": 142.0,
-      "completions/min_terminated_length": 142.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1730.0,
+      "completions/max_terminated_length": 1730.0,
+      "completions/mean_length": 556.984375,
+      "completions/mean_terminated_length": 556.984375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.34833221401387954,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.019412253728347505,
-      "kl": 0.055419921875,
+      "grad_norm": 0.003016609752120695,
+      "kl": 0.00732421875,
       "learning_rate": 1.0665944575060914e-06,
-      "loss": 0.0006,
-      "num_tokens": 233269340.0,
+      "loss": 0.0001,
+      "num_tokens": 231748979.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10907,20 +10907,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1777.0,
-      "completions/mean_length": 598.857421875,
-      "completions/mean_terminated_length": 596.0215454101562,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1937.0,
+      "completions/max_terminated_length": 1937.0,
+      "completions/mean_length": 608.6484375,
+      "completions/mean_terminated_length": 608.6484375,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.34922766957689727,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004042710509275122,
-      "kl": 0.04693603515625,
+      "grad_norm": 0.0022898321321825014,
+      "kl": 0.007659912109375,
       "learning_rate": 1.056959792669997e-06,
-      "loss": 0.0005,
-      "num_tokens": 233893091.0,
+      "loss": 0.0001,
+      "num_tokens": 232377743.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10935,26 +10935,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1739.0,
-      "completions/max_terminated_length": 1739.0,
-      "completions/mean_length": 600.2421875,
-      "completions/mean_terminated_length": 600.2421875,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1715.0,
+      "completions/mean_length": 600.947265625,
+      "completions/mean_terminated_length": 595.2725830078125,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
       "epoch": 0.35012312513991495,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004273522818328221,
-      "kl": 0.046630859375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07120278792436081,
+      "kl": 0.007080078125,
       "learning_rate": 1.0473961644101856e-06,
-      "loss": 0.0005,
-      "num_tokens": 234541871.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0087,
+      "num_tokens": 233026884.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 391
     },
     {
@@ -10963,20 +10963,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1491.0,
-      "completions/max_terminated_length": 1491.0,
-      "completions/mean_length": 619.369140625,
-      "completions/mean_terminated_length": 618.6966552734375,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1701.0,
+      "completions/max_terminated_length": 1701.0,
+      "completions/mean_length": 617.69921875,
+      "completions/mean_terminated_length": 617.69921875,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
       "epoch": 0.3510185807029326,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0040486666283392125,
-      "kl": 0.04632568359375,
+      "grad_norm": 0.002037737372292383,
+      "kl": 0.0066375732421875,
       "learning_rate": 1.037903973997345e-06,
-      "loss": 0.0005,
-      "num_tokens": 235197980.0,
+      "loss": 0.0001,
+      "num_tokens": 233682138.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -10991,26 +10991,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
+      "completions/clipped_ratio": -6.953125,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1881.0,
-      "completions/mean_length": 592.013671875,
-      "completions/mean_terminated_length": 586.303955078125,
-      "completions/min_length": 197.0,
-      "completions/min_terminated_length": 197.0,
+      "completions/max_terminated_length": 1986.0,
+      "completions/mean_length": 600.583984375,
+      "completions/mean_terminated_length": 594.0628662109375,
+      "completions/min_length": 130.0,
+      "completions/min_terminated_length": 130.0,
       "epoch": 0.3519140362659503,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.07832944286109411,
-      "kl": 0.044677734375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 7.623259217540254,
+      "kl": 0.02263641357421875,
       "learning_rate": 1.0284836197047737e-06,
-      "loss": 0.0094,
-      "num_tokens": 235822211.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0048,
+      "num_tokens": 234310757.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 393
     },
     {
@@ -11020,19 +11020,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1323.0,
-      "completions/max_terminated_length": 1323.0,
-      "completions/mean_length": 555.65625,
-      "completions/mean_terminated_length": 555.65625,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_length": 1556.0,
+      "completions/max_terminated_length": 1556.0,
+      "completions/mean_length": 554.560546875,
+      "completions/mean_terminated_length": 554.560546875,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.352809491828968,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.010148834998323347,
-      "kl": 0.04815673828125,
+      "grad_norm": 0.0028379250785953714,
+      "kl": 0.0075531005859375,
       "learning_rate": 1.0191354967916712e-06,
-      "loss": 0.0005,
-      "num_tokens": 236407043.0,
+      "loss": 0.0001,
+      "num_tokens": 234895028.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11048,25 +11048,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1373.0,
-      "completions/max_terminated_length": 1373.0,
-      "completions/mean_length": 587.875,
-      "completions/mean_terminated_length": 586.9177856445312,
-      "completions/min_length": 140.0,
-      "completions/min_terminated_length": 140.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1442.0,
+      "completions/mean_length": 592.939453125,
+      "completions/mean_terminated_length": 590.0919799804688,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.35370494739198566,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.9869550657623536,
-      "kl": 0.27435302734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002487976285381569,
+      "kl": 0.00714111328125,
       "learning_rate": 1.0098599974865515e-06,
-      "loss": 0.013,
-      "num_tokens": 237002851.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 235493429.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 395
     },
     {
@@ -11076,19 +11076,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1436.0,
-      "completions/max_terminated_length": 1436.0,
-      "completions/mean_length": 601.15234375,
-      "completions/mean_terminated_length": 601.15234375,
-      "completions/min_length": 246.0,
-      "completions/min_terminated_length": 246.0,
+      "completions/max_length": 1766.0,
+      "completions/max_terminated_length": 1766.0,
+      "completions/mean_length": 620.3359375,
+      "completions/mean_terminated_length": 620.3359375,
+      "completions/min_length": 221.0,
+      "completions/min_terminated_length": 221.0,
       "epoch": 0.35460040295500334,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004630268582371048,
-      "kl": 0.0477294921875,
+      "grad_norm": 0.0019206177258053912,
+      "kl": 0.0068206787109375,
       "learning_rate": 1.0006575109707898e-06,
-      "loss": 0.0005,
-      "num_tokens": 237636081.0,
+      "loss": 0.0001,
+      "num_tokens": 236136481.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11104,19 +11104,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1576.0,
-      "completions/max_terminated_length": 1576.0,
-      "completions/mean_length": 572.310546875,
-      "completions/mean_terminated_length": 572.310546875,
-      "completions/min_length": 160.0,
-      "completions/min_terminated_length": 160.0,
+      "completions/max_length": 1601.0,
+      "completions/max_terminated_length": 1601.0,
+      "completions/mean_length": 569.744140625,
+      "completions/mean_terminated_length": 569.744140625,
+      "completions/min_length": 133.0,
+      "completions/min_terminated_length": 133.0,
       "epoch": 0.35549585851802107,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004840189114307491,
-      "kl": 0.04644775390625,
+      "grad_norm": 0.002326401178225782,
+      "kl": 0.00687408447265625,
       "learning_rate": 9.915284233622877e-07,
-      "loss": 0.0005,
-      "num_tokens": 238245616.0,
+      "loss": 0.0001,
+      "num_tokens": 236744702.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11132,19 +11132,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1507.0,
-      "completions/max_terminated_length": 1507.0,
-      "completions/mean_length": 587.345703125,
-      "completions/mean_terminated_length": 587.345703125,
-      "completions/min_length": 132.0,
-      "completions/min_terminated_length": 132.0,
+      "completions/max_length": 1616.0,
+      "completions/max_terminated_length": 1616.0,
+      "completions/mean_length": 577.216796875,
+      "completions/mean_terminated_length": 577.216796875,
+      "completions/min_length": 117.0,
+      "completions/min_terminated_length": 117.0,
       "epoch": 0.35639131408103875,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003941142747216441,
-      "kl": 0.0457763671875,
+      "grad_norm": 0.0030862369767653917,
+      "kl": 0.006591796875,
       "learning_rate": 9.824731176992796e-07,
-      "loss": 0.0005,
-      "num_tokens": 238812033.0,
+      "loss": 0.0001,
+      "num_tokens": 237305933.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11159,26 +11159,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1750.0,
-      "completions/max_terminated_length": 1750.0,
-      "completions/mean_length": 608.267578125,
-      "completions/mean_terminated_length": 608.267578125,
-      "completions/min_length": 188.0,
-      "completions/min_terminated_length": 188.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1612.0,
+      "completions/mean_length": 593.181640625,
+      "completions/mean_terminated_length": 587.4765014648438,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.3572867696440564,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0051519468305786,
-      "kl": 0.04791259765625,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.11199865549877461,
+      "kl": 0.0068817138671875,
       "learning_rate": 9.734919739242543e-07,
-      "loss": 0.0005,
-      "num_tokens": 239426762.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0132,
+      "num_tokens": 237912938.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 399
     },
     {
@@ -11188,19 +11188,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1857.0,
-      "completions/max_terminated_length": 1857.0,
-      "completions/mean_length": 608.357421875,
-      "completions/mean_terminated_length": 608.357421875,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/max_length": 1449.0,
+      "completions/max_terminated_length": 1449.0,
+      "completions/mean_length": 616.73828125,
+      "completions/mean_terminated_length": 616.73828125,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.3581822252070741,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0039636086255187275,
-      "kl": 0.04779052734375,
+      "grad_norm": 0.002015596791143499,
+      "kl": 0.00691986083984375,
       "learning_rate": 9.645853688680177e-07,
-      "loss": 0.0005,
-      "num_tokens": 240065009.0,
+      "loss": 0.0001,
+      "num_tokens": 238555476.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11216,19 +11216,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1195.0,
-      "completions/max_terminated_length": 1195.0,
-      "completions/mean_length": 566.44921875,
-      "completions/mean_terminated_length": 566.44921875,
-      "completions/min_length": 148.0,
-      "completions/min_terminated_length": 148.0,
+      "completions/max_length": 1433.0,
+      "completions/max_terminated_length": 1433.0,
+      "completions/mean_length": 559.072265625,
+      "completions/mean_terminated_length": 559.072265625,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.3590776807700918,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0038272234301605793,
-      "kl": 0.044677734375,
+      "grad_norm": 0.0018608514139559509,
+      "kl": 0.0065155029296875,
       "learning_rate": 9.557536762338786e-07,
-      "loss": 0.0004,
-      "num_tokens": 240641927.0,
+      "loss": 0.0001,
+      "num_tokens": 239128617.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11244,19 +11244,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1314.0,
-      "completions/max_terminated_length": 1314.0,
-      "completions/mean_length": 572.609375,
-      "completions/mean_terminated_length": 571.1585083007812,
-      "completions/min_length": 175.0,
-      "completions/min_terminated_length": 175.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1995.0,
+      "completions/mean_length": 571.732421875,
+      "completions/mean_terminated_length": 568.8434448242188,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.35997313633310946,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 2.830266598053953,
-      "kl": 0.78790283203125,
+      "grad_norm": 0.0024477307731748934,
+      "kl": 0.0071258544921875,
       "learning_rate": 9.46997266581973e-07,
-      "loss": 0.0079,
-      "num_tokens": 241219695.0,
+      "loss": 0.0001,
+      "num_tokens": 239705936.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11272,25 +11272,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1621.0,
-      "completions/max_terminated_length": 1621.0,
-      "completions/mean_length": 601.609375,
-      "completions/mean_terminated_length": 600.4070434570312,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1932.0,
+      "completions/mean_length": 626.830078125,
+      "completions/mean_terminated_length": 624.0488891601562,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.36086859189612713,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 2.0982466909243627,
-      "kl": 0.50506591796875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.09673933475786087,
+      "kl": 0.0070343017578125,
       "learning_rate": 9.383165073137115e-07,
-      "loss": 0.0051,
-      "num_tokens": 241828759.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0075,
+      "num_tokens": 240327913.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 403
     },
     {
@@ -11299,26 +11299,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1362.0,
-      "completions/max_terminated_length": 1362.0,
-      "completions/mean_length": 592.77734375,
-      "completions/mean_terminated_length": 592.77734375,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1525.0,
+      "completions/max_terminated_length": 1525.0,
+      "completions/mean_length": 591.482421875,
+      "completions/mean_terminated_length": 590.0509033203125,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.36176404745914487,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00595576175026029,
-      "kl": 0.04461669921875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.05941957788832946,
+      "kl": 0.01541900634765625,
       "learning_rate": 9.297117626563687e-07,
-      "loss": 0.0004,
-      "num_tokens": 242427573.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0002,
+      "num_tokens": 240926064.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 404
     },
     {
@@ -11327,26 +11327,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1312.0,
-      "completions/mean_length": 591.677734375,
-      "completions/mean_terminated_length": 585.5462036132812,
-      "completions/min_length": 235.0,
-      "completions/min_terminated_length": 235.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1363.0,
+      "completions/max_terminated_length": 1363.0,
+      "completions/mean_length": 601.333984375,
+      "completions/mean_terminated_length": 601.333984375,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.36265950302216254,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.15349545116743918,
-      "kl": 0.0850830078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.006173233714714381,
+      "kl": 0.006805419921875,
       "learning_rate": 9.211833936477957e-07,
-      "loss": 0.0262,
-      "num_tokens": 243010848.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 241514283.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 405
     },
     {
@@ -11355,26 +11355,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1596.0,
-      "completions/max_terminated_length": 1596.0,
-      "completions/mean_length": 592.9375,
-      "completions/mean_terminated_length": 591.60986328125,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1885.0,
+      "completions/max_terminated_length": 1885.0,
+      "completions/mean_length": 584.126953125,
+      "completions/mean_terminated_length": 584.126953125,
+      "completions/min_length": 122.0,
+      "completions/min_terminated_length": 122.0,
       "epoch": 0.3635549585851802,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.16264218504894148,
-      "kl": 0.112060546875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.054228472972914075,
+      "kl": 0.00666046142578125,
       "learning_rate": 9.127317581212753e-07,
-      "loss": 0.0011,
-      "num_tokens": 243610144.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0001,
+      "num_tokens": 242109068.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 406
     },
     {
@@ -11383,20 +11383,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1240.0,
-      "completions/max_terminated_length": 1240.0,
-      "completions/mean_length": 571.162109375,
-      "completions/mean_terminated_length": 570.0997924804688,
-      "completions/min_length": 241.0,
-      "completions/min_terminated_length": 241.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1264.0,
+      "completions/max_terminated_length": 1264.0,
+      "completions/mean_length": 570.783203125,
+      "completions/mean_terminated_length": 570.783203125,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
       "epoch": 0.3644504141481979,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.11354577971589774,
-      "kl": 0.08062744140625,
+      "grad_norm": 0.001946101119368566,
+      "kl": 0.00690460205078125,
       "learning_rate": 9.043572106905084e-07,
-      "loss": 0.0008,
-      "num_tokens": 244225875.0,
+      "loss": 0.0001,
+      "num_tokens": 242724605.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11412,19 +11412,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1684.0,
-      "completions/max_terminated_length": 1684.0,
-      "completions/mean_length": 583.146484375,
-      "completions/mean_terminated_length": 583.146484375,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/max_length": 1845.0,
+      "completions/max_terminated_length": 1845.0,
+      "completions/mean_length": 587.302734375,
+      "completions/mean_terminated_length": 587.302734375,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.3653458697112156,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.008619318026623175,
-      "kl": 0.04681396484375,
+      "grad_norm": 0.0019191158558493225,
+      "kl": 0.007293701171875,
       "learning_rate": 8.960601027347321e-07,
-      "loss": 0.0005,
-      "num_tokens": 244830526.0,
+      "loss": 0.0001,
+      "num_tokens": 243331384.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11440,19 +11440,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1284.0,
-      "completions/max_terminated_length": 1284.0,
-      "completions/mean_length": 559.591796875,
-      "completions/mean_terminated_length": 559.591796875,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/max_length": 1259.0,
+      "completions/max_terminated_length": 1259.0,
+      "completions/mean_length": 550.9140625,
+      "completions/mean_terminated_length": 550.9140625,
+      "completions/min_length": 222.0,
+      "completions/min_terminated_length": 222.0,
       "epoch": 0.36624132527423325,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004150486122063207,
-      "kl": 0.04461669921875,
+      "grad_norm": 0.002127355767792274,
+      "kl": 0.00656890869140625,
       "learning_rate": 8.878407823839788e-07,
-      "loss": 0.0004,
-      "num_tokens": 245402845.0,
+      "loss": 0.0001,
+      "num_tokens": 243899260.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11467,20 +11467,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1663.0,
-      "completions/mean_length": 618.41796875,
-      "completions/mean_terminated_length": 614.60986328125,
-      "completions/min_length": 174.0,
-      "completions/min_terminated_length": 174.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1719.0,
+      "completions/max_terminated_length": 1719.0,
+      "completions/mean_length": 598.248046875,
+      "completions/mean_terminated_length": 598.248046875,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.36713678083725093,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.21832381676817567,
-      "kl": 0.13763427734375,
+      "grad_norm": 0.052649635615735615,
+      "kl": 0.0068817138671875,
       "learning_rate": 8.796995945044689e-07,
-      "loss": 0.0153,
-      "num_tokens": 246025779.0,
+      "loss": 0.0003,
+      "num_tokens": 244511867.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -11495,26 +11495,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1534.0,
-      "completions/mean_length": 598.365234375,
-      "completions/mean_terminated_length": 592.680419921875,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1455.0,
+      "completions/max_terminated_length": 1455.0,
+      "completions/mean_length": 605.73828125,
+      "completions/mean_terminated_length": 605.73828125,
+      "completions/min_length": 140.0,
+      "completions/min_terminated_length": 140.0,
       "epoch": 0.36803223640026866,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.25126353697075565,
-      "kl": 0.16729736328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0023921493243053795,
+      "kl": 0.00725555419921875,
       "learning_rate": 8.716368806841405e-07,
-      "loss": 0.0146,
-      "num_tokens": 246641358.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 245131221.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 411
     },
     {
@@ -11523,26 +11523,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1309.0,
-      "completions/max_terminated_length": 1309.0,
-      "completions/mean_length": 574.302734375,
-      "completions/mean_terminated_length": 574.302734375,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1290.0,
+      "completions/mean_length": 578.29296875,
+      "completions/mean_terminated_length": 575.4168090820312,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.36892769196328634,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004008716988509208,
-      "kl": 0.0467529296875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07554786393229868,
+      "kl": 0.00659942626953125,
       "learning_rate": 8.636529792183171e-07,
-      "loss": 0.0005,
-      "num_tokens": 247228921.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0104,
+      "num_tokens": 245720827.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 412
     },
     {
@@ -11551,26 +11551,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
+      "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1640.0,
-      "completions/mean_length": 584.1484375,
-      "completions/mean_terminated_length": 581.2837524414062,
-      "completions/min_length": 183.0,
-      "completions/min_terminated_length": 183.0,
+      "completions/max_terminated_length": 1502.0,
+      "completions/mean_length": 595.05859375,
+      "completions/mean_terminated_length": 590.431396484375,
+      "completions/min_length": 146.0,
+      "completions/min_terminated_length": 146.0,
       "epoch": 0.369823147526304,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04699799709615979,
-      "kl": 0.0460205078125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.019885190989181234,
+      "kl": 0.01816558837890625,
       "learning_rate": 8.557482250955144e-07,
-      "loss": -0.0042,
-      "num_tokens": 247835893.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 246333385.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 413
     },
     {
@@ -11579,20 +11579,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1638.0,
-      "completions/max_terminated_length": 1638.0,
-      "completions/mean_length": 612.640625,
-      "completions/mean_terminated_length": 611.1076049804688,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1871.0,
+      "completions/max_terminated_length": 1871.0,
+      "completions/mean_length": 607.44140625,
+      "completions/mean_terminated_length": 607.44140625,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.3707186030893217,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004123288560446476,
-      "kl": 0.048583984375,
+      "grad_norm": 0.002526845240160502,
+      "kl": 0.0074310302734375,
       "learning_rate": 8.479229499833844e-07,
-      "loss": 0.0005,
-      "num_tokens": 248473933.0,
+      "loss": 0.0001,
+      "num_tokens": 246968763.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11607,20 +11607,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1199.0,
-      "completions/max_terminated_length": 1199.0,
-      "completions/mean_length": 560.51953125,
-      "completions/mean_terminated_length": 559.4755249023438,
-      "completions/min_length": 154.0,
-      "completions/min_terminated_length": 154.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1883.0,
+      "completions/max_terminated_length": 1883.0,
+      "completions/mean_length": 569.833984375,
+      "completions/mean_terminated_length": 569.833984375,
+      "completions/min_length": 136.0,
+      "completions/min_terminated_length": 136.0,
       "epoch": 0.3716140586523394,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.011763473343105632,
-      "kl": 0.05010986328125,
+      "grad_norm": 0.00203404503319221,
+      "kl": 0.00699615478515625,
       "learning_rate": 8.401774822147976e-07,
-      "loss": 0.0005,
-      "num_tokens": 248995639.0,
+      "loss": 0.0001,
+      "num_tokens": 247495238.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11636,19 +11636,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1477.0,
-      "completions/max_terminated_length": 1477.0,
-      "completions/mean_length": 575.220703125,
-      "completions/mean_terminated_length": 575.220703125,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 1630.0,
+      "completions/max_terminated_length": 1630.0,
+      "completions/mean_length": 587.775390625,
+      "completions/mean_terminated_length": 587.775390625,
+      "completions/min_length": 202.0,
+      "completions/min_terminated_length": 202.0,
       "epoch": 0.37250951421535705,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004097140634106235,
-      "kl": 0.04595947265625,
+      "grad_norm": 0.002779264425300328,
+      "kl": 0.00716400146484375,
       "learning_rate": 8.325121467740695e-07,
-      "loss": 0.0005,
-      "num_tokens": 249611144.0,
+      "loss": 0.0001,
+      "num_tokens": 248117171.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11663,20 +11663,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1443.0,
-      "completions/mean_length": 585.287109375,
-      "completions/mean_terminated_length": 582.4246826171875,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1172.0,
+      "completions/max_terminated_length": 1172.0,
+      "completions/mean_length": 575.365234375,
+      "completions/mean_terminated_length": 575.365234375,
+      "completions/min_length": 143.0,
+      "completions/min_terminated_length": 143.0,
       "epoch": 0.37340496977837473,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004884809695062555,
-      "kl": 0.0440673828125,
+      "grad_norm": 0.0018850938887697785,
+      "kl": 0.00659942626953125,
       "learning_rate": 8.249272652833226e-07,
-      "loss": 0.0004,
-      "num_tokens": 250236827.0,
+      "loss": 0.0001,
+      "num_tokens": 248737774.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11692,25 +11692,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1809.0,
-      "completions/max_terminated_length": 1809.0,
-      "completions/mean_length": 554.861328125,
-      "completions/mean_terminated_length": 554.861328125,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/max_length": 1624.0,
+      "completions/max_terminated_length": 1624.0,
+      "completions/mean_length": 543.0390625,
+      "completions/mean_terminated_length": 543.0390625,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.3743004253413924,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004598838734909383,
-      "kl": 0.0479736328125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07549294657539767,
+      "kl": 0.0076904296875,
       "learning_rate": 8.174231559889931e-07,
-      "loss": 0.0005,
-      "num_tokens": 250816820.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0014,
+      "num_tokens": 249311714.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 418
     },
     {
@@ -11720,19 +11720,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1482.0,
-      "completions/max_terminated_length": 1482.0,
-      "completions/mean_length": 541.486328125,
-      "completions/mean_terminated_length": 541.486328125,
-      "completions/min_length": 194.0,
-      "completions/min_terminated_length": 194.0,
+      "completions/max_length": 1836.0,
+      "completions/max_terminated_length": 1836.0,
+      "completions/mean_length": 528.9765625,
+      "completions/mean_terminated_length": 528.9765625,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.37519588090441014,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004336303560681598,
-      "kl": 0.04498291015625,
+      "grad_norm": 0.00231041407725594,
+      "kl": 0.00714111328125,
       "learning_rate": 8.100001337484787e-07,
-      "loss": 0.0004,
-      "num_tokens": 251358157.0,
+      "loss": 0.0001,
+      "num_tokens": 249846646.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11747,26 +11747,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1565.0,
-      "completions/max_terminated_length": 1565.0,
-      "completions/mean_length": 575.189453125,
-      "completions/mean_terminated_length": 573.25244140625,
-      "completions/min_length": 245.0,
-      "completions/min_terminated_length": 245.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1219.0,
+      "completions/max_terminated_length": 1219.0,
+      "completions/mean_length": 559.14453125,
+      "completions/mean_terminated_length": 559.14453125,
+      "completions/min_length": 150.0,
+      "completions/min_terminated_length": 150.0,
       "epoch": 0.3760913364674278,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04522967734559762,
-      "kl": 0.05145263671875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0024977967915594834,
+      "kl": 0.00714874267578125,
       "learning_rate": 8.026585100169251e-07,
-      "loss": 0.0003,
-      "num_tokens": 251974558.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 250454832.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 420
     },
     {
@@ -11776,25 +11776,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1519.0,
-      "completions/max_terminated_length": 1519.0,
-      "completions/mean_length": 600.025390625,
-      "completions/mean_terminated_length": 599.1056518554688,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1959.0,
+      "completions/mean_length": 621.669921875,
+      "completions/mean_terminated_length": 618.878662109375,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.3769867920304455,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.14856992304676514,
-      "kl": 0.12725830078125,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 0.06166413332258108,
+      "kl": 0.0085296630859375,
       "learning_rate": 7.953985928341601e-07,
-      "loss": 0.0013,
-      "num_tokens": 252602715.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.012,
+      "num_tokens": 251094071.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 421
     },
     {
@@ -11804,19 +11804,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1414.0,
-      "completions/max_terminated_length": 1414.0,
-      "completions/mean_length": 600.701171875,
-      "completions/mean_terminated_length": 600.701171875,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/max_length": 1575.0,
+      "completions/max_terminated_length": 1575.0,
+      "completions/mean_length": 600.337890625,
+      "completions/mean_terminated_length": 600.337890625,
+      "completions/min_length": 154.0,
+      "completions/min_terminated_length": 154.0,
       "epoch": 0.37788224759346317,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004152663083082165,
-      "kl": 0.04595947265625,
+      "grad_norm": 0.0024472020636609228,
+      "kl": 0.00649261474609375,
       "learning_rate": 7.882206868117693e-07,
-      "loss": 0.0005,
-      "num_tokens": 253209602.0,
+      "loss": 0.0001,
+      "num_tokens": 251700772.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11831,26 +11831,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1494.0,
-      "completions/max_terminated_length": 1494.0,
-      "completions/mean_length": 585.802734375,
-      "completions/mean_terminated_length": 585.802734375,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1759.0,
+      "completions/mean_length": 579.787109375,
+      "completions/mean_terminated_length": 576.9138793945312,
+      "completions/min_length": 182.0,
+      "completions/min_terminated_length": 182.0,
       "epoch": 0.37877770315648085,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06489189022575756,
-      "kl": 0.0447998046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0025695653842804063,
+      "kl": 0.0072021484375,
       "learning_rate": 7.81125093120313e-07,
-      "loss": 0.0013,
-      "num_tokens": 253831053.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 252319143.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 423
     },
     {
@@ -11859,20 +11859,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1164.0,
-      "completions/max_terminated_length": 1164.0,
-      "completions/mean_length": 569.109375,
-      "completions/mean_terminated_length": 568.2015380859375,
-      "completions/min_length": 256.0,
-      "completions/min_terminated_length": 256.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1465.0,
+      "completions/max_terminated_length": 1465.0,
+      "completions/mean_length": 563.19140625,
+      "completions/mean_terminated_length": 563.19140625,
+      "completions/min_length": 251.0,
+      "completions/min_terminated_length": 251.0,
       "epoch": 0.3796731587194985,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.04129359182029064,
-      "kl": 0.06884765625,
+      "grad_norm": 0.0024216522522944213,
+      "kl": 0.00678253173828125,
       "learning_rate": 7.741121094766916e-07,
-      "loss": 0.0007,
-      "num_tokens": 254448773.0,
+      "loss": 0.0001,
+      "num_tokens": 252933833.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11888,19 +11888,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1438.0,
-      "completions/max_terminated_length": 1438.0,
-      "completions/mean_length": 552.451171875,
-      "completions/mean_terminated_length": 552.451171875,
-      "completions/min_length": 128.0,
-      "completions/min_terminated_length": 128.0,
+      "completions/max_length": 1950.0,
+      "completions/max_terminated_length": 1950.0,
+      "completions/mean_length": 542.798828125,
+      "completions/mean_terminated_length": 542.798828125,
+      "completions/min_length": 151.0,
+      "completions/min_terminated_length": 151.0,
       "epoch": 0.3805686142825162,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004063387730342484,
-      "kl": 0.04522705078125,
+      "grad_norm": 0.003995445221210926,
+      "kl": 0.00765228271484375,
       "learning_rate": 7.671820301316532e-07,
-      "loss": 0.0005,
-      "num_tokens": 255008300.0,
+      "loss": 0.0001,
+      "num_tokens": 253488418.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11916,25 +11916,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1368.0,
-      "completions/max_terminated_length": 1368.0,
-      "completions/mean_length": 570.951171875,
-      "completions/mean_terminated_length": 570.951171875,
-      "completions/min_length": 226.0,
-      "completions/min_terminated_length": 226.0,
+      "completions/max_length": 1771.0,
+      "completions/max_terminated_length": 1771.0,
+      "completions/mean_length": 558.029296875,
+      "completions/mean_terminated_length": 558.029296875,
+      "completions/min_length": 230.0,
+      "completions/min_terminated_length": 230.0,
       "epoch": 0.38146406984553394,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05431180203096875,
-      "kl": 0.04412841796875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0030221782126988606,
+      "kl": 0.00658416748046875,
       "learning_rate": 7.603351458574474e-07,
-      "loss": -0.0001,
-      "num_tokens": 255591923.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 254065425.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 426
     },
     {
@@ -11943,26 +11943,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1706.0,
-      "completions/max_terminated_length": 1706.0,
-      "completions/mean_length": 592.3515625,
-      "completions/mean_terminated_length": 592.3515625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/clipped_ratio": -6.953125,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1650.0,
+      "completions/mean_length": 569.091796875,
+      "completions/mean_terminated_length": 561.3104248046875,
+      "completions/min_length": 177.0,
+      "completions/min_terminated_length": 177.0,
       "epoch": 0.3823595254085516,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.021289388295280785,
-      "kl": 0.051513671875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.08739034946246164,
+      "kl": 0.0099334716796875,
       "learning_rate": 7.535717439356255e-07,
-      "loss": 0.0005,
-      "num_tokens": 256274487.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.012,
+      "num_tokens": 254736080.0,
+      "reward": 0.099609375,
+      "reward_std": 0.0010673906654119492,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.99609375,
+      "rewards/format_reward/std": 0.06243881583213806,
       "step": 427
     },
     {
@@ -11971,20 +11971,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1369.0,
-      "completions/max_terminated_length": 1369.0,
-      "completions/mean_length": 569.8515625,
-      "completions/mean_terminated_length": 569.1956787109375,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1306.0,
+      "completions/max_terminated_length": 1306.0,
+      "completions/mean_length": 532.998046875,
+      "completions/mean_terminated_length": 532.998046875,
+      "completions/min_length": 118.0,
+      "completions/min_terminated_length": 118.0,
       "epoch": 0.3832549809715693,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0046493196129765504,
-      "kl": 0.047119140625,
+      "grad_norm": 0.0032836914118878156,
+      "kl": 0.00711822509765625,
       "learning_rate": 7.46892108144986e-07,
-      "loss": 0.0005,
-      "num_tokens": 256854235.0,
+      "loss": 0.0001,
+      "num_tokens": 255296959.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -11999,26 +11999,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1685.0,
-      "completions/max_terminated_length": 1685.0,
-      "completions/mean_length": 594.58203125,
-      "completions/mean_terminated_length": 592.876708984375,
-      "completions/min_length": 216.0,
-      "completions/min_terminated_length": 216.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1549.0,
+      "completions/max_terminated_length": 1549.0,
+      "completions/mean_length": 565.09765625,
+      "completions/mean_terminated_length": 565.09765625,
+      "completions/min_length": 205.0,
+      "completions/min_terminated_length": 205.0,
       "epoch": 0.38415043653458697,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.030024377459490645,
-      "kl": 0.06475830078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.028838282068600722,
+      "kl": 0.0084075927734375,
       "learning_rate": 7.402965187496697e-07,
-      "loss": 0.0006,
-      "num_tokens": 257477957.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": -0.0028,
+      "num_tokens": 255905585.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 429
     },
     {
@@ -12029,24 +12029,24 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.96875,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1847.0,
-      "completions/mean_length": 612.123046875,
-      "completions/mean_terminated_length": 607.6902465820312,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/max_terminated_length": 1784.0,
+      "completions/mean_length": 580.353515625,
+      "completions/mean_terminated_length": 574.5980834960938,
+      "completions/min_length": 187.0,
+      "completions/min_terminated_length": 187.0,
       "epoch": 0.38504589209760465,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.059584893596399784,
-      "kl": 0.0477294921875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0027135705194481804,
+      "kl": 0.006805419921875,
       "learning_rate": 7.337852524873974e-07,
-      "loss": 0.0081,
-      "num_tokens": 258102660.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0010673906654119492,
+      "loss": 0.0001,
+      "num_tokens": 256514022.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 430
     },
     {
@@ -12055,20 +12055,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1323.0,
-      "completions/max_terminated_length": 1323.0,
-      "completions/mean_length": 572.529296875,
-      "completions/mean_terminated_length": 572.529296875,
-      "completions/min_length": 184.0,
-      "completions/min_terminated_length": 184.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1552.0,
+      "completions/mean_length": 555.763671875,
+      "completions/mean_terminated_length": 552.8434448242188,
+      "completions/min_length": 138.0,
+      "completions/min_terminated_length": 138.0,
       "epoch": 0.3859413476606223,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003820837122162048,
-      "kl": 0.04449462890625,
+      "grad_norm": 0.0035836046428584386,
+      "kl": 0.00743865966796875,
       "learning_rate": 7.273585825578608e-07,
-      "loss": 0.0004,
-      "num_tokens": 258666979.0,
+      "loss": 0.0001,
+      "num_tokens": 257069757.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12083,20 +12083,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1434.0,
-      "completions/max_terminated_length": 1434.0,
-      "completions/mean_length": 601.86328125,
-      "completions/mean_terminated_length": 600.2348022460938,
-      "completions/min_length": 209.0,
-      "completions/min_terminated_length": 209.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1635.0,
+      "completions/max_terminated_length": 1635.0,
+      "completions/mean_length": 585.365234375,
+      "completions/mean_terminated_length": 585.365234375,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.38683680322364,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005050065975834865,
-      "kl": 0.050537109375,
+      "grad_norm": 0.0026310922549361436,
+      "kl": 0.00643157958984375,
       "learning_rate": 7.21016778611259e-07,
-      "loss": 0.0005,
-      "num_tokens": 259286461.0,
+      "loss": 0.0001,
+      "num_tokens": 257680792.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12112,19 +12112,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1626.0,
-      "completions/max_terminated_length": 1626.0,
-      "completions/mean_length": 583.451171875,
-      "completions/mean_terminated_length": 583.451171875,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_length": 1261.0,
+      "completions/max_terminated_length": 1261.0,
+      "completions/mean_length": 557.9375,
+      "completions/mean_terminated_length": 557.9375,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.38773225878665774,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003638010107531803,
-      "kl": 0.0433349609375,
+      "grad_norm": 0.00411242161557962,
+      "kl": 0.00650787353515625,
       "learning_rate": 7.147601067369835e-07,
-      "loss": 0.0004,
-      "num_tokens": 259884068.0,
+      "loss": 0.0001,
+      "num_tokens": 258265336.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12139,20 +12139,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1436.0,
-      "completions/max_terminated_length": 1436.0,
-      "completions/mean_length": 620.447265625,
-      "completions/mean_terminated_length": 618.9706420898438,
-      "completions/min_length": 242.0,
-      "completions/min_terminated_length": 242.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1468.0,
+      "completions/max_terminated_length": 1468.0,
+      "completions/mean_length": 583.52734375,
+      "completions/mean_terminated_length": 583.52734375,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.3886277143496754,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0049178729245381745,
-      "kl": 0.04931640625,
+      "grad_norm": 0.0026550287253777143,
+      "kl": 0.00661468505859375,
       "learning_rate": 7.085888294524561e-07,
-      "loss": 0.0005,
-      "num_tokens": 260508361.0,
+      "loss": 0.0001,
+      "num_tokens": 258870726.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12168,19 +12168,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1796.0,
-      "completions/max_terminated_length": 1796.0,
-      "completions/mean_length": 614.025390625,
-      "completions/mean_terminated_length": 614.025390625,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
+      "completions/max_length": 1904.0,
+      "completions/max_terminated_length": 1904.0,
+      "completions/mean_length": 593.146484375,
+      "completions/mean_terminated_length": 593.146484375,
+      "completions/min_length": 198.0,
+      "completions/min_terminated_length": 198.0,
       "epoch": 0.3895231699126931,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0053380131336369435,
-      "kl": 0.04571533203125,
+      "grad_norm": 0.005815059007833509,
+      "kl": 0.00765228271484375,
       "learning_rate": 7.025032056921117e-07,
-      "loss": 0.0005,
-      "num_tokens": 261127078.0,
+      "loss": 0.0001,
+      "num_tokens": 259478753.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12195,20 +12195,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1784.0,
-      "completions/max_terminated_length": 1784.0,
-      "completions/mean_length": 594.244140625,
-      "completions/mean_terminated_length": 594.244140625,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1576.0,
+      "completions/max_terminated_length": 1576.0,
+      "completions/mean_length": 570.83203125,
+      "completions/mean_terminated_length": 569.1976318359375,
+      "completions/min_length": 149.0,
+      "completions/min_terminated_length": 149.0,
       "epoch": 0.39041862547571077,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003692024494965669,
-      "kl": 0.0433349609375,
+      "grad_norm": 0.7255448560508342,
+      "kl": 0.04613494873046875,
       "learning_rate": 6.965034907965349e-07,
-      "loss": 0.0004,
-      "num_tokens": 261753283.0,
+      "loss": 0.0005,
+      "num_tokens": 260092971.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12223,20 +12223,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1207.0,
-      "completions/max_terminated_length": 1207.0,
-      "completions/mean_length": 555.828125,
-      "completions/mean_terminated_length": 554.5538330078125,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1158.0,
+      "completions/max_terminated_length": 1158.0,
+      "completions/mean_length": 552.009765625,
+      "completions/mean_terminated_length": 552.009765625,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.39131408103872845,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.09019199102303466,
-      "kl": 0.09649658203125,
+      "grad_norm": 0.0030963488401591077,
+      "kl": 0.00754547119140625,
       "learning_rate": 6.905899365017462e-07,
-      "loss": 0.001,
-      "num_tokens": 262324347.0,
+      "loss": 0.0001,
+      "num_tokens": 260662080.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12251,26 +12251,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1715.0,
-      "completions/max_terminated_length": 1433.0,
-      "completions/mean_length": 584.7890625,
-      "completions/mean_terminated_length": 581.9176635742188,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1591.0,
+      "completions/max_terminated_length": 1591.0,
+      "completions/mean_length": 578.076171875,
+      "completions/mean_terminated_length": 576.890380859375,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.3922095366017461,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07482831067859354,
-      "kl": 0.05682373046875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.02801037982963157,
+      "kl": 0.0189666748046875,
       "learning_rate": 6.847627909286409e-07,
-      "loss": 0.009,
-      "num_tokens": 262965391.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0002,
+      "num_tokens": 261299687.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 438
     },
     {
@@ -12280,19 +12280,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1697.0,
-      "completions/max_terminated_length": 1697.0,
-      "completions/mean_length": 603.103515625,
-      "completions/mean_terminated_length": 603.103515625,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/max_length": 1346.0,
+      "completions/max_terminated_length": 1346.0,
+      "completions/mean_length": 577.248046875,
+      "completions/mean_terminated_length": 577.248046875,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.3931049921647638,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0036304390520191508,
-      "kl": 0.04461669921875,
+      "grad_norm": 0.0026557993041991315,
+      "kl": 0.00693511962890625,
       "learning_rate": 6.790222985725761e-07,
-      "loss": 0.0004,
-      "num_tokens": 263574116.0,
+      "loss": 0.0001,
+      "num_tokens": 261895174.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12307,20 +12307,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.71875,
-      "completions/max_length": 1304.0,
-      "completions/max_terminated_length": 1304.0,
-      "completions/mean_length": 592.533203125,
-      "completions/mean_terminated_length": 567.8157958984375,
-      "completions/min_length": 185.0,
-      "completions/min_terminated_length": 185.0,
+      "completions/clipped_ratio": -6.75,
+      "completions/max_length": 1197.0,
+      "completions/max_terminated_length": 1197.0,
+      "completions/mean_length": 576.236328125,
+      "completions/mean_terminated_length": 556.211669921875,
+      "completions/min_length": 186.0,
+      "completions/min_terminated_length": 186.0,
       "epoch": 0.39400044772778153,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.19342053013638,
-      "kl": 0.71484375,
+      "grad_norm": 15.334091278669593,
+      "kl": 0.416748046875,
       "learning_rate": 6.733687002931141e-07,
-      "loss": 0.0072,
-      "num_tokens": 264180917.0,
+      "loss": 0.0042,
+      "num_tokens": 262493631.0,
       "reward": 0.09687499701976776,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12335,26 +12335,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1393.0,
-      "completions/max_terminated_length": 1393.0,
-      "completions/mean_length": 589.984375,
-      "completions/mean_terminated_length": 588.74169921875,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1230.0,
+      "completions/max_terminated_length": 1230.0,
+      "completions/mean_length": 571.33203125,
+      "completions/mean_terminated_length": 571.33203125,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.3948959032907992,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07111130745860231,
-      "kl": 0.04693603515625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002444742961490628,
+      "kl": 0.00711822509765625,
       "learning_rate": 6.678022333039158e-07,
-      "loss": 0.0022,
-      "num_tokens": 264768493.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 263071657.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 441
     },
     {
@@ -12363,26 +12363,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1582.0,
-      "completions/max_terminated_length": 1582.0,
-      "completions/mean_length": 626.0546875,
-      "completions/mean_terminated_length": 626.0546875,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1682.0,
+      "completions/mean_length": 620.861328125,
+      "completions/mean_terminated_length": 616.933349609375,
+      "completions/min_length": 178.0,
+      "completions/min_terminated_length": 178.0,
       "epoch": 0.3957913588538169,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.0522103422216749,
-      "kl": 0.0443115234375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.011201392862572901,
+      "kl": 0.014007568359375,
       "learning_rate": 6.623231311627876e-07,
-      "loss": 0.0023,
-      "num_tokens": 265408825.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 263709330.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 442
     },
     {
@@ -12392,25 +12392,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1747.0,
-      "completions/max_terminated_length": 1747.0,
-      "completions/mean_length": 606.671875,
-      "completions/mean_terminated_length": 606.671875,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/max_length": 1731.0,
+      "completions/max_terminated_length": 1731.0,
+      "completions/mean_length": 610.037109375,
+      "completions/mean_terminated_length": 610.037109375,
+      "completions/min_length": 147.0,
+      "completions/min_terminated_length": 147.0,
       "epoch": 0.39668681441683457,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04443605509556234,
-      "kl": 0.044677734375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0021737767712625513,
+      "kl": 0.0066070556640625,
       "learning_rate": 6.569316237618811e-07,
-      "loss": 0.0006,
-      "num_tokens": 266008289.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 264310517.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 443
     },
     {
@@ -12419,26 +12419,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1765.0,
-      "completions/mean_length": 629.169921875,
-      "completions/mean_terminated_length": 626.3933715820312,
-      "completions/min_length": 230.0,
-      "completions/min_terminated_length": 230.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1484.0,
+      "completions/max_terminated_length": 1484.0,
+      "completions/mean_length": 605.30078125,
+      "completions/mean_terminated_length": 605.30078125,
+      "completions/min_length": 214.0,
+      "completions/min_terminated_length": 214.0,
       "epoch": 0.39758226997985224,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.079977521203583,
-      "kl": 0.048095703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.003722959764073968,
+      "kl": 0.00848388671875,
       "learning_rate": 6.516279373180499e-07,
-      "loss": 0.0147,
-      "num_tokens": 266666344.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 264956351.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 444
     },
     {
@@ -12448,19 +12448,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1862.0,
-      "completions/max_terminated_length": 1862.0,
-      "completions/mean_length": 570.20703125,
-      "completions/mean_terminated_length": 570.20703125,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/max_length": 1252.0,
+      "completions/max_terminated_length": 1252.0,
+      "completions/mean_length": 549.498046875,
+      "completions/mean_terminated_length": 549.498046875,
+      "completions/min_length": 162.0,
+      "completions/min_terminated_length": 162.0,
       "epoch": 0.3984777255428699,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00374445862358689,
-      "kl": 0.04571533203125,
+      "grad_norm": 0.00215060811327745,
+      "kl": 0.0074615478515625,
       "learning_rate": 6.464122943633543e-07,
-      "loss": 0.0005,
-      "num_tokens": 267287378.0,
+      "loss": 0.0001,
+      "num_tokens": 265566782.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12475,20 +12475,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1577.0,
-      "completions/max_terminated_length": 1577.0,
-      "completions/mean_length": 603.232421875,
-      "completions/mean_terminated_length": 603.232421875,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1516.0,
+      "completions/max_terminated_length": 1516.0,
+      "completions/mean_length": 594.345703125,
+      "completions/mean_terminated_length": 594.345703125,
+      "completions/min_length": 181.0,
+      "completions/min_terminated_length": 181.0,
       "epoch": 0.3993731811058876,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004544902573679008,
-      "kl": 0.0443115234375,
+      "grad_norm": 0.001984884141250643,
+      "kl": 0.00677490234375,
       "learning_rate": 6.412849137357271e-07,
-      "loss": 0.0004,
-      "num_tokens": 267914873.0,
+      "loss": 0.0001,
+      "num_tokens": 266189727.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12503,20 +12503,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1172.0,
-      "completions/max_terminated_length": 1172.0,
-      "completions/mean_length": 569.06640625,
-      "completions/mean_terminated_length": 569.06640625,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1500.0,
+      "completions/max_terminated_length": 1500.0,
+      "completions/mean_length": 560.974609375,
+      "completions/mean_terminated_length": 559.136962890625,
+      "completions/min_length": 171.0,
+      "completions/min_terminated_length": 171.0,
       "epoch": 0.40026863666890533,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.006639340072597727,
-      "kl": 0.045654296875,
+      "grad_norm": 0.012607005481500596,
+      "kl": 0.0207061767578125,
       "learning_rate": 6.3624601056979e-07,
-      "loss": 0.0005,
-      "num_tokens": 268518443.0,
+      "loss": 0.0002,
+      "num_tokens": 266789154.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12531,26 +12531,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1834.0,
-      "completions/max_terminated_length": 1834.0,
-      "completions/mean_length": 592.236328125,
-      "completions/mean_terminated_length": 592.236328125,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1827.0,
+      "completions/max_terminated_length": 1827.0,
+      "completions/mean_length": 601.62890625,
+      "completions/mean_terminated_length": 600.0332641601562,
+      "completions/min_length": 242.0,
+      "completions/min_terminated_length": 242.0,
       "epoch": 0.401164092231923,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0037418093289070475,
-      "kl": 0.04522705078125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 7.872691267196718,
+      "kl": 0.02170562744140625,
       "learning_rate": 6.312957962878278e-07,
-      "loss": 0.0005,
-      "num_tokens": 269094420.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0116,
+      "num_tokens": 267369940.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 448
     },
     {
@@ -12560,19 +12560,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1501.0,
-      "completions/max_terminated_length": 1501.0,
-      "completions/mean_length": 577.109375,
-      "completions/mean_terminated_length": 577.109375,
-      "completions/min_length": 171.0,
-      "completions/min_terminated_length": 171.0,
+      "completions/max_length": 1537.0,
+      "completions/max_terminated_length": 1537.0,
+      "completions/mean_length": 577.041015625,
+      "completions/mean_terminated_length": 577.041015625,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.4020595477949407,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004226852697815755,
-      "kl": 0.043701171875,
+      "grad_norm": 0.0026289677378162537,
+      "kl": 0.007049560546875,
       "learning_rate": 6.264344785909181e-07,
-      "loss": 0.0004,
-      "num_tokens": 269683676.0,
+      "loss": 0.0001,
+      "num_tokens": 267959161.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12587,20 +12587,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1750.0,
-      "completions/max_terminated_length": 1750.0,
-      "completions/mean_length": 608.45703125,
-      "completions/mean_terminated_length": 606.75927734375,
-      "completions/min_length": 191.0,
-      "completions/min_terminated_length": 191.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 2023.0,
+      "completions/max_terminated_length": 2023.0,
+      "completions/mean_length": 628.451171875,
+      "completions/mean_terminated_length": 628.451171875,
+      "completions/min_length": 245.0,
+      "completions/min_terminated_length": 245.0,
       "epoch": 0.40295500335795836,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004989757138908895,
-      "kl": 0.04827880859375,
+      "grad_norm": 0.0020281249287714726,
+      "kl": 0.0069122314453125,
       "learning_rate": 6.216622614502149e-07,
-      "loss": 0.0005,
-      "num_tokens": 270333542.0,
+      "loss": 0.0001,
+      "num_tokens": 268619264.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12615,20 +12615,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1637.0,
-      "completions/max_terminated_length": 1637.0,
-      "completions/mean_length": 630.955078125,
-      "completions/mean_terminated_length": 628.9862670898438,
-      "completions/min_length": 195.0,
-      "completions/min_terminated_length": 195.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1761.0,
+      "completions/max_terminated_length": 1761.0,
+      "completions/mean_length": 631.857421875,
+      "completions/mean_terminated_length": 631.857421875,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.40385045892097604,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0045917782708162535,
-      "kl": 0.05010986328125,
+      "grad_norm": 0.0019557582531050284,
+      "kl": 0.006744384765625,
       "learning_rate": 6.169793450983916e-07,
-      "loss": 0.0005,
-      "num_tokens": 270970975.0,
+      "loss": 0.0001,
+      "num_tokens": 269257159.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12644,25 +12644,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1564.0,
-      "completions/max_terminated_length": 1564.0,
-      "completions/mean_length": 619.875,
-      "completions/mean_terminated_length": 618.6692504882812,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1748.0,
+      "completions/mean_length": 632.13671875,
+      "completions/mean_terminated_length": 629.365966796875,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
       "epoch": 0.4047459144839937,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004445821112373912,
-      "kl": 0.04986572265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07942781207559449,
+      "kl": 0.0071868896484375,
       "learning_rate": 6.123859260212393e-07,
-      "loss": 0.0005,
-      "num_tokens": 271632015.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0115,
+      "num_tokens": 269924477.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 452
     },
     {
@@ -12672,19 +12672,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1271.0,
-      "completions/max_terminated_length": 1271.0,
-      "completions/mean_length": 571.25,
-      "completions/mean_terminated_length": 571.25,
-      "completions/min_length": 207.0,
-      "completions/min_terminated_length": 207.0,
+      "completions/max_length": 1287.0,
+      "completions/max_terminated_length": 1287.0,
+      "completions/mean_length": 553.845703125,
+      "completions/mean_terminated_length": 553.845703125,
+      "completions/min_length": 180.0,
+      "completions/min_terminated_length": 180.0,
       "epoch": 0.4056413700470114,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0035904161477165997,
-      "kl": 0.045654296875,
+      "grad_norm": 0.002260666693581993,
+      "kl": 0.00655364990234375,
       "learning_rate": 6.07882196949423e-07,
-      "loss": 0.0005,
-      "num_tokens": 272225167.0,
+      "loss": 0.0001,
+      "num_tokens": 270508718.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12700,25 +12700,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1623.0,
-      "completions/max_terminated_length": 1623.0,
-      "completions/mean_length": 607.998046875,
-      "completions/mean_terminated_length": 607.998046875,
-      "completions/min_length": 202.0,
-      "completions/min_terminated_length": 202.0,
+      "completions/max_length": 1669.0,
+      "completions/max_terminated_length": 1669.0,
+      "completions/mean_length": 605.912109375,
+      "completions/mean_terminated_length": 605.912109375,
+      "completions/min_length": 204.0,
+      "completions/min_terminated_length": 204.0,
       "epoch": 0.40653682561002913,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.047268762318116685,
-      "kl": 0.043701171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002447637498558581,
+      "kl": 0.00658416748046875,
       "learning_rate": 6.034683468503948e-07,
-      "loss": -0.0001,
-      "num_tokens": 272816494.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 271098977.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 454
     },
     {
@@ -12727,22 +12727,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.75,
-      "completions/max_length": 1386.0,
-      "completions/max_terminated_length": 1386.0,
-      "completions/mean_length": 616.38671875,
-      "completions/mean_terminated_length": 591.7217407226562,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -6.734375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1368.0,
+      "completions/mean_length": 615.451171875,
+      "completions/mean_terminated_length": 592.5556030273438,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.4074322811730468,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 1.0018677373618705,
-      "kl": 0.2841796875,
+      "frac_reward_zero_std": 0.9375,
+      "grad_norm": 2.4369472254890088,
+      "kl": 0.1077880859375,
       "learning_rate": 5.991445609204641e-07,
-      "loss": 0.0028,
-      "num_tokens": 273475268.0,
+      "loss": 0.022,
+      "num_tokens": 271757272.0,
       "reward": 0.09687499701976776,
-      "reward_std": 0.0,
+      "reward_std": 0.0015625000232830644,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
       "rewards/format_reward/mean": 0.96875,
@@ -12756,19 +12756,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1678.0,
-      "completions/max_terminated_length": 1678.0,
-      "completions/mean_length": 574.8203125,
-      "completions/mean_terminated_length": 573.5812377929688,
-      "completions/min_length": 214.0,
-      "completions/min_terminated_length": 214.0,
+      "completions/max_length": 1747.0,
+      "completions/max_terminated_length": 1747.0,
+      "completions/mean_length": 581.82421875,
+      "completions/mean_terminated_length": 579.5440063476562,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.4083277367360645,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.005519132621378801,
-      "kl": 0.05157470703125,
+      "grad_norm": 0.08771054633359765,
+      "kl": 0.0449371337890625,
       "learning_rate": 5.949110205770292e-07,
-      "loss": 0.0005,
-      "num_tokens": 274102872.0,
+      "loss": 0.0004,
+      "num_tokens": 272388462.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12783,20 +12783,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1485.0,
-      "completions/max_terminated_length": 1485.0,
-      "completions/mean_length": 590.720703125,
-      "completions/mean_terminated_length": 588.6372680664062,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1665.0,
+      "completions/max_terminated_length": 1665.0,
+      "completions/mean_length": 602.40625,
+      "completions/mean_terminated_length": 602.40625,
+      "completions/min_length": 188.0,
+      "completions/min_terminated_length": 188.0,
       "epoch": 0.40922319229908216,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.00569307441138671,
-      "kl": 0.05108642578125,
+      "grad_norm": 0.004411391036507665,
+      "kl": 0.00644683837890625,
       "learning_rate": 5.90767903450964e-07,
-      "loss": 0.0005,
-      "num_tokens": 274712729.0,
+      "loss": 0.0001,
+      "num_tokens": 273004302.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -12811,26 +12811,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1336.0,
-      "completions/max_terminated_length": 1336.0,
-      "completions/mean_length": 597.15234375,
-      "completions/mean_terminated_length": 593.3929443359375,
-      "completions/min_length": 205.0,
-      "completions/min_terminated_length": 205.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1573.0,
+      "completions/mean_length": 608.103515625,
+      "completions/mean_terminated_length": 605.2857055664062,
+      "completions/min_length": 234.0,
+      "completions/min_terminated_length": 234.0,
       "epoch": 0.41011864786209984,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.6059039024411552,
-      "kl": 0.0643310546875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0017929355017531458,
+      "kl": 0.00606536865234375,
       "learning_rate": 5.867153833791652e-07,
-      "loss": 0.0072,
-      "num_tokens": 275301255.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 273598435.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 458
     },
     {
@@ -12839,26 +12839,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1331.0,
-      "completions/max_terminated_length": 1331.0,
-      "completions/mean_length": 591.708984375,
-      "completions/mean_terminated_length": 591.708984375,
-      "completions/min_length": 155.0,
-      "completions/min_terminated_length": 155.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1965.0,
+      "completions/mean_length": 606.548828125,
+      "completions/mean_terminated_length": 601.058837890625,
+      "completions/min_length": 135.0,
+      "completions/min_terminated_length": 135.0,
       "epoch": 0.4110141034251175,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003911482325395383,
-      "kl": 0.04339599609375,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 202.70294884152713,
+      "kl": 47.255271911621094,
       "learning_rate": 5.827536303972587e-07,
-      "loss": 0.0004,
-      "num_tokens": 275874290.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.4874,
+      "num_tokens": 274179068.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 459
     },
     {
@@ -12867,26 +12867,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1864.0,
-      "completions/max_terminated_length": 1864.0,
-      "completions/mean_length": 587.716796875,
-      "completions/mean_terminated_length": 586.3072509765625,
-      "completions/min_length": 168.0,
-      "completions/min_terminated_length": 168.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1687.0,
+      "completions/max_terminated_length": 1687.0,
+      "completions/mean_length": 597.40625,
+      "completions/mean_terminated_length": 597.40625,
+      "completions/min_length": 185.0,
+      "completions/min_terminated_length": 185.0,
       "epoch": 0.4119095589881352,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.05776806822294325,
-      "kl": 0.0504150390625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.001968268256640002,
+      "kl": 0.007049560546875,
       "learning_rate": 5.78882810732465e-07,
-      "loss": -0.0014,
-      "num_tokens": 276456769.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 274766508.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 460
     },
     {
@@ -12895,26 +12895,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1766.0,
-      "completions/mean_length": 578.732421875,
-      "completions/mean_terminated_length": 575.305908203125,
-      "completions/min_length": 180.0,
-      "completions/min_terminated_length": 180.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1438.0,
+      "completions/max_terminated_length": 1438.0,
+      "completions/mean_length": 565.31640625,
+      "completions/mean_terminated_length": 565.31640625,
+      "completions/min_length": 183.0,
+      "completions/min_terminated_length": 183.0,
       "epoch": 0.4128050145511529,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07438159135182086,
-      "kl": 0.0528564453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.001874408283947134,
+      "kl": 0.00655364990234375,
       "learning_rate": 5.75103086796625e-07,
-      "loss": 0.0161,
-      "num_tokens": 277074408.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 275377278.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 461
     },
     {
@@ -12923,26 +12923,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1618.0,
-      "completions/max_terminated_length": 1618.0,
-      "completions/mean_length": 642.39453125,
-      "completions/mean_terminated_length": 640.9647827148438,
-      "completions/min_length": 280.0,
-      "completions/min_terminated_length": 280.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1562.0,
+      "completions/max_terminated_length": 1562.0,
+      "completions/mean_length": 656.626953125,
+      "completions/mean_terminated_length": 656.626953125,
+      "completions/min_length": 233.0,
+      "completions/min_terminated_length": 233.0,
       "epoch": 0.4137004701141706,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06372747249738105,
-      "kl": 0.05255126953125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0018452101093293508,
+      "kl": 0.00627899169921875,
       "learning_rate": 5.714146171793846e-07,
-      "loss": 0.0026,
-      "num_tokens": 277747186.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 276057343.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 462
     },
     {
@@ -12952,25 +12952,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1767.0,
-      "completions/max_terminated_length": 1767.0,
-      "completions/mean_length": 682.73828125,
-      "completions/mean_terminated_length": 681.872802734375,
-      "completions/min_length": 210.0,
-      "completions/min_terminated_length": 210.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1889.0,
+      "completions/mean_length": 695.85546875,
+      "completions/mean_terminated_length": 693.2094116210938,
+      "completions/min_length": 231.0,
+      "completions/min_terminated_length": 231.0,
       "epoch": 0.4145959256771883,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004279885734629871,
-      "kl": 0.05035400390625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.0656524479101266,
+      "kl": 0.00646209716796875,
       "learning_rate": 5.678175566415422e-07,
-      "loss": 0.0005,
-      "num_tokens": 278462364.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0074,
+      "num_tokens": 276779237.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 463
     },
     {
@@ -12979,20 +12979,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1377.0,
-      "completions/max_terminated_length": 1377.0,
-      "completions/mean_length": 605.052734375,
-      "completions/mean_terminated_length": 603.4569091796875,
-      "completions/min_length": 189.0,
-      "completions/min_terminated_length": 189.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1756.0,
+      "completions/max_terminated_length": 1756.0,
+      "completions/mean_length": 615.26953125,
+      "completions/mean_terminated_length": 615.26953125,
+      "completions/min_length": 215.0,
+      "completions/min_terminated_length": 215.0,
       "epoch": 0.41549138124020596,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004562879684523193,
-      "kl": 0.0518798828125,
+      "grad_norm": 0.002116149555581525,
+      "kl": 0.00698089599609375,
       "learning_rate": 5.643120561085528e-07,
-      "loss": 0.0005,
-      "num_tokens": 279099863.0,
+      "loss": 0.0001,
+      "num_tokens": 277421967.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13007,26 +13007,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1470.0,
-      "completions/max_terminated_length": 1470.0,
-      "completions/mean_length": 597.873046875,
-      "completions/mean_terminated_length": 595.24169921875,
-      "completions/min_length": 179.0,
-      "completions/min_terminated_length": 179.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1530.0,
+      "completions/max_terminated_length": 1530.0,
+      "completions/mean_length": 616.572265625,
+      "completions/mean_terminated_length": 615.7123413085938,
+      "completions/min_length": 213.0,
+      "completions/min_terminated_length": 213.0,
       "epoch": 0.41638683680322364,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 1.532391883408515,
-      "kl": 0.059814453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 12.454797402118519,
+      "kl": 0.9109954833984375,
       "learning_rate": 5.608982626641991e-07,
       "loss": 0.0091,
-      "num_tokens": 279734294.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "num_tokens": 278065972.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 465
     },
     {
@@ -13035,20 +13035,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1271.0,
-      "completions/max_terminated_length": 1271.0,
-      "completions/mean_length": 556.6953125,
-      "completions/mean_terminated_length": 556.0665283203125,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1607.0,
+      "completions/max_terminated_length": 1607.0,
+      "completions/mean_length": 584.4609375,
+      "completions/mean_terminated_length": 584.4609375,
+      "completions/min_length": 113.0,
+      "completions/min_terminated_length": 113.0,
       "epoch": 0.4172822923662413,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004546148613238247,
-      "kl": 0.04937744140625,
+      "grad_norm": 0.0022458003083646212,
+      "kl": 0.00685882568359375,
       "learning_rate": 5.575763195444166e-07,
-      "loss": 0.0005,
-      "num_tokens": 280266922.0,
+      "loss": 0.0001,
+      "num_tokens": 278612816.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13064,19 +13064,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1401.0,
-      "completions/max_terminated_length": 1401.0,
-      "completions/mean_length": 585.609375,
-      "completions/mean_terminated_length": 584.6829833984375,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1844.0,
+      "completions/mean_length": 608.923828125,
+      "completions/mean_terminated_length": 606.1076049804688,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.418177747929259,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009347005608886688,
-      "kl": 0.05615234375,
+      "grad_norm": 0.006077576169920933,
+      "kl": 0.0076141357421875,
       "learning_rate": 5.543463661312847e-07,
-      "loss": 0.0006,
-      "num_tokens": 280885298.0,
+      "loss": 0.0001,
+      "num_tokens": 279243129.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13091,26 +13091,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1803.0,
-      "completions/mean_length": 650.8828125,
-      "completions/mean_terminated_length": 645.8843383789062,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1712.0,
+      "completions/max_terminated_length": 1712.0,
+      "completions/mean_length": 655.77734375,
+      "completions/mean_terminated_length": 655.77734375,
+      "completions/min_length": 201.0,
+      "completions/min_terminated_length": 201.0,
       "epoch": 0.41907320349227667,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.044976200251507704,
-      "kl": 0.0523681640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.00508351729525104,
+      "kl": 0.00701141357421875,
       "learning_rate": 5.512085379471808e-07,
-      "loss": 0.0119,
-      "num_tokens": 281563062.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 279923399.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 468
     },
     {
@@ -13119,26 +13119,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1638.0,
-      "completions/max_terminated_length": 1638.0,
-      "completions/mean_length": 569.3828125,
-      "completions/mean_terminated_length": 565.9019775390625,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1716.0,
+      "completions/mean_length": 567.76171875,
+      "completions/mean_terminated_length": 564.864990234375,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.4199686590552944,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.060898844246720894,
-      "kl": 0.06231689453125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0019400963630317191,
+      "kl": 0.00716400146484375,
       "learning_rate": 5.481629666490903e-07,
-      "loss": -0.0039,
-      "num_tokens": 282165226.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 280524733.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 469
     },
     {
@@ -13147,26 +13147,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1286.0,
-      "completions/max_terminated_length": 1286.0,
-      "completions/mean_length": 584.701171875,
-      "completions/mean_terminated_length": 584.701171875,
-      "completions/min_length": 204.0,
-      "completions/min_terminated_length": 204.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1541.0,
+      "completions/mean_length": 628.064453125,
+      "completions/mean_terminated_length": 622.49609375,
+      "completions/min_length": 228.0,
+      "completions/min_terminated_length": 228.0,
       "epoch": 0.4208641146183121,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003834240543409795,
-      "kl": 0.04595947265625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07028124365023734,
+      "kl": 0.00714874267578125,
       "learning_rate": 5.452097800230853e-07,
-      "loss": 0.0005,
-      "num_tokens": 282778881.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0077,
+      "num_tokens": 281160590.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 470
     },
     {
@@ -13176,19 +13176,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1541.0,
-      "completions/max_terminated_length": 1541.0,
-      "completions/mean_length": 611.36328125,
-      "completions/mean_terminated_length": 611.36328125,
-      "completions/min_length": 186.0,
-      "completions/min_terminated_length": 186.0,
+      "completions/max_length": 1393.0,
+      "completions/max_terminated_length": 1393.0,
+      "completions/mean_length": 614.638671875,
+      "completions/mean_terminated_length": 614.638671875,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.42175957018132976,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003375383832642503,
-      "kl": 0.043701171875,
+      "grad_norm": 0.002522479085201929,
+      "kl": 0.00661468505859375,
       "learning_rate": 5.423491019789623e-07,
-      "loss": 0.0004,
-      "num_tokens": 283407803.0,
+      "loss": 0.0001,
+      "num_tokens": 281791189.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13203,20 +13203,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1437.0,
-      "completions/max_terminated_length": 1437.0,
-      "completions/mean_length": 588.123046875,
-      "completions/mean_terminated_length": 588.123046875,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1525.0,
+      "completions/mean_length": 595.560546875,
+      "completions/mean_terminated_length": 592.7182006835938,
+      "completions/min_length": 196.0,
+      "completions/min_terminated_length": 196.0,
       "epoch": 0.42265502574434743,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007763949257554806,
-      "kl": 0.04583740234375,
+      "grad_norm": 0.002388905997514264,
+      "kl": 0.00653076171875,
       "learning_rate": 5.395810525450425e-07,
-      "loss": 0.0005,
-      "num_tokens": 284015322.0,
+      "loss": 0.0001,
+      "num_tokens": 282402516.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13231,26 +13231,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1981.0,
-      "completions/max_terminated_length": 1981.0,
-      "completions/mean_length": 589.7265625,
-      "completions/mean_terminated_length": 588.6594848632812,
-      "completions/min_length": 135.0,
-      "completions/min_terminated_length": 135.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1697.0,
+      "completions/max_terminated_length": 1697.0,
+      "completions/mean_length": 606.19921875,
+      "completions/mean_terminated_length": 606.19921875,
+      "completions/min_length": 245.0,
+      "completions/min_terminated_length": 245.0,
       "epoch": 0.4235504813073651,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06245932963948891,
-      "kl": 0.0523681640625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002615205230137125,
+      "kl": 0.00702667236328125,
       "learning_rate": 5.369057478631359e-07,
-      "loss": 0.003,
-      "num_tokens": 284655150.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 283050778.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 473
     },
     {
@@ -13259,26 +13259,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.90625,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1275.0,
-      "completions/mean_length": 588.333984375,
-      "completions/mean_terminated_length": 573.5711669921875,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1604.0,
+      "completions/max_terminated_length": 1604.0,
+      "completions/mean_length": 594.029296875,
+      "completions/mean_terminated_length": 594.029296875,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.4244459368703828,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 1.393684754940159,
-      "kl": 0.079345703125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002635326102091178,
+      "kl": 0.00766754150390625,
       "learning_rate": 5.343233001836694e-07,
-      "loss": 0.0239,
-      "num_tokens": 285277209.0,
-      "reward": 0.09902343899011612,
-      "reward_std": 0.0021787926089018583,
+      "loss": 0.0001,
+      "num_tokens": 283675753.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.990234375,
-      "rewards/format_reward/std": 0.09843364357948303,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 474
     },
     {
@@ -13287,26 +13287,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1996.0,
-      "completions/max_terminated_length": 1996.0,
-      "completions/mean_length": 620.279296875,
-      "completions/mean_terminated_length": 616.6882934570312,
-      "completions/min_length": 178.0,
-      "completions/min_terminated_length": 178.0,
+      "completions/clipped_ratio": -6.921875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1640.0,
+      "completions/mean_length": 635.048828125,
+      "completions/mean_terminated_length": 625.1538696289062,
+      "completions/min_length": 195.0,
+      "completions/min_terminated_length": 195.0,
       "epoch": 0.42534139243340047,
       "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.08588302090778102,
-      "kl": 0.04718017578125,
+      "grad_norm": 0.07288263680226181,
+      "kl": 0.00714874267578125,
       "learning_rate": 5.318338178609754e-07,
-      "loss": 0.014,
-      "num_tokens": 285910792.0,
-      "reward": 0.09941406548023224,
-      "reward_std": 0.0018486406188458204,
+      "loss": 0.0314,
+      "num_tokens": 284316898.0,
+      "reward": 0.09902343899011612,
+      "reward_std": 0.0021787926089018583,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.994140625,
-      "rewards/format_reward/std": 0.07639661431312561,
+      "rewards/format_reward/mean": 0.990234375,
+      "rewards/format_reward/std": 0.09843364357948303,
       "step": 475
     },
     {
@@ -13315,20 +13315,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1860.0,
-      "completions/max_terminated_length": 1860.0,
-      "completions/mean_length": 572.791015625,
-      "completions/mean_terminated_length": 571.6007690429688,
-      "completions/min_length": 228.0,
-      "completions/min_terminated_length": 228.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1498.0,
+      "completions/max_terminated_length": 1498.0,
+      "completions/mean_length": 568.4375,
+      "completions/mean_terminated_length": 568.4375,
+      "completions/min_length": 179.0,
+      "completions/min_terminated_length": 179.0,
       "epoch": 0.4262368479964182,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02767124078956891,
-      "kl": 0.06396484375,
+      "grad_norm": 0.0024556427027511017,
+      "kl": 0.00702667236328125,
       "learning_rate": 5.294374053487459e-07,
-      "loss": 0.0006,
-      "num_tokens": 286487181.0,
+      "loss": 0.0001,
+      "num_tokens": 284891058.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13343,20 +13343,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1404.0,
-      "completions/max_terminated_length": 1404.0,
-      "completions/mean_length": 537.857421875,
-      "completions/mean_terminated_length": 536.6569213867188,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1848.0,
+      "completions/max_terminated_length": 1848.0,
+      "completions/mean_length": 553.7578125,
+      "completions/mean_terminated_length": 553.7578125,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.4271323035594359,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01657319794506783,
-      "kl": 0.0655517578125,
+      "grad_norm": 0.002743355850282284,
+      "kl": 0.00701141357421875,
       "learning_rate": 5.271341631956511e-07,
-      "loss": 0.0007,
-      "num_tokens": 287043348.0,
+      "loss": 0.0001,
+      "num_tokens": 285455366.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13372,19 +13372,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1387.0,
-      "completions/max_terminated_length": 1387.0,
-      "completions/mean_length": 567.2109375,
-      "completions/mean_terminated_length": 567.2109375,
-      "completions/min_length": 230.0,
-      "completions/min_terminated_length": 230.0,
+      "completions/max_length": 1466.0,
+      "completions/max_terminated_length": 1466.0,
+      "completions/mean_length": 570.58984375,
+      "completions/mean_terminated_length": 570.58984375,
+      "completions/min_length": 167.0,
+      "completions/min_terminated_length": 167.0,
       "epoch": 0.42802775912245355,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004291858410065178,
-      "kl": 0.04486083984375,
+      "grad_norm": 0.006622609032114254,
+      "kl": 0.00746917724609375,
       "learning_rate": 5.249241880411181e-07,
-      "loss": 0.0004,
-      "num_tokens": 287651904.0,
+      "loss": 0.0001,
+      "num_tokens": 286065652.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13399,26 +13399,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.9375,
-      "completions/max_length": 1304.0,
-      "completions/max_terminated_length": 1304.0,
-      "completions/mean_length": 596.017578125,
-      "completions/mean_terminated_length": 592.281494140625,
-      "completions/min_length": 144.0,
-      "completions/min_terminated_length": 144.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1574.0,
+      "completions/mean_length": 627.298828125,
+      "completions/mean_terminated_length": 624.5186157226562,
+      "completions/min_length": 193.0,
+      "completions/min_terminated_length": 193.0,
       "epoch": 0.42892321468547123,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.007262425387433753,
-      "kl": 0.0606689453125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07069570079703043,
+      "kl": 0.006683349609375,
       "learning_rate": 5.228075726112785e-07,
-      "loss": 0.0006,
-      "num_tokens": 288291881.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0087,
+      "num_tokens": 286721645.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 479
     },
     {
@@ -13427,20 +13427,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1754.0,
-      "completions/mean_length": 625.6953125,
-      "completions/mean_terminated_length": 620.923583984375,
-      "completions/min_length": 181.0,
-      "completions/min_terminated_length": 181.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1667.0,
+      "completions/max_terminated_length": 1667.0,
+      "completions/mean_length": 615.96484375,
+      "completions/mean_terminated_length": 614.74951171875,
+      "completions/min_length": 169.0,
+      "completions/min_terminated_length": 169.0,
       "epoch": 0.4298186702484889,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.07604608796303157,
-      "kl": 0.0860595703125,
+      "grad_norm": 0.5664508010867297,
+      "kl": 0.00927734375,
       "learning_rate": 5.207844057150768e-07,
-      "loss": 0.0169,
-      "num_tokens": 288940557.0,
+      "loss": 0.0052,
+      "num_tokens": 287365339.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -13456,19 +13456,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1301.0,
-      "completions/max_terminated_length": 1301.0,
-      "completions/mean_length": 566.443359375,
-      "completions/mean_terminated_length": 566.443359375,
-      "completions/min_length": 132.0,
-      "completions/min_terminated_length": 132.0,
+      "completions/max_length": 1478.0,
+      "completions/max_terminated_length": 1478.0,
+      "completions/mean_length": 594.375,
+      "completions/mean_terminated_length": 594.375,
+      "completions/min_length": 121.0,
+      "completions/min_terminated_length": 121.0,
       "epoch": 0.4307141258115066,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0056663762582022026,
-      "kl": 0.04766845703125,
+      "grad_norm": 0.0022399177386503632,
+      "kl": 0.00719451904296875,
       "learning_rate": 5.188547722405437e-07,
-      "loss": 0.0005,
-      "num_tokens": 289558448.0,
+      "loss": 0.0001,
+      "num_tokens": 287997531.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13484,19 +13484,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1752.0,
-      "completions/max_terminated_length": 1752.0,
-      "completions/mean_length": 601.892578125,
-      "completions/mean_terminated_length": 601.892578125,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_length": 1655.0,
+      "completions/max_terminated_length": 1655.0,
+      "completions/mean_length": 620.845703125,
+      "completions/mean_terminated_length": 620.845703125,
+      "completions/min_length": 153.0,
+      "completions/min_terminated_length": 153.0,
       "epoch": 0.43160958137452426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0035106948516688734,
-      "kl": 0.04449462890625,
+      "grad_norm": 0.0017809752727835867,
+      "kl": 0.00690460205078125,
       "learning_rate": 5.170187531512351e-07,
-      "loss": 0.0004,
-      "num_tokens": 290198697.0,
+      "loss": 0.0001,
+      "num_tokens": 288647484.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13511,26 +13511,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1490.0,
-      "completions/max_terminated_length": 1490.0,
-      "completions/mean_length": 573.759765625,
-      "completions/mean_terminated_length": 572.25048828125,
-      "completions/min_length": 156.0,
-      "completions/min_terminated_length": 156.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1235.0,
+      "completions/max_terminated_length": 1235.0,
+      "completions/mean_length": 587.107421875,
+      "completions/mean_terminated_length": 587.107421875,
+      "completions/min_length": 203.0,
+      "completions/min_terminated_length": 203.0,
       "epoch": 0.432505036937542,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.03392016125236342,
-      "kl": 0.0640869140625,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07380820125009971,
+      "kl": 0.006591796875,
       "learning_rate": 5.152764254828348e-07,
-      "loss": 0.0006,
-      "num_tokens": 290813902.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0026,
+      "num_tokens": 289269523.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 483
     },
     {
@@ -13539,20 +13539,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1524.0,
-      "completions/max_terminated_length": 1524.0,
-      "completions/mean_length": 576.0390625,
-      "completions/mean_terminated_length": 576.0390625,
-      "completions/min_length": 169.0,
-      "completions/min_terminated_length": 169.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1385.0,
+      "completions/max_terminated_length": 1385.0,
+      "completions/mean_length": 592.1484375,
+      "completions/mean_terminated_length": 591.4285888671875,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
       "epoch": 0.4334004925005597,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.04602480512840631,
-      "kl": 0.0469970703125,
+      "grad_norm": 1.367785098713213,
+      "kl": 0.0121002197265625,
       "learning_rate": 5.136278623399225e-07,
-      "loss": -0.0029,
-      "num_tokens": 291436194.0,
+      "loss": 0.0037,
+      "num_tokens": 289900063.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -13567,26 +13567,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
-      "completions/max_length": 1310.0,
-      "completions/max_terminated_length": 1310.0,
-      "completions/mean_length": 566.806640625,
-      "completions/mean_terminated_length": 563.4970703125,
-      "completions/min_length": 196.0,
-      "completions/min_terminated_length": 196.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1317.0,
+      "completions/max_terminated_length": 1317.0,
+      "completions/mean_length": 572.5546875,
+      "completions/mean_terminated_length": 572.5546875,
+      "completions/min_length": 172.0,
+      "completions/min_terminated_length": 172.0,
       "epoch": 0.43429594806357735,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 1.4297514894292327,
-      "kl": 0.0731201171875,
-      "learning_rate": 5.120731328929058e-07,
-      "loss": 0.0004,
-      "num_tokens": 292035263.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0019026093751945668,
+      "kl": 0.00656890869140625,
+      "learning_rate": 5.120731328929058e-07,
+      "loss": 0.0001,
+      "num_tokens": 290502075.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 485
     },
     {
@@ -13595,20 +13595,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1402.0,
-      "completions/mean_length": 593.1875,
-      "completions/mean_terminated_length": 587.4823608398438,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 1751.0,
+      "completions/max_terminated_length": 1751.0,
+      "completions/mean_length": 610.455078125,
+      "completions/mean_terminated_length": 609.8023681640625,
+      "completions/min_length": 211.0,
+      "completions/min_terminated_length": 211.0,
       "epoch": 0.43519140362659503,
       "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.2048485997684941,
-      "kl": 0.08758544921875,
+      "grad_norm": 1.277845158123168,
+      "kl": 0.01782989501953125,
       "learning_rate": 5.106123023751187e-07,
-      "loss": 0.0123,
-      "num_tokens": 292637039.0,
+      "loss": 0.0043,
+      "num_tokens": 291112692.0,
       "reward": 0.09980468451976776,
       "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
@@ -13623,20 +13623,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1409.0,
-      "completions/mean_length": 620.24609375,
-      "completions/mean_terminated_length": 617.4520263671875,
-      "completions/min_length": 238.0,
-      "completions/min_terminated_length": 238.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1923.0,
+      "completions/max_terminated_length": 1923.0,
+      "completions/mean_length": 634.310546875,
+      "completions/mean_terminated_length": 634.310546875,
+      "completions/min_length": 212.0,
+      "completions/min_terminated_length": 212.0,
       "epoch": 0.4360868591896127,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004771668900955531,
-      "kl": 0.046630859375,
+      "grad_norm": 0.0029675438707230098,
+      "kl": 0.00637054443359375,
       "learning_rate": 5.092454320800833e-07,
-      "loss": 0.0005,
-      "num_tokens": 293296445.0,
+      "loss": 0.0001,
+      "num_tokens": 291779299.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13651,20 +13651,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1690.0,
-      "completions/max_terminated_length": 1690.0,
-      "completions/mean_length": 562.4921875,
-      "completions/mean_terminated_length": 562.4921875,
-      "completions/min_length": 165.0,
-      "completions/min_terminated_length": 165.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1793.0,
+      "completions/mean_length": 582.9453125,
+      "completions/mean_terminated_length": 580.0782470703125,
+      "completions/min_length": 133.0,
+      "completions/min_terminated_length": 133.0,
       "epoch": 0.4369823147526304,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004316334400510252,
-      "kl": 0.04510498046875,
+      "grad_norm": 0.002218209993171591,
+      "kl": 0.0069427490234375,
       "learning_rate": 5.079725793589405e-07,
-      "loss": 0.0005,
-      "num_tokens": 293894857.0,
+      "loss": 0.0001,
+      "num_tokens": 292388183.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13680,19 +13680,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1488.0,
-      "completions/max_terminated_length": 1488.0,
-      "completions/mean_length": 586.365234375,
-      "completions/mean_terminated_length": 586.365234375,
-      "completions/min_length": 201.0,
-      "completions/min_terminated_length": 201.0,
+      "completions/max_length": 1947.0,
+      "completions/max_terminated_length": 1947.0,
+      "completions/mean_length": 619.53125,
+      "completions/mean_terminated_length": 619.53125,
+      "completions/min_length": 247.0,
+      "completions/min_terminated_length": 247.0,
       "epoch": 0.43787777031564806,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0038624710298466006,
-      "kl": 0.0440673828125,
+      "grad_norm": 0.002973729317383643,
+      "kl": 0.00652313232421875,
       "learning_rate": 5.067937976180407e-07,
-      "loss": 0.0004,
-      "num_tokens": 294499636.0,
+      "loss": 0.0001,
+      "num_tokens": 293009943.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13707,20 +13707,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2036.0,
-      "completions/max_terminated_length": 2036.0,
-      "completions/mean_length": 591.830078125,
-      "completions/mean_terminated_length": 589.8392333984375,
-      "completions/min_length": 166.0,
-      "completions/min_terminated_length": 166.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1907.0,
+      "completions/max_terminated_length": 1907.0,
+      "completions/mean_length": 606.29296875,
+      "completions/mean_terminated_length": 606.29296875,
+      "completions/min_length": 173.0,
+      "completions/min_terminated_length": 173.0,
       "epoch": 0.4387732258786658,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.08591994390512282,
-      "kl": 0.093994140625,
+      "grad_norm": 0.002162476695238198,
+      "kl": 0.0070037841796875,
       "learning_rate": 5.057091363167046e-07,
-      "loss": 0.0009,
-      "num_tokens": 295119581.0,
+      "loss": 0.0001,
+      "num_tokens": 293637293.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13735,20 +13735,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1211.0,
-      "completions/max_terminated_length": 1211.0,
-      "completions/mean_length": 550.6796875,
-      "completions/mean_terminated_length": 548.62158203125,
-      "completions/min_length": 220.0,
-      "completions/min_terminated_length": 220.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1456.0,
+      "completions/mean_length": 564.47265625,
+      "completions/mean_terminated_length": 561.5694580078125,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.4396686814416835,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.05506883519186736,
-      "kl": 0.0794677734375,
+      "grad_norm": 0.0024995506608288215,
+      "kl": 0.00746917724609375,
       "learning_rate": 5.047186409651489e-07,
-      "loss": 0.0008,
-      "num_tokens": 295698137.0,
+      "loss": 0.0001,
+      "num_tokens": 294222911.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13763,20 +13763,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1543.0,
-      "completions/max_terminated_length": 1543.0,
-      "completions/mean_length": 608.87109375,
-      "completions/mean_terminated_length": 608.0391235351562,
-      "completions/min_length": 153.0,
-      "completions/min_terminated_length": 153.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1721.0,
+      "completions/max_terminated_length": 1721.0,
+      "completions/mean_length": 635.138671875,
+      "completions/mean_terminated_length": 635.138671875,
+      "completions/min_length": 165.0,
+      "completions/min_terminated_length": 165.0,
       "epoch": 0.44056413700470115,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0038651946743403155,
-      "kl": 0.04791259765625,
+      "grad_norm": 0.003279801989016285,
+      "kl": 0.0062103271484375,
       "learning_rate": 5.038223531225742e-07,
-      "loss": 0.0005,
-      "num_tokens": 296304295.0,
+      "loss": 0.0001,
+      "num_tokens": 294842518.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13792,25 +13792,25 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1739.0,
-      "completions/max_terminated_length": 1739.0,
-      "completions/mean_length": 569.330078125,
-      "completions/mean_terminated_length": 569.330078125,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/max_length": 1358.0,
+      "completions/max_terminated_length": 1358.0,
+      "completions/mean_length": 588.244140625,
+      "completions/mean_terminated_length": 588.244140625,
+      "completions/min_length": 197.0,
+      "completions/min_terminated_length": 197.0,
       "epoch": 0.44145959256771883,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.003558131893094743,
-      "kl": 0.04473876953125,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.07219605536662821,
+      "kl": 0.00728607177734375,
       "learning_rate": 5.030203103954232e-07,
-      "loss": 0.0004,
-      "num_tokens": 296895328.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0006,
+      "num_tokens": 295443235.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 493
     },
     {
@@ -13820,19 +13820,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1505.0,
-      "completions/max_terminated_length": 1505.0,
-      "completions/mean_length": 579.6875,
-      "completions/mean_terminated_length": 579.6875,
-      "completions/min_length": 192.0,
-      "completions/min_terminated_length": 192.0,
+      "completions/max_length": 1510.0,
+      "completions/max_terminated_length": 1510.0,
+      "completions/mean_length": 594.796875,
+      "completions/mean_terminated_length": 594.796875,
+      "completions/min_length": 206.0,
+      "completions/min_terminated_length": 206.0,
       "epoch": 0.4423550481307365,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.01139014607119812,
-      "kl": 0.04290771484375,
+      "grad_norm": 0.003095639432087293,
+      "kl": 0.0069122314453125,
       "learning_rate": 5.023125464358026e-07,
-      "loss": 0.0004,
-      "num_tokens": 297475296.0,
+      "loss": 0.0001,
+      "num_tokens": 296030939.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13847,26 +13847,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1456.0,
-      "completions/max_terminated_length": 1456.0,
-      "completions/mean_length": 560.001953125,
-      "completions/mean_terminated_length": 559.4677124023438,
-      "completions/min_length": 227.0,
-      "completions/min_terminated_length": 227.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1350.0,
+      "completions/max_terminated_length": 1350.0,
+      "completions/mean_length": 586.44921875,
+      "completions/mean_terminated_length": 586.44921875,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.4432505036937542,
-      "frac_reward_zero_std": 0.9375,
-      "grad_norm": 0.09503865862602394,
-      "kl": 0.06842041015625,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0032109704817087368,
+      "kl": 0.006134033203125,
       "learning_rate": 5.016990909400709e-07,
-      "loss": 0.003,
-      "num_tokens": 298037489.0,
-      "reward": 0.099609375,
-      "reward_std": 0.0015625000232830644,
+      "loss": 0.0001,
+      "num_tokens": 296606673.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.99609375,
-      "rewards/format_reward/std": 0.06243881583213806,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 495
     },
     {
@@ -13876,19 +13876,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1940.0,
-      "completions/max_terminated_length": 1940.0,
-      "completions/mean_length": 616.166015625,
-      "completions/mean_terminated_length": 616.166015625,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/max_length": 1649.0,
+      "completions/max_terminated_length": 1649.0,
+      "completions/mean_length": 647.8203125,
+      "completions/mean_terminated_length": 647.8203125,
+      "completions/min_length": 271.0,
+      "completions/min_terminated_length": 271.0,
       "epoch": 0.44414595925677186,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0038706228148763185,
-      "kl": 0.04656982421875,
+      "grad_norm": 0.0030476575163748028,
+      "kl": 0.0064697265625,
       "learning_rate": 5.011799696475915e-07,
-      "loss": 0.0005,
-      "num_tokens": 298647558.0,
+      "loss": 0.0001,
+      "num_tokens": 297232949.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13903,20 +13903,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1132.0,
-      "completions/max_terminated_length": 1132.0,
-      "completions/mean_length": 553.35546875,
-      "completions/mean_terminated_length": 553.35546875,
-      "completions/min_length": 176.0,
-      "completions/min_terminated_length": 176.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1572.0,
+      "completions/mean_length": 567.07421875,
+      "completions/mean_terminated_length": 564.1761474609375,
+      "completions/min_length": 161.0,
+      "completions/min_terminated_length": 161.0,
       "epoch": 0.4450414148197896,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004194278776945329,
-      "kl": 0.0443115234375,
+      "grad_norm": 0.00517923432566091,
+      "kl": 0.00667572021484375,
       "learning_rate": 5.007552043396547e-07,
-      "loss": 0.0004,
-      "num_tokens": 299232572.0,
+      "loss": 0.0001,
+      "num_tokens": 297824987.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13931,20 +13931,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1570.0,
-      "completions/max_terminated_length": 1570.0,
-      "completions/mean_length": 569.85546875,
-      "completions/mean_terminated_length": 568.876708984375,
-      "completions/min_length": 177.0,
-      "completions/min_terminated_length": 177.0,
+      "completions/clipped_ratio": -6.96875,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1440.0,
+      "completions/mean_length": 593.87109375,
+      "completions/mean_terminated_length": 588.1686401367188,
+      "completions/min_length": 210.0,
+      "completions/min_terminated_length": 210.0,
       "epoch": 0.44593687038280727,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004197022412970387,
-      "kl": 0.0450439453125,
+      "grad_norm": 0.002854169465541202,
+      "kl": 0.00717926025390625,
       "learning_rate": 5.004248128385618e-07,
-      "loss": 0.0005,
-      "num_tokens": 299849282.0,
+      "loss": 0.0001,
+      "num_tokens": 298453993.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13959,20 +13959,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1234.0,
-      "completions/max_terminated_length": 1234.0,
-      "completions/mean_length": 557.01953125,
-      "completions/mean_terminated_length": 555.427490234375,
-      "completions/min_length": 187.0,
-      "completions/min_terminated_length": 187.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1089.0,
+      "completions/max_terminated_length": 1089.0,
+      "completions/mean_length": 588.5625,
+      "completions/mean_terminated_length": 588.5625,
+      "completions/min_length": 190.0,
+      "completions/min_terminated_length": 190.0,
       "epoch": 0.44683232594582495,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.06481738260450282,
-      "kl": 0.08477783203125,
+      "grad_norm": 0.002690500225541875,
+      "kl": 0.006744384765625,
       "learning_rate": 5.001888090068784e-07,
-      "loss": 0.0008,
-      "num_tokens": 300418732.0,
+      "loss": 0.0001,
+      "num_tokens": 299039593.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -13988,19 +13988,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1447.0,
-      "completions/max_terminated_length": 1447.0,
-      "completions/mean_length": 601.1328125,
-      "completions/mean_terminated_length": 601.1328125,
-      "completions/min_length": 208.0,
-      "completions/min_terminated_length": 208.0,
+      "completions/max_length": 1525.0,
+      "completions/max_terminated_length": 1525.0,
+      "completions/mean_length": 610.150390625,
+      "completions/mean_terminated_length": 610.150390625,
+      "completions/min_length": 238.0,
+      "completions/min_terminated_length": 238.0,
       "epoch": 0.4477277815088426,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.009343548268136184,
-      "kl": 0.046630859375,
+      "grad_norm": 0.0029109843458220755,
+      "kl": 0.006561279296875,
       "learning_rate": 5.000472027468528e-07,
-      "loss": 0.0005,
-      "num_tokens": 301071744.0,
+      "loss": 0.0001,
+      "num_tokens": 299697222.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14015,20 +14015,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1460.0,
-      "completions/max_terminated_length": 1460.0,
-      "completions/mean_length": 603.708984375,
-      "completions/mean_terminated_length": 602.6790771484375,
-      "completions/min_length": 212.0,
-      "completions/min_terminated_length": 212.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1554.0,
+      "completions/max_terminated_length": 1554.0,
+      "completions/mean_length": 629.177734375,
+      "completions/mean_terminated_length": 629.177734375,
+      "completions/min_length": 184.0,
+      "completions/min_terminated_length": 184.0,
       "epoch": 0.4486232370718603,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.034444504126124674,
-      "kl": 0.065673828125,
+      "grad_norm": 0.003181584471698959,
+      "kl": 0.006500244140625,
       "learning_rate": 5.000000000000001e-07,
-      "loss": 0.0007,
-      "num_tokens": 301684331.0,
+      "loss": 0.0001,
+      "num_tokens": 300322849.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14043,26 +14043,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -7.0,
-      "completions/max_length": 1435.0,
-      "completions/max_terminated_length": 1435.0,
-      "completions/mean_length": 607.1640625,
-      "completions/mean_terminated_length": 607.1640625,
-      "completions/min_length": 198.0,
-      "completions/min_terminated_length": 198.0,
+      "completions/clipped_ratio": -6.984375,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1848.0,
+      "completions/mean_length": 628.576171875,
+      "completions/mean_terminated_length": 625.7984008789062,
+      "completions/min_length": 194.0,
+      "completions/min_terminated_length": 194.0,
       "epoch": 0.449518692634878,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004137901914690784,
-      "kl": 0.044921875,
+      "frac_reward_zero_std": 0.96875,
+      "grad_norm": 0.03514131702249418,
+      "kl": 0.00738525390625,
       "learning_rate": 5.000472027468528e-07,
-      "loss": 0.0004,
-      "num_tokens": 302343407.0,
-      "reward": 0.10000000149011612,
-      "reward_std": 0.0,
+      "loss": 0.0154,
+      "num_tokens": 300992888.0,
+      "reward": 0.09980468451976776,
+      "reward_std": 0.0007812500116415322,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 1.0,
-      "rewards/format_reward/std": 0.0,
+      "rewards/format_reward/mean": 0.998046875,
+      "rewards/format_reward/std": 0.04419417306780815,
       "step": 502
     },
     {
@@ -14071,26 +14071,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1381.0,
-      "completions/mean_length": 597.3046875,
-      "completions/mean_terminated_length": 591.61572265625,
-      "completions/min_length": 190.0,
-      "completions/min_terminated_length": 190.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1356.0,
+      "completions/max_terminated_length": 1356.0,
+      "completions/mean_length": 598.283203125,
+      "completions/mean_terminated_length": 598.283203125,
+      "completions/min_length": 208.0,
+      "completions/min_terminated_length": 208.0,
       "epoch": 0.45041414819789566,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 2.232073607669622,
-      "kl": 0.076171875,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0028078886176980417,
+      "kl": 0.00676727294921875,
       "learning_rate": 5.001888090068784e-07,
-      "loss": 0.0171,
-      "num_tokens": 302935259.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 301585241.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 503
     },
     {
@@ -14099,20 +14099,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1283.0,
-      "completions/max_terminated_length": 1283.0,
-      "completions/mean_length": 577.80078125,
-      "completions/mean_terminated_length": 577.1917724609375,
-      "completions/min_length": 264.0,
-      "completions/min_terminated_length": 264.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1856.0,
+      "completions/max_terminated_length": 1856.0,
+      "completions/mean_length": 600.822265625,
+      "completions/mean_terminated_length": 600.822265625,
+      "completions/min_length": 253.0,
+      "completions/min_terminated_length": 253.0,
       "epoch": 0.4513096037609134,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.004036384939905687,
-      "kl": 0.04498291015625,
+      "grad_norm": 0.002869992594641441,
+      "kl": 0.0061187744140625,
       "learning_rate": 5.004248128385618e-07,
-      "loss": 0.0004,
-      "num_tokens": 303528165.0,
+      "loss": 0.0001,
+      "num_tokens": 302189934.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14127,26 +14127,26 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.953125,
+      "completions/clipped_ratio": -6.984375,
       "completions/max_length": 2048.0,
-      "completions/max_terminated_length": 1884.0,
-      "completions/mean_length": 626.9296875,
-      "completions/mean_terminated_length": 622.58154296875,
-      "completions/min_length": 205.0,
-      "completions/min_terminated_length": 205.0,
+      "completions/max_terminated_length": 1485.0,
+      "completions/mean_length": 642.93359375,
+      "completions/mean_terminated_length": 640.1839599609375,
+      "completions/min_length": 253.0,
+      "completions/min_terminated_length": 253.0,
       "epoch": 0.45220505932393107,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.056029796429855026,
-      "kl": 0.04736328125,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.0033069623666026116,
+      "kl": 0.00682830810546875,
       "learning_rate": 5.007552043396547e-07,
-      "loss": 0.0122,
-      "num_tokens": 304199121.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 302869084.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 505
     },
     {
@@ -14155,20 +14155,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.96875,
-      "completions/max_length": 1688.0,
-      "completions/max_terminated_length": 1688.0,
-      "completions/mean_length": 585.638671875,
-      "completions/mean_terminated_length": 583.049072265625,
-      "completions/min_length": 167.0,
-      "completions/min_terminated_length": 167.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1852.0,
+      "completions/max_terminated_length": 1852.0,
+      "completions/mean_length": 608.78515625,
+      "completions/mean_terminated_length": 608.78515625,
+      "completions/min_length": 175.0,
+      "completions/min_terminated_length": 175.0,
       "epoch": 0.45310051488694875,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.16384123749604673,
-      "kl": 0.14178466796875,
+      "grad_norm": 0.019256567921365382,
+      "kl": 0.0113372802734375,
       "learning_rate": 5.011799696475915e-07,
-      "loss": 0.0014,
-      "num_tokens": 304819768.0,
+      "loss": 0.0001,
+      "num_tokens": 303501582.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14183,20 +14183,20 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1327.0,
-      "completions/max_terminated_length": 1327.0,
-      "completions/mean_length": 600.677734375,
-      "completions/mean_terminated_length": 599.6751708984375,
-      "completions/min_length": 164.0,
-      "completions/min_terminated_length": 164.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1764.0,
+      "completions/max_terminated_length": 1764.0,
+      "completions/mean_length": 621.85546875,
+      "completions/mean_terminated_length": 621.85546875,
+      "completions/min_length": 257.0,
+      "completions/min_terminated_length": 257.0,
       "epoch": 0.4539959704499664,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.02131276823537405,
-      "kl": 0.06005859375,
+      "grad_norm": 0.004709655692336034,
+      "kl": 0.006866455078125,
       "learning_rate": 5.016990909400706e-07,
-      "loss": 0.0006,
-      "num_tokens": 305445491.0,
+      "loss": 0.0001,
+      "num_tokens": 304138148.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14212,19 +14212,19 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1673.0,
-      "completions/max_terminated_length": 1673.0,
-      "completions/mean_length": 571.474609375,
-      "completions/mean_terminated_length": 570.0215454101562,
-      "completions/min_length": 213.0,
-      "completions/min_terminated_length": 213.0,
+      "completions/max_length": 2048.0,
+      "completions/max_terminated_length": 1990.0,
+      "completions/mean_length": 596.705078125,
+      "completions/mean_terminated_length": 593.864990234375,
+      "completions/min_length": 237.0,
+      "completions/min_terminated_length": 237.0,
       "epoch": 0.4548914260129841,
       "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.8224324394461862,
-      "kl": 0.51495361328125,
+      "grad_norm": 0.0027508053263992434,
+      "kl": 0.00679779052734375,
       "learning_rate": 5.023125464358026e-07,
-      "loss": 0.0051,
-      "num_tokens": 306010102.0,
+      "loss": 0.0001,
+      "num_tokens": 304715677.0,
       "reward": 0.10000000149011612,
       "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
@@ -14239,41 +14239,41 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": -6.984375,
-      "completions/max_length": 1119.0,
-      "completions/max_terminated_length": 1119.0,
-      "completions/mean_length": 553.423828125,
-      "completions/mean_terminated_length": 552.639892578125,
-      "completions/min_length": 211.0,
-      "completions/min_terminated_length": 211.0,
+      "completions/clipped_ratio": -7.0,
+      "completions/max_length": 1464.0,
+      "completions/max_terminated_length": 1464.0,
+      "completions/mean_length": 579.806640625,
+      "completions/mean_terminated_length": 579.806640625,
+      "completions/min_length": 233.0,
+      "completions/min_terminated_length": 233.0,
       "epoch": 0.4557868815760018,
-      "frac_reward_zero_std": 0.96875,
-      "grad_norm": 0.06463336869087515,
-      "kl": 0.046630859375,
+      "frac_reward_zero_std": 1.0,
+      "grad_norm": 0.002570745420249915,
+      "kl": 0.00681304931640625,
       "learning_rate": 5.03020310395423e-07,
-      "loss": 0.0005,
-      "num_tokens": 306603551.0,
-      "reward": 0.09980468451976776,
-      "reward_std": 0.0007812500116415322,
+      "loss": 0.0001,
+      "num_tokens": 305322634.0,
+      "reward": 0.10000000149011612,
+      "reward_std": 0.0,
       "rewards/code_reward/mean": 0.0,
       "rewards/code_reward/std": 0.0,
-      "rewards/format_reward/mean": 0.998046875,
-      "rewards/format_reward/std": 0.04419417306780815,
+      "rewards/format_reward/mean": 1.0,
+      "rewards/format_reward/std": 0.0,
       "step": 509
     },
     {
       "epoch": 0.4557868815760018,
       "step": 509,
       "total_flos": 0.0,
-      "train_loss": 1.0038919752364893e-06,
-      "train_runtime": 66.6209,
-      "train_samples_per_second": 3842.636,
-      "train_steps_per_second": 7.505
+      "train_loss": 1.3396084333325358e-07,
+      "train_runtime": 69.6904,
+      "train_samples_per_second": 3673.392,
+      "train_steps_per_second": 7.175
     }
   ],
   "logging_steps": 1,
   "max_steps": 500,
-  "num_input_tokens_seen": 306603551,
+  "num_input_tokens_seen": 305322634,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {