diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,9833 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.294117647058824,
+  "eval_steps": 500,
+  "global_step": 700,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.74038696289062,
+      "epoch": 0.014705882352941176,
+      "grad_norm": 4.535776925340435,
+      "kl": 0.0,
+      "learning_rate": 9.992647058823528e-07,
+      "loss": -0.0,
+      "reward": 1.7768200635910034,
+      "reward_std": 0.06235586851835251,
+      "rewards/accuracy_reward": 0.7828295826911926,
+      "rewards/format_reward": 0.993990421295166,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.58173370361328,
+      "epoch": 0.029411764705882353,
+      "grad_norm": 4.950891299350819,
+      "kl": 0.000789642333984375,
+      "learning_rate": 9.98529411764706e-07,
+      "loss": 0.0,
+      "reward": 1.797240972518921,
+      "reward_std": 0.06516173481941223,
+      "rewards/accuracy_reward": 0.8056544661521912,
+      "rewards/format_reward": 0.9915865659713745,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.5252456665039,
+      "epoch": 0.04411764705882353,
+      "grad_norm": 1.8505526818416387,
+      "kl": 0.00093841552734375,
+      "learning_rate": 9.977941176470588e-07,
+      "loss": 0.0,
+      "reward": 1.7827500104904175,
+      "reward_std": 0.06552266329526901,
+      "rewards/accuracy_reward": 0.7887594699859619,
+      "rewards/format_reward": 0.993990421295166,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.68870544433594,
+      "epoch": 0.058823529411764705,
+      "grad_norm": 1.3899081066234504,
+      "kl": 0.00102996826171875,
+      "learning_rate": 9.970588235294117e-07,
+      "loss": 0.0,
+      "reward": 1.7779340744018555,
+      "reward_std": 0.06615827977657318,
+      "rewards/accuracy_reward": 0.7851455807685852,
+      "rewards/format_reward": 0.9927884936332703,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.76563262939453,
+      "epoch": 0.07352941176470588,
+      "grad_norm": 2.31071643885251,
+      "kl": 0.00112152099609375,
+      "learning_rate": 9.963235294117647e-07,
+      "loss": 0.0,
+      "reward": 1.790160894393921,
+      "reward_std": 0.06119891628623009,
+      "rewards/accuracy_reward": 0.7949684262275696,
+      "rewards/format_reward": 0.9951923489570618,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.93389892578125,
+      "epoch": 0.08823529411764706,
+      "grad_norm": 5.892350008855451,
+      "kl": 0.001983642578125,
+      "learning_rate": 9.955882352941176e-07,
+      "loss": 0.0001,
+      "reward": 1.8021262884140015,
+      "reward_std": 0.06035131961107254,
+      "rewards/accuracy_reward": 0.8033280372619629,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.10938262939453,
+      "epoch": 0.10294117647058823,
+      "grad_norm": 1.4454822561506098,
+      "kl": 0.002685546875,
+      "learning_rate": 9.948529411764705e-07,
+      "loss": 0.0001,
+      "reward": 1.7747721672058105,
+      "reward_std": 0.06199384108185768,
+      "rewards/accuracy_reward": 0.7795796990394592,
+      "rewards/format_reward": 0.9951923489570618,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.11418914794922,
+      "epoch": 0.11764705882352941,
+      "grad_norm": 2.5592127837818732,
+      "kl": 0.0029449462890625,
+      "learning_rate": 9.941176470588236e-07,
+      "loss": 0.0001,
+      "reward": 1.7927086353302002,
+      "reward_std": 0.05324796214699745,
+      "rewards/accuracy_reward": 0.793910562992096,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.32933044433594,
+      "epoch": 0.1323529411764706,
+      "grad_norm": 1.2070688702059713,
+      "kl": 0.003509521484375,
+      "learning_rate": 9.933823529411765e-07,
+      "loss": 0.0001,
+      "reward": 1.807510495185852,
+      "reward_std": 0.05738683044910431,
+      "rewards/accuracy_reward": 0.8099143505096436,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.88702392578125,
+      "epoch": 0.14705882352941177,
+      "grad_norm": 1.1236907795600153,
+      "kl": 0.004150390625,
+      "learning_rate": 9.926470588235293e-07,
+      "loss": 0.0002,
+      "reward": 1.7909963130950928,
+      "reward_std": 0.05266621708869934,
+      "rewards/accuracy_reward": 0.7921981811523438,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.37981414794922,
+      "epoch": 0.16176470588235295,
+      "grad_norm": 1.6831920052248819,
+      "kl": 0.005645751953125,
+      "learning_rate": 9.919117647058824e-07,
+      "loss": 0.0002,
+      "reward": 1.805780053138733,
+      "reward_std": 0.05035817250609398,
+      "rewards/accuracy_reward": 0.8069819808006287,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.015625,
+      "epoch": 0.17647058823529413,
+      "grad_norm": 1.7093381711239366,
+      "kl": 0.0076904296875,
+      "learning_rate": 9.911764705882353e-07,
+      "loss": 0.0003,
+      "reward": 1.829386830329895,
+      "reward_std": 0.04737670719623566,
+      "rewards/accuracy_reward": 0.8305887579917908,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.73798370361328,
+      "epoch": 0.19117647058823528,
+      "grad_norm": 1.5253862652760264,
+      "kl": 0.00909423828125,
+      "learning_rate": 9.904411764705882e-07,
+      "loss": 0.0004,
+      "reward": 1.7849040031433105,
+      "reward_std": 0.05029381439089775,
+      "rewards/accuracy_reward": 0.7849039435386658,
+      "rewards/format_reward": 1.0,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.33534240722656,
+      "epoch": 0.20588235294117646,
+      "grad_norm": 1.5658868221984554,
+      "kl": 0.01055908203125,
+      "learning_rate": 9.897058823529412e-07,
+      "loss": 0.0004,
+      "reward": 1.803476095199585,
+      "reward_std": 0.04388910159468651,
+      "rewards/accuracy_reward": 0.803476095199585,
+      "rewards/format_reward": 1.0,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.25601196289062,
+      "epoch": 0.22058823529411764,
+      "grad_norm": 9.7316953003379,
+      "kl": 0.01141357421875,
+      "learning_rate": 9.889705882352941e-07,
+      "loss": 0.0005,
+      "reward": 1.791574239730835,
+      "reward_std": 0.05360911041498184,
+      "rewards/accuracy_reward": 0.7927761077880859,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.83293914794922,
+      "epoch": 0.23529411764705882,
+      "grad_norm": 2.8478472800650585,
+      "kl": 0.01220703125,
+      "learning_rate": 9.88235294117647e-07,
+      "loss": 0.0005,
+      "reward": 1.7902849912643433,
+      "reward_std": 0.051616914570331573,
+      "rewards/accuracy_reward": 0.7914868593215942,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.93509674072266,
+      "epoch": 0.25,
+      "grad_norm": 2.544057708424019,
+      "kl": 0.013427734375,
+      "learning_rate": 9.875e-07,
+      "loss": 0.0005,
+      "reward": 1.8025610446929932,
+      "reward_std": 0.04462622106075287,
+      "rewards/accuracy_reward": 0.8025608658790588,
+      "rewards/format_reward": 1.0,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.12860870361328,
+      "epoch": 0.2647058823529412,
+      "grad_norm": 2.256336954402879,
+      "kl": 0.0137939453125,
+      "learning_rate": 9.86764705882353e-07,
+      "loss": 0.0005,
+      "reward": 1.8160377740859985,
+      "reward_std": 0.049329712986946106,
+      "rewards/accuracy_reward": 0.81844162940979,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.83413696289062,
+      "epoch": 0.27941176470588236,
+      "grad_norm": 1.341985441593011,
+      "kl": 0.01318359375,
+      "learning_rate": 9.860294117647058e-07,
+      "loss": 0.0005,
+      "reward": 1.7914906740188599,
+      "reward_std": 0.04698958992958069,
+      "rewards/accuracy_reward": 0.7914904952049255,
+      "rewards/format_reward": 1.0,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.64183044433594,
+      "epoch": 0.29411764705882354,
+      "grad_norm": 1.1417020647628722,
+      "kl": 0.0126953125,
+      "learning_rate": 9.85294117647059e-07,
+      "loss": 0.0005,
+      "reward": 1.785807728767395,
+      "reward_std": 0.05590025335550308,
+      "rewards/accuracy_reward": 0.7906152606010437,
+      "rewards/format_reward": 0.9951923489570618,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.41106414794922,
+      "epoch": 0.3088235294117647,
+      "grad_norm": 1.3996656626391477,
+      "kl": 0.0118408203125,
+      "learning_rate": 9.845588235294118e-07,
+      "loss": 0.0005,
+      "reward": 1.8126260042190552,
+      "reward_std": 0.04200880602002144,
+      "rewards/accuracy_reward": 0.8126258254051208,
+      "rewards/format_reward": 1.0,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.72956848144531,
+      "epoch": 0.3235294117647059,
+      "grad_norm": 1.472735800879064,
+      "kl": 0.01214599609375,
+      "learning_rate": 9.838235294117647e-07,
+      "loss": 0.0005,
+      "reward": 1.819865107536316,
+      "reward_std": 0.046603430062532425,
+      "rewards/accuracy_reward": 0.8210669755935669,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.89904022216797,
+      "epoch": 0.3382352941176471,
+      "grad_norm": 1.5355589885672927,
+      "kl": 0.011962890625,
+      "learning_rate": 9.830882352941175e-07,
+      "loss": 0.0005,
+      "reward": 1.795883059501648,
+      "reward_std": 0.047029443085193634,
+      "rewards/accuracy_reward": 0.795883059501648,
+      "rewards/format_reward": 1.0,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.52764892578125,
+      "epoch": 0.35294117647058826,
+      "grad_norm": 1.7659139234832795,
+      "kl": 0.01116943359375,
+      "learning_rate": 9.823529411764704e-07,
+      "loss": 0.0005,
+      "reward": 1.819062352180481,
+      "reward_std": 0.045326538383960724,
+      "rewards/accuracy_reward": 0.819062352180481,
+      "rewards/format_reward": 1.0,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.76202392578125,
+      "epoch": 0.36764705882352944,
+      "grad_norm": 1.8387756083505316,
+      "kl": 0.0118408203125,
+      "learning_rate": 9.816176470588235e-07,
+      "loss": 0.0005,
+      "reward": 1.8244253396987915,
+      "reward_std": 0.04212662950158119,
+      "rewards/accuracy_reward": 0.8256272673606873,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.07572174072266,
+      "epoch": 0.38235294117647056,
+      "grad_norm": 2.5672205894145668,
+      "kl": 0.01123046875,
+      "learning_rate": 9.808823529411764e-07,
+      "loss": 0.0004,
+      "reward": 1.8216837644577026,
+      "reward_std": 0.03997746482491493,
+      "rewards/accuracy_reward": 0.8216836452484131,
+      "rewards/format_reward": 1.0,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.86659240722656,
+      "epoch": 0.39705882352941174,
+      "grad_norm": 1.9492202384769068,
+      "kl": 0.0111083984375,
+      "learning_rate": 9.801470588235292e-07,
+      "loss": 0.0004,
+      "reward": 1.8016602993011475,
+      "reward_std": 0.041357386857271194,
+      "rewards/accuracy_reward": 0.8028621077537537,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.33173370361328,
+      "epoch": 0.4117647058823529,
+      "grad_norm": 1.5368419770860244,
+      "kl": 0.01300048828125,
+      "learning_rate": 9.794117647058823e-07,
+      "loss": 0.0005,
+      "reward": 1.8133355379104614,
+      "reward_std": 0.040824707597494125,
+      "rewards/accuracy_reward": 0.8145372271537781,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.08413696289062,
+      "epoch": 0.4264705882352941,
+      "grad_norm": 1.922839715769838,
+      "kl": 0.01397705078125,
+      "learning_rate": 9.786764705882352e-07,
+      "loss": 0.0006,
+      "reward": 1.828223705291748,
+      "reward_std": 0.038493864238262177,
+      "rewards/accuracy_reward": 0.8294253945350647,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.17668914794922,
+      "epoch": 0.4411764705882353,
+      "grad_norm": 1.136750055086542,
+      "kl": 0.0137939453125,
+      "learning_rate": 9.77941176470588e-07,
+      "loss": 0.0006,
+      "reward": 1.7888784408569336,
+      "reward_std": 0.03950018435716629,
+      "rewards/accuracy_reward": 0.7888783812522888,
+      "rewards/format_reward": 1.0,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.23678588867188,
+      "epoch": 0.45588235294117646,
+      "grad_norm": 1.5207542293833838,
+      "kl": 0.013916015625,
+      "learning_rate": 9.772058823529412e-07,
+      "loss": 0.0006,
+      "reward": 1.8419325351715088,
+      "reward_std": 0.04379018768668175,
+      "rewards/accuracy_reward": 0.8443362712860107,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.47956848144531,
+      "epoch": 0.47058823529411764,
+      "grad_norm": 1.7773456348293046,
+      "kl": 0.0145263671875,
+      "learning_rate": 9.76470588235294e-07,
+      "loss": 0.0006,
+      "reward": 1.8314036130905151,
+      "reward_std": 0.03341993689537048,
+      "rewards/accuracy_reward": 0.8314037919044495,
+      "rewards/format_reward": 1.0,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.88702392578125,
+      "epoch": 0.4852941176470588,
+      "grad_norm": 1.4109993423092686,
+      "kl": 0.01544189453125,
+      "learning_rate": 9.75735294117647e-07,
+      "loss": 0.0006,
+      "reward": 1.8262065649032593,
+      "reward_std": 0.039565280079841614,
+      "rewards/accuracy_reward": 0.8286103010177612,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.64063262939453,
+      "epoch": 0.5,
+      "grad_norm": 1.8369759357942195,
+      "kl": 0.0150146484375,
+      "learning_rate": 9.75e-07,
+      "loss": 0.0006,
+      "reward": 1.7878358364105225,
+      "reward_std": 0.03487839177250862,
+      "rewards/accuracy_reward": 0.7890377044677734,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.1658706665039,
+      "epoch": 0.5147058823529411,
+      "grad_norm": 1.4792457830014905,
+      "kl": 0.0169677734375,
+      "learning_rate": 9.742647058823529e-07,
+      "loss": 0.0007,
+      "reward": 1.8230891227722168,
+      "reward_std": 0.03256754204630852,
+      "rewards/accuracy_reward": 0.823089063167572,
+      "rewards/format_reward": 1.0,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.7596206665039,
+      "epoch": 0.5294117647058824,
+      "grad_norm": 1.2939561711920475,
+      "kl": 0.0167236328125,
+      "learning_rate": 9.735294117647057e-07,
+      "loss": 0.0007,
+      "reward": 1.8332778215408325,
+      "reward_std": 0.03907008469104767,
+      "rewards/accuracy_reward": 0.835681676864624,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.78366088867188,
+      "epoch": 0.5441176470588235,
+      "grad_norm": 1.1548157711618783,
+      "kl": 0.0172119140625,
+      "learning_rate": 9.727941176470588e-07,
+      "loss": 0.0007,
+      "reward": 1.8284510374069214,
+      "reward_std": 0.03708866238594055,
+      "rewards/accuracy_reward": 0.8296528458595276,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.41346740722656,
+      "epoch": 0.5588235294117647,
+      "grad_norm": 1.811998457382516,
+      "kl": 0.0174560546875,
+      "learning_rate": 9.720588235294117e-07,
+      "loss": 0.0007,
+      "reward": 1.8531993627548218,
+      "reward_std": 0.03520160913467407,
+      "rewards/accuracy_reward": 0.8544012308120728,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.73798370361328,
+      "epoch": 0.5735294117647058,
+      "grad_norm": 5.782853992835495,
+      "kl": 0.0179443359375,
+      "learning_rate": 9.713235294117646e-07,
+      "loss": 0.0007,
+      "reward": 1.820478081703186,
+      "reward_std": 0.0349760465323925,
+      "rewards/accuracy_reward": 0.8204782009124756,
+      "rewards/format_reward": 1.0,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.25360870361328,
+      "epoch": 0.5882352941176471,
+      "grad_norm": 2.095875673750828,
+      "kl": 0.02001953125,
+      "learning_rate": 9.705882352941176e-07,
+      "loss": 0.0008,
+      "reward": 1.818333625793457,
+      "reward_std": 0.036586832255125046,
+      "rewards/accuracy_reward": 0.8195353746414185,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.5564956665039,
+      "epoch": 0.6029411764705882,
+      "grad_norm": 1.5695506518790454,
+      "kl": 0.0205078125,
+      "learning_rate": 9.698529411764705e-07,
+      "loss": 0.0008,
+      "reward": 1.7950727939605713,
+      "reward_std": 0.043350886553525925,
+      "rewards/accuracy_reward": 0.797476589679718,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.74159240722656,
+      "epoch": 0.6176470588235294,
+      "grad_norm": 4.732453320301059,
+      "kl": 0.0220947265625,
+      "learning_rate": 9.691176470588234e-07,
+      "loss": 0.0009,
+      "reward": 1.8049029111862183,
+      "reward_std": 0.03959096223115921,
+      "rewards/accuracy_reward": 0.8061047792434692,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.66706848144531,
+      "epoch": 0.6323529411764706,
+      "grad_norm": 2.4880576589131533,
+      "kl": 0.021484375,
+      "learning_rate": 9.683823529411765e-07,
+      "loss": 0.0009,
+      "reward": 1.7938587665557861,
+      "reward_std": 0.03622143343091011,
+      "rewards/accuracy_reward": 0.7938587665557861,
+      "rewards/format_reward": 1.0,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.97356414794922,
+      "epoch": 0.6470588235294118,
+      "grad_norm": 2.4237436345937122,
+      "kl": 0.022705078125,
+      "learning_rate": 9.676470588235294e-07,
+      "loss": 0.0009,
+      "reward": 1.8250298500061035,
+      "reward_std": 0.03410777822136879,
+      "rewards/accuracy_reward": 0.8262317776679993,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.92788696289062,
+      "epoch": 0.6617647058823529,
+      "grad_norm": 2.2338566414155463,
+      "kl": 0.022705078125,
+      "learning_rate": 9.669117647058822e-07,
+      "loss": 0.0009,
+      "reward": 1.8153414726257324,
+      "reward_std": 0.043387699872255325,
+      "rewards/accuracy_reward": 0.8189471364021301,
+      "rewards/format_reward": 0.9963942766189575,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.89543914794922,
+      "epoch": 0.6764705882352942,
+      "grad_norm": 11.225902629577423,
+      "kl": 0.0225830078125,
+      "learning_rate": 9.661764705882353e-07,
+      "loss": 0.0009,
+      "reward": 1.8124586343765259,
+      "reward_std": 0.033101461827754974,
+      "rewards/accuracy_reward": 0.8124586343765259,
+      "rewards/format_reward": 1.0,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.8738021850586,
+      "epoch": 0.6911764705882353,
+      "grad_norm": 2.4337206941159564,
+      "kl": 0.0216064453125,
+      "learning_rate": 9.654411764705882e-07,
+      "loss": 0.0009,
+      "reward": 1.82453191280365,
+      "reward_std": 0.033501800149679184,
+      "rewards/accuracy_reward": 0.8257339596748352,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.66466522216797,
+      "epoch": 0.7058823529411765,
+      "grad_norm": 1.6698840088478868,
+      "kl": 0.02587890625,
+      "learning_rate": 9.64705882352941e-07,
+      "loss": 0.001,
+      "reward": 1.8177635669708252,
+      "reward_std": 0.03327854722738266,
+      "rewards/accuracy_reward": 0.8177635073661804,
+      "rewards/format_reward": 1.0,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.1189956665039,
+      "epoch": 0.7205882352941176,
+      "grad_norm": 2.838491633259207,
+      "kl": 0.02294921875,
+      "learning_rate": 9.639705882352941e-07,
+      "loss": 0.0009,
+      "reward": 1.8294517993927002,
+      "reward_std": 0.03862498328089714,
+      "rewards/accuracy_reward": 0.8318556547164917,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.31250762939453,
+      "epoch": 0.7352941176470589,
+      "grad_norm": 1.6937318420428593,
+      "kl": 0.0234375,
+      "learning_rate": 9.63235294117647e-07,
+      "loss": 0.0009,
+      "reward": 1.8327486515045166,
+      "reward_std": 0.03481902554631233,
+      "rewards/accuracy_reward": 0.8339505195617676,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.50241088867188,
+      "epoch": 0.75,
+      "grad_norm": 1.9451567930831182,
+      "kl": 0.0245361328125,
+      "learning_rate": 9.624999999999999e-07,
+      "loss": 0.001,
+      "reward": 1.839182734489441,
+      "reward_std": 0.029496613889932632,
+      "rewards/accuracy_reward": 0.8391827940940857,
+      "rewards/format_reward": 1.0,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.64904022216797,
+      "epoch": 0.7647058823529411,
+      "grad_norm": 1.9537307288218644,
+      "kl": 0.023681640625,
+      "learning_rate": 9.61764705882353e-07,
+      "loss": 0.0009,
+      "reward": 1.8222689628601074,
+      "reward_std": 0.038959018886089325,
+      "rewards/accuracy_reward": 0.8234708309173584,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.05529022216797,
+      "epoch": 0.7794117647058824,
+      "grad_norm": 2.0869715773084634,
+      "kl": 0.022705078125,
+      "learning_rate": 9.610294117647058e-07,
+      "loss": 0.0009,
+      "reward": 1.8396120071411133,
+      "reward_std": 0.030058706179261208,
+      "rewards/accuracy_reward": 0.8396119475364685,
+      "rewards/format_reward": 1.0,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.25241088867188,
+      "epoch": 0.7941176470588235,
+      "grad_norm": 1.399697409291221,
+      "kl": 0.0230712890625,
+      "learning_rate": 9.602941176470587e-07,
+      "loss": 0.0009,
+      "reward": 1.8265293836593628,
+      "reward_std": 0.02903612144291401,
+      "rewards/accuracy_reward": 0.8265291452407837,
+      "rewards/format_reward": 1.0,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.35697174072266,
+      "epoch": 0.8088235294117647,
+      "grad_norm": 1.9480357994696749,
+      "kl": 0.022705078125,
+      "learning_rate": 9.595588235294118e-07,
+      "loss": 0.0009,
+      "reward": 1.8046581745147705,
+      "reward_std": 0.030282124876976013,
+      "rewards/accuracy_reward": 0.8046582341194153,
+      "rewards/format_reward": 1.0,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.97356414794922,
+      "epoch": 0.8235294117647058,
+      "grad_norm": 1.8194034842124802,
+      "kl": 0.0218505859375,
+      "learning_rate": 9.588235294117647e-07,
+      "loss": 0.0009,
+      "reward": 1.8263245820999146,
+      "reward_std": 0.029453594237565994,
+      "rewards/accuracy_reward": 0.8275262713432312,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.06731414794922,
+      "epoch": 0.8382352941176471,
+      "grad_norm": 1.747459486428414,
+      "kl": 0.02001953125,
+      "learning_rate": 9.580882352941176e-07,
+      "loss": 0.0008,
+      "reward": 1.8315790891647339,
+      "reward_std": 0.02826782502233982,
+      "rewards/accuracy_reward": 0.8315791487693787,
+      "rewards/format_reward": 1.0,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.09616088867188,
+      "epoch": 0.8529411764705882,
+      "grad_norm": 2.2648825774011856,
+      "kl": 0.019775390625,
+      "learning_rate": 9.573529411764706e-07,
+      "loss": 0.0008,
+      "reward": 1.8353791236877441,
+      "reward_std": 0.028635086491703987,
+      "rewards/accuracy_reward": 0.8353790640830994,
+      "rewards/format_reward": 1.0,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.56851196289062,
+      "epoch": 0.8676470588235294,
+      "grad_norm": 1.3353893541096462,
+      "kl": 0.02001953125,
+      "learning_rate": 9.566176470588235e-07,
+      "loss": 0.0008,
+      "reward": 1.8366105556488037,
+      "reward_std": 0.029847299680113792,
+      "rewards/accuracy_reward": 0.8378124833106995,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.87741088867188,
+      "epoch": 0.8823529411764706,
+      "grad_norm": 1.2231786582956035,
+      "kl": 0.021484375,
+      "learning_rate": 9.558823529411764e-07,
+      "loss": 0.0009,
+      "reward": 1.8441990613937378,
+      "reward_std": 0.029375040903687477,
+      "rewards/accuracy_reward": 0.8441989421844482,
+      "rewards/format_reward": 1.0,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.85697174072266,
+      "epoch": 0.8970588235294118,
+      "grad_norm": 1.6690678560710506,
+      "kl": 0.01904296875,
+      "learning_rate": 9.551470588235295e-07,
+      "loss": 0.0008,
+      "reward": 1.8236721754074097,
+      "reward_std": 0.02699529379606247,
+      "rewards/accuracy_reward": 0.8236721158027649,
+      "rewards/format_reward": 1.0,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.2908706665039,
+      "epoch": 0.9117647058823529,
+      "grad_norm": 6.089899029100757,
+      "kl": 0.019287109375,
+      "learning_rate": 9.544117647058823e-07,
+      "loss": 0.0008,
+      "reward": 1.8286242485046387,
+      "reward_std": 0.02875707298517227,
+      "rewards/accuracy_reward": 0.8286241888999939,
+      "rewards/format_reward": 1.0,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.53846740722656,
+      "epoch": 0.9264705882352942,
+      "grad_norm": 1.8276415184151562,
+      "kl": 0.0201416015625,
+      "learning_rate": 9.536764705882352e-07,
+      "loss": 0.0008,
+      "reward": 1.8171989917755127,
+      "reward_std": 0.033383674919605255,
+      "rewards/accuracy_reward": 0.8184008002281189,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.20553588867188,
+      "epoch": 0.9411764705882353,
+      "grad_norm": 2.253394799784533,
+      "kl": 0.0206298828125,
+      "learning_rate": 9.529411764705881e-07,
+      "loss": 0.0008,
+      "reward": 1.8275461196899414,
+      "reward_std": 0.033603277057409286,
+      "rewards/accuracy_reward": 0.8287478685379028,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.28366088867188,
+      "epoch": 0.9558823529411765,
+      "grad_norm": 1.9640142035092505,
+      "kl": 0.0203857421875,
+      "learning_rate": 9.522058823529411e-07,
+      "loss": 0.0008,
+      "reward": 1.8377821445465088,
+      "reward_std": 0.027394887059926987,
+      "rewards/accuracy_reward": 0.8389840126037598,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.56971740722656,
+      "epoch": 0.9705882352941176,
+      "grad_norm": 1.7539526199640756,
+      "kl": 0.021484375,
+      "learning_rate": 9.51470588235294e-07,
+      "loss": 0.0009,
+      "reward": 1.8167920112609863,
+      "reward_std": 0.02794465236365795,
+      "rewards/accuracy_reward": 0.8167920112609863,
+      "rewards/format_reward": 1.0,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.77284240722656,
+      "epoch": 0.9852941176470589,
+      "grad_norm": 1.2641268462386457,
+      "kl": 0.0228271484375,
+      "learning_rate": 9.507352941176469e-07,
+      "loss": 0.0009,
+      "reward": 1.8030400276184082,
+      "reward_std": 0.03198782354593277,
+      "rewards/accuracy_reward": 0.8030399680137634,
+      "rewards/format_reward": 1.0,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.8310546875,
+      "epoch": 1.0,
+      "grad_norm": 1.63346443563275,
+      "kl": 0.022216796875,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0009,
+      "reward": 1.8402751684188843,
+      "reward_std": 0.028587892651557922,
+      "rewards/accuracy_reward": 0.8402751684188843,
+      "rewards/format_reward": 1.0,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.18991088867188,
+      "epoch": 1.0147058823529411,
+      "grad_norm": 2.0248373653875955,
+      "kl": 0.0225830078125,
+      "learning_rate": 9.492647058823529e-07,
+      "loss": 0.0009,
+      "reward": 1.8379939794540405,
+      "reward_std": 0.02731536142528057,
+      "rewards/accuracy_reward": 0.8379938006401062,
+      "rewards/format_reward": 1.0,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.76923370361328,
+      "epoch": 1.0294117647058822,
+      "grad_norm": 1.2467348089074899,
+      "kl": 0.0224609375,
+      "learning_rate": 9.485294117647058e-07,
+      "loss": 0.0009,
+      "reward": 1.839119553565979,
+      "reward_std": 0.027092870324850082,
+      "rewards/accuracy_reward": 0.8391194939613342,
+      "rewards/format_reward": 1.0,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.04447174072266,
+      "epoch": 1.0441176470588236,
+      "grad_norm": 1.7733203946188572,
+      "kl": 0.024169921875,
+      "learning_rate": 9.477941176470587e-07,
+      "loss": 0.001,
+      "reward": 1.835349678993225,
+      "reward_std": 0.028652215376496315,
+      "rewards/accuracy_reward": 0.8365515470504761,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.54206848144531,
+      "epoch": 1.0588235294117647,
+      "grad_norm": 1.417496550727975,
+      "kl": 0.0255126953125,
+      "learning_rate": 9.470588235294117e-07,
+      "loss": 0.001,
+      "reward": 1.8174885511398315,
+      "reward_std": 0.03328551724553108,
+      "rewards/accuracy_reward": 0.8186904191970825,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.22956848144531,
+      "epoch": 1.0735294117647058,
+      "grad_norm": 2.347300550273842,
+      "kl": 0.022705078125,
+      "learning_rate": 9.463235294117646e-07,
+      "loss": 0.0009,
+      "reward": 1.8195103406906128,
+      "reward_std": 0.028913108631968498,
+      "rewards/accuracy_reward": 0.8195103406906128,
+      "rewards/format_reward": 1.0,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.82572174072266,
+      "epoch": 1.088235294117647,
+      "grad_norm": 2.5948956191316306,
+      "kl": 0.024169921875,
+      "learning_rate": 9.455882352941176e-07,
+      "loss": 0.001,
+      "reward": 1.8160063028335571,
+      "reward_std": 0.03166431561112404,
+      "rewards/accuracy_reward": 0.8160063028335571,
+      "rewards/format_reward": 1.0,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.91106414794922,
+      "epoch": 1.1029411764705883,
+      "grad_norm": 2.2984611372489847,
+      "kl": 0.0245361328125,
+      "learning_rate": 9.448529411764705e-07,
+      "loss": 0.001,
+      "reward": 1.841678261756897,
+      "reward_std": 0.028328273445367813,
+      "rewards/accuracy_reward": 0.8416782021522522,
+      "rewards/format_reward": 1.0,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.30769348144531,
+      "epoch": 1.1176470588235294,
+      "grad_norm": 1.2558402021274238,
+      "kl": 0.025146484375,
+      "learning_rate": 9.441176470588234e-07,
+      "loss": 0.001,
+      "reward": 1.8383431434631348,
+      "reward_std": 0.02794811502099037,
+      "rewards/accuracy_reward": 0.8395450711250305,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.67428588867188,
+      "epoch": 1.1323529411764706,
+      "grad_norm": 1.4150029392598835,
+      "kl": 0.02392578125,
+      "learning_rate": 9.433823529411764e-07,
+      "loss": 0.001,
+      "reward": 1.8399676084518433,
+      "reward_std": 0.025521790608763695,
+      "rewards/accuracy_reward": 0.8399675488471985,
+      "rewards/format_reward": 1.0,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.96034240722656,
+      "epoch": 1.1470588235294117,
+      "grad_norm": 1.1950823033585802,
+      "kl": 0.0264892578125,
+      "learning_rate": 9.426470588235294e-07,
+      "loss": 0.0011,
+      "reward": 1.83869206905365,
+      "reward_std": 0.030266188085079193,
+      "rewards/accuracy_reward": 0.8398938775062561,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.77284240722656,
+      "epoch": 1.161764705882353,
+      "grad_norm": 3.5214271570942355,
+      "kl": 0.0272216796875,
+      "learning_rate": 9.419117647058823e-07,
+      "loss": 0.0011,
+      "reward": 1.8469470739364624,
+      "reward_std": 0.025917939841747284,
+      "rewards/accuracy_reward": 0.8469470143318176,
+      "rewards/format_reward": 1.0,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.43991088867188,
+      "epoch": 1.1764705882352942,
+      "grad_norm": 1.5192920476546525,
+      "kl": 0.027099609375,
+      "learning_rate": 9.411764705882352e-07,
+      "loss": 0.0011,
+      "reward": 1.8514484167099,
+      "reward_std": 0.02656027302145958,
+      "rewards/accuracy_reward": 0.8514483571052551,
+      "rewards/format_reward": 1.0,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.13581848144531,
+      "epoch": 1.1911764705882353,
+      "grad_norm": 1.4307870600496726,
+      "kl": 0.027099609375,
+      "learning_rate": 9.404411764705882e-07,
+      "loss": 0.0011,
+      "reward": 1.848158359527588,
+      "reward_std": 0.03505507484078407,
+      "rewards/accuracy_reward": 0.8505621552467346,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.22356414794922,
+      "epoch": 1.2058823529411764,
+      "grad_norm": 1.9122780286883485,
+      "kl": 0.042724609375,
+      "learning_rate": 9.397058823529411e-07,
+      "loss": 0.0017,
+      "reward": 1.8507599830627441,
+      "reward_std": 0.027777796611189842,
+      "rewards/accuracy_reward": 0.8519617319107056,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.24159240722656,
+      "epoch": 1.2205882352941178,
+      "grad_norm": 1.2485868300090033,
+      "kl": 0.0281982421875,
+      "learning_rate": 9.389705882352941e-07,
+      "loss": 0.0011,
+      "reward": 1.8573682308197021,
+      "reward_std": 0.028743639588356018,
+      "rewards/accuracy_reward": 0.8585702180862427,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.99038696289062,
+      "epoch": 1.2352941176470589,
+      "grad_norm": 1.6395721460640882,
+      "kl": 0.02587890625,
+      "learning_rate": 9.38235294117647e-07,
+      "loss": 0.001,
+      "reward": 1.849989414215088,
+      "reward_std": 0.03292028233408928,
+      "rewards/accuracy_reward": 0.8523932695388794,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.15144348144531,
+      "epoch": 1.25,
+      "grad_norm": 1.8788708892556627,
+      "kl": 0.029296875,
+      "learning_rate": 9.374999999999999e-07,
+      "loss": 0.0012,
+      "reward": 1.8299332857131958,
+      "reward_std": 0.030824122950434685,
+      "rewards/accuracy_reward": 0.832336962223053,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.70072174072266,
+      "epoch": 1.2647058823529411,
+      "grad_norm": 1.284626382800343,
+      "kl": 0.0257568359375,
+      "learning_rate": 9.367647058823529e-07,
+      "loss": 0.001,
+      "reward": 1.8247534036636353,
+      "reward_std": 0.02943887747824192,
+      "rewards/accuracy_reward": 0.825955331325531,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.10938262939453,
+      "epoch": 1.2794117647058822,
+      "grad_norm": 1.800940971026441,
+      "kl": 0.0291748046875,
+      "learning_rate": 9.360294117647059e-07,
+      "loss": 0.0012,
+      "reward": 1.829021692276001,
+      "reward_std": 0.03014710173010826,
+      "rewards/accuracy_reward": 0.8290215730667114,
+      "rewards/format_reward": 1.0,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.72356414794922,
+      "epoch": 1.2941176470588236,
+      "grad_norm": 1.6669503142403084,
+      "kl": 0.027587890625,
+      "learning_rate": 9.352941176470588e-07,
+      "loss": 0.0011,
+      "reward": 1.8365869522094727,
+      "reward_std": 0.035319458693265915,
+      "rewards/accuracy_reward": 0.8389906287193298,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.84495544433594,
+      "epoch": 1.3088235294117647,
+      "grad_norm": 1.9827979032477325,
+      "kl": 0.0267333984375,
+      "learning_rate": 9.345588235294117e-07,
+      "loss": 0.0011,
+      "reward": 1.8515337705612183,
+      "reward_std": 0.032718025147914886,
+      "rewards/accuracy_reward": 0.8527354598045349,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.54688262939453,
+      "epoch": 1.3235294117647058,
+      "grad_norm": 1.855181016525942,
+      "kl": 0.029541015625,
+      "learning_rate": 9.338235294117647e-07,
+      "loss": 0.0012,
+      "reward": 1.8531250953674316,
+      "reward_std": 0.03333236649632454,
+      "rewards/accuracy_reward": 0.8555287718772888,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.51683044433594,
+      "epoch": 1.3382352941176472,
+      "grad_norm": 1.542405666476553,
+      "kl": 0.0299072265625,
+      "learning_rate": 9.330882352941176e-07,
+      "loss": 0.0012,
+      "reward": 1.8587943315505981,
+      "reward_std": 0.02777322567999363,
+      "rewards/accuracy_reward": 0.8587943315505981,
+      "rewards/format_reward": 1.0,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.65866088867188,
+      "epoch": 1.3529411764705883,
+      "grad_norm": 6.044720279013163,
+      "kl": 0.027099609375,
+      "learning_rate": 9.323529411764706e-07,
+      "loss": 0.0011,
+      "reward": 1.8246790170669556,
+      "reward_std": 0.03338155895471573,
+      "rewards/accuracy_reward": 0.8258809447288513,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.25481414794922,
+      "epoch": 1.3676470588235294,
+      "grad_norm": 3.698518846068984,
+      "kl": 0.026611328125,
+      "learning_rate": 9.316176470588235e-07,
+      "loss": 0.0011,
+      "reward": 1.8631399869918823,
+      "reward_std": 0.027002353221178055,
+      "rewards/accuracy_reward": 0.8631399869918823,
+      "rewards/format_reward": 1.0,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.18870544433594,
+      "epoch": 1.3823529411764706,
+      "grad_norm": 7.435870249889465,
+      "kl": 0.027587890625,
+      "learning_rate": 9.308823529411764e-07,
+      "loss": 0.0011,
+      "reward": 1.8605844974517822,
+      "reward_std": 0.02926478162407875,
+      "rewards/accuracy_reward": 0.8605843782424927,
+      "rewards/format_reward": 1.0,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.17188262939453,
+      "epoch": 1.3970588235294117,
+      "grad_norm": 3.53776006589639,
+      "kl": 0.038818359375,
+      "learning_rate": 9.301470588235294e-07,
+      "loss": 0.0016,
+      "reward": 1.8662550449371338,
+      "reward_std": 0.028098618611693382,
+      "rewards/accuracy_reward": 0.8662549257278442,
+      "rewards/format_reward": 1.0,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.53606414794922,
+      "epoch": 1.4117647058823528,
+      "grad_norm": 2.008765777352944,
+      "kl": 0.0277099609375,
+      "learning_rate": 9.294117647058824e-07,
+      "loss": 0.0011,
+      "reward": 1.8323391675949097,
+      "reward_std": 0.03244079276919365,
+      "rewards/accuracy_reward": 0.8335410356521606,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.32813262939453,
+      "epoch": 1.4264705882352942,
+      "grad_norm": 1.6277308554524441,
+      "kl": 0.026123046875,
+      "learning_rate": 9.286764705882352e-07,
+      "loss": 0.001,
+      "reward": 1.8551498651504517,
+      "reward_std": 0.027026960626244545,
+      "rewards/accuracy_reward": 0.8551498055458069,
+      "rewards/format_reward": 1.0,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.96514892578125,
+      "epoch": 1.4411764705882353,
+      "grad_norm": 1.8674445586310056,
+      "kl": 0.027587890625,
+      "learning_rate": 9.279411764705882e-07,
+      "loss": 0.0011,
+      "reward": 1.8556426763534546,
+      "reward_std": 0.02985842153429985,
+      "rewards/accuracy_reward": 0.855642557144165,
+      "rewards/format_reward": 1.0,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.22116088867188,
+      "epoch": 1.4558823529411764,
+      "grad_norm": 2.564978355227823,
+      "kl": 0.031494140625,
+      "learning_rate": 9.272058823529412e-07,
+      "loss": 0.0013,
+      "reward": 1.8616751432418823,
+      "reward_std": 0.028857456520199776,
+      "rewards/accuracy_reward": 0.8628771305084229,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.96034240722656,
+      "epoch": 1.4705882352941178,
+      "grad_norm": 1.5062130671090712,
+      "kl": 0.029052734375,
+      "learning_rate": 9.264705882352941e-07,
+      "loss": 0.0012,
+      "reward": 1.8399274349212646,
+      "reward_std": 0.033529430627822876,
+      "rewards/accuracy_reward": 0.8411291837692261,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.31731414794922,
+      "epoch": 1.4852941176470589,
+      "grad_norm": 1.7672687152261588,
+      "kl": 0.0301513671875,
+      "learning_rate": 9.257352941176471e-07,
+      "loss": 0.0012,
+      "reward": 1.8533381223678589,
+      "reward_std": 0.03230126574635506,
+      "rewards/accuracy_reward": 0.8545399308204651,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.50601196289062,
+      "epoch": 1.5,
+      "grad_norm": 1.2513961138527014,
+      "kl": 0.0294189453125,
+      "learning_rate": 9.25e-07,
+      "loss": 0.0012,
+      "reward": 1.8355013132095337,
+      "reward_std": 0.03523796796798706,
+      "rewards/accuracy_reward": 0.8367031812667847,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.85096740722656,
+      "epoch": 1.5147058823529411,
+      "grad_norm": 1.7612446743928787,
+      "kl": 0.0286865234375,
+      "learning_rate": 9.242647058823529e-07,
+      "loss": 0.0011,
+      "reward": 1.835523247718811,
+      "reward_std": 0.028739692643284798,
+      "rewards/accuracy_reward": 0.8355231285095215,
+      "rewards/format_reward": 1.0,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.06971740722656,
+      "epoch": 1.5294117647058822,
+      "grad_norm": 1.2684013897529154,
+      "kl": 0.0296630859375,
+      "learning_rate": 9.235294117647059e-07,
+      "loss": 0.0012,
+      "reward": 1.8417116403579712,
+      "reward_std": 0.03294159471988678,
+      "rewards/accuracy_reward": 0.8429136872291565,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.39063262939453,
+      "epoch": 1.5441176470588234,
+      "grad_norm": 1.5110927030527903,
+      "kl": 0.0263671875,
+      "learning_rate": 9.227941176470589e-07,
+      "loss": 0.0011,
+      "reward": 1.8465179204940796,
+      "reward_std": 0.035068195313215256,
+      "rewards/accuracy_reward": 0.8489217162132263,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.76202392578125,
+      "epoch": 1.5588235294117647,
+      "grad_norm": 4.89157193030413,
+      "kl": 0.0299072265625,
+      "learning_rate": 9.220588235294117e-07,
+      "loss": 0.0012,
+      "reward": 1.8534250259399414,
+      "reward_std": 0.029864592477679253,
+      "rewards/accuracy_reward": 0.8534249663352966,
+      "rewards/format_reward": 1.0,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.63822174072266,
+      "epoch": 1.5735294117647058,
+      "grad_norm": 1.397310457995284,
+      "kl": 0.0264892578125,
+      "learning_rate": 9.213235294117646e-07,
+      "loss": 0.0011,
+      "reward": 1.8602780103683472,
+      "reward_std": 0.0308562982827425,
+      "rewards/accuracy_reward": 0.8614799380302429,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.93870544433594,
+      "epoch": 1.5882352941176472,
+      "grad_norm": 2.2153563584401867,
+      "kl": 0.027587890625,
+      "learning_rate": 9.205882352941176e-07,
+      "loss": 0.0011,
+      "reward": 1.8766840696334839,
+      "reward_std": 0.032767944037914276,
+      "rewards/accuracy_reward": 0.8778861165046692,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.41706848144531,
+      "epoch": 1.6029411764705883,
+      "grad_norm": 1.3441095236077332,
+      "kl": 0.0277099609375,
+      "learning_rate": 9.198529411764705e-07,
+      "loss": 0.0011,
+      "reward": 1.8476648330688477,
+      "reward_std": 0.03170119971036911,
+      "rewards/accuracy_reward": 0.8476646542549133,
+      "rewards/format_reward": 1.0,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.37500762939453,
+      "epoch": 1.6176470588235294,
+      "grad_norm": 2.087440395995137,
+      "kl": 0.0302734375,
+      "learning_rate": 9.191176470588234e-07,
+      "loss": 0.0012,
+      "reward": 1.8472872972488403,
+      "reward_std": 0.032663196325302124,
+      "rewards/accuracy_reward": 0.8484892249107361,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.37259674072266,
+      "epoch": 1.6323529411764706,
+      "grad_norm": 1.540582934098744,
+      "kl": 0.031494140625,
+      "learning_rate": 9.183823529411764e-07,
+      "loss": 0.0013,
+      "reward": 1.8425360918045044,
+      "reward_std": 0.0323411226272583,
+      "rewards/accuracy_reward": 0.8425360918045044,
+      "rewards/format_reward": 1.0,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.81851196289062,
+      "epoch": 1.6470588235294117,
+      "grad_norm": 2.4116409582345297,
+      "kl": 0.02978515625,
+      "learning_rate": 9.176470588235293e-07,
+      "loss": 0.0012,
+      "reward": 1.8361573219299316,
+      "reward_std": 0.0333629846572876,
+      "rewards/accuracy_reward": 0.8373592495918274,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.86659240722656,
+      "epoch": 1.6617647058823528,
+      "grad_norm": 2.020278808245713,
+      "kl": 0.031005859375,
+      "learning_rate": 9.169117647058823e-07,
+      "loss": 0.0012,
+      "reward": 1.8621249198913574,
+      "reward_std": 0.030686957761645317,
+      "rewards/accuracy_reward": 0.8621248602867126,
+      "rewards/format_reward": 1.0,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.34134674072266,
+      "epoch": 1.6764705882352942,
+      "grad_norm": 2.599149661763795,
+      "kl": 0.0272216796875,
+      "learning_rate": 9.161764705882353e-07,
+      "loss": 0.0011,
+      "reward": 1.8338128328323364,
+      "reward_std": 0.03273686766624451,
+      "rewards/accuracy_reward": 0.8350145220756531,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.04206848144531,
+      "epoch": 1.6911764705882353,
+      "grad_norm": 1.696964274521337,
+      "kl": 0.02783203125,
+      "learning_rate": 9.154411764705881e-07,
+      "loss": 0.0011,
+      "reward": 1.8655701875686646,
+      "reward_std": 0.026736797764897346,
+      "rewards/accuracy_reward": 0.8655699491500854,
+      "rewards/format_reward": 1.0,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.17788696289062,
+      "epoch": 1.7058823529411766,
+      "grad_norm": 1.6668443133949866,
+      "kl": 0.02880859375,
+      "learning_rate": 9.147058823529411e-07,
+      "loss": 0.0012,
+      "reward": 1.859294056892395,
+      "reward_std": 0.02911749854683876,
+      "rewards/accuracy_reward": 0.8604959845542908,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.41706848144531,
+      "epoch": 1.7205882352941178,
+      "grad_norm": 1.8917048442273852,
+      "kl": 0.0289306640625,
+      "learning_rate": 9.139705882352941e-07,
+      "loss": 0.0012,
+      "reward": 1.8411531448364258,
+      "reward_std": 0.03658343106508255,
+      "rewards/accuracy_reward": 0.8471627831459045,
+      "rewards/format_reward": 0.993990421295166,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.32813262939453,
+      "epoch": 1.7352941176470589,
+      "grad_norm": 1.6450031082088983,
+      "kl": 0.0279541015625,
+      "learning_rate": 9.13235294117647e-07,
+      "loss": 0.0011,
+      "reward": 1.8670862913131714,
+      "reward_std": 0.026890011504292488,
+      "rewards/accuracy_reward": 0.8670862913131714,
+      "rewards/format_reward": 1.0,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.88341522216797,
+      "epoch": 1.75,
+      "grad_norm": 3.248672120006505,
+      "kl": 0.029052734375,
+      "learning_rate": 9.124999999999999e-07,
+      "loss": 0.0012,
+      "reward": 1.8559081554412842,
+      "reward_std": 0.02590222656726837,
+      "rewards/accuracy_reward": 0.8559079170227051,
+      "rewards/format_reward": 1.0,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.36779022216797,
+      "epoch": 1.7647058823529411,
+      "grad_norm": 20.773145033253595,
+      "kl": 0.0279541015625,
+      "learning_rate": 9.117647058823529e-07,
+      "loss": 0.0011,
+      "reward": 1.8575422763824463,
+      "reward_std": 0.024342134594917297,
+      "rewards/accuracy_reward": 0.8575422763824463,
+      "rewards/format_reward": 1.0,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.59375762939453,
+      "epoch": 1.7794117647058822,
+      "grad_norm": 1.4987523214554013,
+      "kl": 0.0283203125,
+      "learning_rate": 9.110294117647058e-07,
+      "loss": 0.0011,
+      "reward": 1.8683563470840454,
+      "reward_std": 0.023474786430597305,
+      "rewards/accuracy_reward": 0.8683561682701111,
+      "rewards/format_reward": 1.0,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.4363021850586,
+      "epoch": 1.7941176470588234,
+      "grad_norm": 2.899436959677707,
+      "kl": 0.02783203125,
+      "learning_rate": 9.102941176470588e-07,
+      "loss": 0.0011,
+      "reward": 1.82746160030365,
+      "reward_std": 0.027238713577389717,
+      "rewards/accuracy_reward": 0.8286634683609009,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.24279022216797,
+      "epoch": 1.8088235294117647,
+      "grad_norm": 1.4877031960129163,
+      "kl": 0.0286865234375,
+      "learning_rate": 9.095588235294118e-07,
+      "loss": 0.0011,
+      "reward": 1.8483201265335083,
+      "reward_std": 0.024529799818992615,
+      "rewards/accuracy_reward": 0.8483200669288635,
+      "rewards/format_reward": 1.0,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.35456848144531,
+      "epoch": 1.8235294117647058,
+      "grad_norm": 2.1149935830252473,
+      "kl": 0.0308837890625,
+      "learning_rate": 9.088235294117646e-07,
+      "loss": 0.0012,
+      "reward": 1.8580820560455322,
+      "reward_std": 0.026496434584259987,
+      "rewards/accuracy_reward": 0.8580819964408875,
+      "rewards/format_reward": 1.0,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.50721740722656,
+      "epoch": 1.8382352941176472,
+      "grad_norm": 5.027371264695678,
+      "kl": 0.031005859375,
+      "learning_rate": 9.080882352941176e-07,
+      "loss": 0.0012,
+      "reward": 1.867140293121338,
+      "reward_std": 0.02153668738901615,
+      "rewards/accuracy_reward": 0.8671402931213379,
+      "rewards/format_reward": 1.0,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.4675521850586,
+      "epoch": 1.8529411764705883,
+      "grad_norm": 1.2899741793420323,
+      "kl": 0.028076171875,
+      "learning_rate": 9.073529411764706e-07,
+      "loss": 0.0011,
+      "reward": 1.8652453422546387,
+      "reward_std": 0.02116318792104721,
+      "rewards/accuracy_reward": 0.8652453422546387,
+      "rewards/format_reward": 1.0,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.6502456665039,
+      "epoch": 1.8676470588235294,
+      "grad_norm": 2.179252488296008,
+      "kl": 0.031494140625,
+      "learning_rate": 9.066176470588235e-07,
+      "loss": 0.0013,
+      "reward": 1.8535821437835693,
+      "reward_std": 0.02199378050863743,
+      "rewards/accuracy_reward": 0.8535822033882141,
+      "rewards/format_reward": 1.0,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.88581848144531,
+      "epoch": 1.8823529411764706,
+      "grad_norm": 1.4612044453819617,
+      "kl": 0.033447265625,
+      "learning_rate": 9.058823529411764e-07,
+      "loss": 0.0013,
+      "reward": 1.8556383848190308,
+      "reward_std": 0.018579188734292984,
+      "rewards/accuracy_reward": 0.8556383848190308,
+      "rewards/format_reward": 1.0,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.41346740722656,
+      "epoch": 1.8970588235294117,
+      "grad_norm": 1.5286682380949737,
+      "kl": 0.0296630859375,
+      "learning_rate": 9.051470588235294e-07,
+      "loss": 0.0012,
+      "reward": 1.83783757686615,
+      "reward_std": 0.025251705199480057,
+      "rewards/accuracy_reward": 0.8378376364707947,
+      "rewards/format_reward": 1.0,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.98438262939453,
+      "epoch": 1.9117647058823528,
+      "grad_norm": 2.4536921352293892,
+      "kl": 0.029296875,
+      "learning_rate": 9.044117647058823e-07,
+      "loss": 0.0012,
+      "reward": 1.866936445236206,
+      "reward_std": 0.022518165409564972,
+      "rewards/accuracy_reward": 0.8669363260269165,
+      "rewards/format_reward": 1.0,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.33534240722656,
+      "epoch": 1.9264705882352942,
+      "grad_norm": 1.5117857063832663,
+      "kl": 0.0284423828125,
+      "learning_rate": 9.036764705882353e-07,
+      "loss": 0.0011,
+      "reward": 1.8531559705734253,
+      "reward_std": 0.024603500962257385,
+      "rewards/accuracy_reward": 0.8543578386306763,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 131
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.3533706665039,
+      "epoch": 1.9411764705882353,
+      "grad_norm": 2.703283581674193,
+      "kl": 0.0308837890625,
+      "learning_rate": 9.029411764705883e-07,
+      "loss": 0.0012,
+      "reward": 1.8581082820892334,
+      "reward_std": 0.02527976781129837,
+      "rewards/accuracy_reward": 0.8581082820892334,
+      "rewards/format_reward": 1.0,
+      "step": 132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.80048370361328,
+      "epoch": 1.9558823529411766,
+      "grad_norm": 2.173287786867425,
+      "kl": 0.029052734375,
+      "learning_rate": 9.022058823529411e-07,
+      "loss": 0.0012,
+      "reward": 1.8408198356628418,
+      "reward_std": 0.02469984069466591,
+      "rewards/accuracy_reward": 0.8408196568489075,
+      "rewards/format_reward": 1.0,
+      "step": 133
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.5769271850586,
+      "epoch": 1.9705882352941178,
+      "grad_norm": 1.4534120952263125,
+      "kl": 0.0306396484375,
+      "learning_rate": 9.014705882352941e-07,
+      "loss": 0.0012,
+      "reward": 1.828810214996338,
+      "reward_std": 0.023025646805763245,
+      "rewards/accuracy_reward": 0.8288100361824036,
+      "rewards/format_reward": 1.0,
+      "step": 134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.14904022216797,
+      "epoch": 1.9852941176470589,
+      "grad_norm": 2.390365626608761,
+      "kl": 0.031494140625,
+      "learning_rate": 9.007352941176471e-07,
+      "loss": 0.0013,
+      "reward": 1.8384226560592651,
+      "reward_std": 0.023323819041252136,
+      "rewards/accuracy_reward": 0.8384228348731995,
+      "rewards/format_reward": 1.0,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.38813018798828,
+      "epoch": 2.0,
+      "grad_norm": 1.3167695891772675,
+      "kl": 0.033203125,
+      "learning_rate": 9e-07,
+      "loss": 0.0013,
+      "reward": 1.8518694639205933,
+      "reward_std": 0.021665828302502632,
+      "rewards/accuracy_reward": 0.8518694639205933,
+      "rewards/format_reward": 1.0,
+      "step": 136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.4158706665039,
+      "epoch": 2.014705882352941,
+      "grad_norm": 1.1129043277785302,
+      "kl": 0.0296630859375,
+      "learning_rate": 8.992647058823529e-07,
+      "loss": 0.0012,
+      "reward": 1.8364787101745605,
+      "reward_std": 0.02524709887802601,
+      "rewards/accuracy_reward": 0.8364787697792053,
+      "rewards/format_reward": 1.0,
+      "step": 137
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.6189956665039,
+      "epoch": 2.0294117647058822,
+      "grad_norm": 11.77858401808986,
+      "kl": 0.0341796875,
+      "learning_rate": 8.985294117647059e-07,
+      "loss": 0.0014,
+      "reward": 1.8593671321868896,
+      "reward_std": 0.02559875324368477,
+      "rewards/accuracy_reward": 0.8605691194534302,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.89063262939453,
+      "epoch": 2.0441176470588234,
+      "grad_norm": 1.4515857339540483,
+      "kl": 0.035400390625,
+      "learning_rate": 8.977941176470588e-07,
+      "loss": 0.0014,
+      "reward": 1.8674319982528687,
+      "reward_std": 0.02322627604007721,
+      "rewards/accuracy_reward": 0.8674321174621582,
+      "rewards/format_reward": 1.0,
+      "step": 139
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.36058044433594,
+      "epoch": 2.0588235294117645,
+      "grad_norm": 1.7247871087577205,
+      "kl": 0.030517578125,
+      "learning_rate": 8.970588235294118e-07,
+      "loss": 0.0012,
+      "reward": 1.8335715532302856,
+      "reward_std": 0.028056330978870392,
+      "rewards/accuracy_reward": 0.8347733020782471,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.83894348144531,
+      "epoch": 2.073529411764706,
+      "grad_norm": 1.700568612592954,
+      "kl": 0.029296875,
+      "learning_rate": 8.963235294117648e-07,
+      "loss": 0.0012,
+      "reward": 1.841809868812561,
+      "reward_std": 0.02446519024670124,
+      "rewards/accuracy_reward": 0.8418098092079163,
+      "rewards/format_reward": 1.0,
+      "step": 141
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.85577392578125,
+      "epoch": 2.088235294117647,
+      "grad_norm": 1.9489963115328834,
+      "kl": 0.0296630859375,
+      "learning_rate": 8.955882352941176e-07,
+      "loss": 0.0012,
+      "reward": 1.8537096977233887,
+      "reward_std": 0.025433821603655815,
+      "rewards/accuracy_reward": 0.8537096381187439,
+      "rewards/format_reward": 1.0,
+      "step": 142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.17788696289062,
+      "epoch": 2.1029411764705883,
+      "grad_norm": 1.930319438615514,
+      "kl": 0.0303955078125,
+      "learning_rate": 8.948529411764706e-07,
+      "loss": 0.0012,
+      "reward": 1.8577303886413574,
+      "reward_std": 0.025013966485857964,
+      "rewards/accuracy_reward": 0.858932375907898,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 143
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.77404022216797,
+      "epoch": 2.1176470588235294,
+      "grad_norm": 2.741052616431375,
+      "kl": 0.031005859375,
+      "learning_rate": 8.941176470588236e-07,
+      "loss": 0.0012,
+      "reward": 1.8680880069732666,
+      "reward_std": 0.025277523323893547,
+      "rewards/accuracy_reward": 0.8692898750305176,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.88581848144531,
+      "epoch": 2.1323529411764706,
+      "grad_norm": 2.1856417966833543,
+      "kl": 0.030029296875,
+      "learning_rate": 8.933823529411765e-07,
+      "loss": 0.0012,
+      "reward": 1.851528286933899,
+      "reward_std": 0.024104848504066467,
+      "rewards/accuracy_reward": 0.8515281081199646,
+      "rewards/format_reward": 1.0,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.79808044433594,
+      "epoch": 2.1470588235294117,
+      "grad_norm": 1.7095764168354908,
+      "kl": 0.03076171875,
+      "learning_rate": 8.926470588235294e-07,
+      "loss": 0.0012,
+      "reward": 1.8471791744232178,
+      "reward_std": 0.028594160452485085,
+      "rewards/accuracy_reward": 0.8483811020851135,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.95673370361328,
+      "epoch": 2.161764705882353,
+      "grad_norm": 1.8204981844802233,
+      "kl": 0.03125,
+      "learning_rate": 8.919117647058824e-07,
+      "loss": 0.0013,
+      "reward": 1.8616198301315308,
+      "reward_std": 0.0273142047226429,
+      "rewards/accuracy_reward": 0.8628216981887817,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 147
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.73438262939453,
+      "epoch": 2.176470588235294,
+      "grad_norm": 2.6286179471239906,
+      "kl": 0.0299072265625,
+      "learning_rate": 8.911764705882353e-07,
+      "loss": 0.0012,
+      "reward": 1.864734172821045,
+      "reward_std": 0.027529988437891006,
+      "rewards/accuracy_reward": 0.8647341132164001,
+      "rewards/format_reward": 1.0,
+      "step": 148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.94471740722656,
+      "epoch": 2.1911764705882355,
+      "grad_norm": 1.2094663657381695,
+      "kl": 0.033447265625,
+      "learning_rate": 8.904411764705882e-07,
+      "loss": 0.0013,
+      "reward": 1.8421967029571533,
+      "reward_std": 0.030433619394898415,
+      "rewards/accuracy_reward": 0.8446005582809448,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 149
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.39904022216797,
+      "epoch": 2.2058823529411766,
+      "grad_norm": 1.9663042039131189,
+      "kl": 0.032470703125,
+      "learning_rate": 8.89705882352941e-07,
+      "loss": 0.0013,
+      "reward": 1.8600176572799683,
+      "reward_std": 0.024169858545064926,
+      "rewards/accuracy_reward": 0.860017716884613,
+      "rewards/format_reward": 1.0,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.9627456665039,
+      "epoch": 2.2205882352941178,
+      "grad_norm": 2.5776707309950115,
+      "kl": 0.02978515625,
+      "learning_rate": 8.88970588235294e-07,
+      "loss": 0.0012,
+      "reward": 1.871897578239441,
+      "reward_std": 0.023594854399561882,
+      "rewards/accuracy_reward": 0.8718973398208618,
+      "rewards/format_reward": 1.0,
+      "step": 151
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.98678588867188,
+      "epoch": 2.235294117647059,
+      "grad_norm": 1.5441369185409464,
+      "kl": 0.0306396484375,
+      "learning_rate": 8.88235294117647e-07,
+      "loss": 0.0012,
+      "reward": 1.8490618467330933,
+      "reward_std": 0.027410874143242836,
+      "rewards/accuracy_reward": 0.8502638339996338,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.43029022216797,
+      "epoch": 2.25,
+      "grad_norm": 1.8027917380235519,
+      "kl": 0.031494140625,
+      "learning_rate": 8.874999999999999e-07,
+      "loss": 0.0013,
+      "reward": 1.8541759252548218,
+      "reward_std": 0.024694515392184258,
+      "rewards/accuracy_reward": 0.8553777933120728,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 153
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.06491088867188,
+      "epoch": 2.264705882352941,
+      "grad_norm": 2.271542926940373,
+      "kl": 0.0308837890625,
+      "learning_rate": 8.867647058823528e-07,
+      "loss": 0.0012,
+      "reward": 1.8564496040344238,
+      "reward_std": 0.023326044902205467,
+      "rewards/accuracy_reward": 0.856449544429779,
+      "rewards/format_reward": 1.0,
+      "step": 154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.58053588867188,
+      "epoch": 2.2794117647058822,
+      "grad_norm": 1.5375482020791835,
+      "kl": 0.032470703125,
+      "learning_rate": 8.860294117647058e-07,
+      "loss": 0.0013,
+      "reward": 1.8531345129013062,
+      "reward_std": 0.025648273527622223,
+      "rewards/accuracy_reward": 0.8543363809585571,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.21514892578125,
+      "epoch": 2.2941176470588234,
+      "grad_norm": 1.6774011174439099,
+      "kl": 0.041259765625,
+      "learning_rate": 8.852941176470587e-07,
+      "loss": 0.0017,
+      "reward": 1.8623310327529907,
+      "reward_std": 0.024891231209039688,
+      "rewards/accuracy_reward": 0.862330973148346,
+      "rewards/format_reward": 1.0,
+      "step": 156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.63702392578125,
+      "epoch": 2.3088235294117645,
+      "grad_norm": 1.5805094517168279,
+      "kl": 0.03515625,
+      "learning_rate": 8.845588235294117e-07,
+      "loss": 0.0014,
+      "reward": 1.8760714530944824,
+      "reward_std": 0.028571614995598793,
+      "rewards/accuracy_reward": 0.8784750699996948,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 157
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.48798370361328,
+      "epoch": 2.323529411764706,
+      "grad_norm": 2.903562674226702,
+      "kl": 0.03466796875,
+      "learning_rate": 8.838235294117647e-07,
+      "loss": 0.0014,
+      "reward": 1.8514492511749268,
+      "reward_std": 0.023621495813131332,
+      "rewards/accuracy_reward": 0.8514493107795715,
+      "rewards/format_reward": 1.0,
+      "step": 158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.39543914794922,
+      "epoch": 2.338235294117647,
+      "grad_norm": 2.257173366564063,
+      "kl": 0.035888671875,
+      "learning_rate": 8.830882352941175e-07,
+      "loss": 0.0014,
+      "reward": 1.8729958534240723,
+      "reward_std": 0.026356099173426628,
+      "rewards/accuracy_reward": 0.8729958534240723,
+      "rewards/format_reward": 1.0,
+      "step": 159
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.02764892578125,
+      "epoch": 2.3529411764705883,
+      "grad_norm": 1.6931345428419227,
+      "kl": 0.047607421875,
+      "learning_rate": 8.823529411764705e-07,
+      "loss": 0.0019,
+      "reward": 1.871155023574829,
+      "reward_std": 0.02590741030871868,
+      "rewards/accuracy_reward": 0.8723568916320801,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.74519348144531,
+      "epoch": 2.3676470588235294,
+      "grad_norm": 2.3920729679008232,
+      "kl": 0.03955078125,
+      "learning_rate": 8.816176470588235e-07,
+      "loss": 0.0016,
+      "reward": 1.870141863822937,
+      "reward_std": 0.025313733145594597,
+      "rewards/accuracy_reward": 0.870141863822937,
+      "rewards/format_reward": 1.0,
+      "step": 161
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.9831771850586,
+      "epoch": 2.3823529411764706,
+      "grad_norm": 1.7968109427756567,
+      "kl": 0.03955078125,
+      "learning_rate": 8.808823529411764e-07,
+      "loss": 0.0016,
+      "reward": 1.8542630672454834,
+      "reward_std": 0.025294849649071693,
+      "rewards/accuracy_reward": 0.8542630672454834,
+      "rewards/format_reward": 1.0,
+      "step": 162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.81731414794922,
+      "epoch": 2.3970588235294117,
+      "grad_norm": 3.2616152503315536,
+      "kl": 0.040771484375,
+      "learning_rate": 8.801470588235293e-07,
+      "loss": 0.0016,
+      "reward": 1.8570067882537842,
+      "reward_std": 0.02526533231139183,
+      "rewards/accuracy_reward": 0.8570066094398499,
+      "rewards/format_reward": 1.0,
+      "step": 163
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.97716522216797,
+      "epoch": 2.411764705882353,
+      "grad_norm": 1.8975512236370329,
+      "kl": 0.03955078125,
+      "learning_rate": 8.794117647058823e-07,
+      "loss": 0.0016,
+      "reward": 1.8464734554290771,
+      "reward_std": 0.02490406110882759,
+      "rewards/accuracy_reward": 0.8464734554290771,
+      "rewards/format_reward": 1.0,
+      "step": 164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.0769271850586,
+      "epoch": 2.426470588235294,
+      "grad_norm": 3.1230913633145096,
+      "kl": 0.04931640625,
+      "learning_rate": 8.786764705882352e-07,
+      "loss": 0.002,
+      "reward": 1.8685623407363892,
+      "reward_std": 0.024073276668787003,
+      "rewards/accuracy_reward": 0.8685624599456787,
+      "rewards/format_reward": 1.0,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.92668914794922,
+      "epoch": 2.4411764705882355,
+      "grad_norm": 2.501654602692275,
+      "kl": 0.04296875,
+      "learning_rate": 8.779411764705882e-07,
+      "loss": 0.0017,
+      "reward": 1.8564651012420654,
+      "reward_std": 0.024265745654702187,
+      "rewards/accuracy_reward": 0.8564649820327759,
+      "rewards/format_reward": 1.0,
+      "step": 166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.86538696289062,
+      "epoch": 2.4558823529411766,
+      "grad_norm": 6.016419703798391,
+      "kl": 0.0439453125,
+      "learning_rate": 8.772058823529412e-07,
+      "loss": 0.0018,
+      "reward": 1.8798558712005615,
+      "reward_std": 0.021158261224627495,
+      "rewards/accuracy_reward": 0.8798558712005615,
+      "rewards/format_reward": 1.0,
+      "step": 167
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.84014892578125,
+      "epoch": 2.4705882352941178,
+      "grad_norm": 1.9371757598853314,
+      "kl": 0.04052734375,
+      "learning_rate": 8.76470588235294e-07,
+      "loss": 0.0016,
+      "reward": 1.882270336151123,
+      "reward_std": 0.023024968802928925,
+      "rewards/accuracy_reward": 0.882270336151123,
+      "rewards/format_reward": 1.0,
+      "step": 168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.30529022216797,
+      "epoch": 2.485294117647059,
+      "grad_norm": 1.4411287692419261,
+      "kl": 0.036376953125,
+      "learning_rate": 8.75735294117647e-07,
+      "loss": 0.0015,
+      "reward": 1.862653374671936,
+      "reward_std": 0.021055346354842186,
+      "rewards/accuracy_reward": 0.8626533150672913,
+      "rewards/format_reward": 1.0,
+      "step": 169
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.5877456665039,
+      "epoch": 2.5,
+      "grad_norm": 1.4740075609659171,
+      "kl": 0.03857421875,
+      "learning_rate": 8.75e-07,
+      "loss": 0.0015,
+      "reward": 1.8584967851638794,
+      "reward_std": 0.02370167151093483,
+      "rewards/accuracy_reward": 0.8584967255592346,
+      "rewards/format_reward": 1.0,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.28245544433594,
+      "epoch": 2.514705882352941,
+      "grad_norm": 1.3370741346003325,
+      "kl": 0.03662109375,
+      "learning_rate": 8.742647058823529e-07,
+      "loss": 0.0015,
+      "reward": 1.8700354099273682,
+      "reward_std": 0.023715250194072723,
+      "rewards/accuracy_reward": 0.8700354099273682,
+      "rewards/format_reward": 1.0,
+      "step": 171
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.4050521850586,
+      "epoch": 2.5294117647058822,
+      "grad_norm": 2.6769462130534345,
+      "kl": 0.03369140625,
+      "learning_rate": 8.735294117647058e-07,
+      "loss": 0.0013,
+      "reward": 1.8732657432556152,
+      "reward_std": 0.01899472437798977,
+      "rewards/accuracy_reward": 0.8732655644416809,
+      "rewards/format_reward": 1.0,
+      "step": 172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.87741088867188,
+      "epoch": 2.5441176470588234,
+      "grad_norm": 2.5769454874734152,
+      "kl": 0.0322265625,
+      "learning_rate": 8.727941176470588e-07,
+      "loss": 0.0013,
+      "reward": 1.8616483211517334,
+      "reward_std": 0.02438032068312168,
+      "rewards/accuracy_reward": 0.8616481423377991,
+      "rewards/format_reward": 1.0,
+      "step": 173
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.12259674072266,
+      "epoch": 2.5588235294117645,
+      "grad_norm": 1.8562667369690105,
+      "kl": 0.033935546875,
+      "learning_rate": 8.720588235294117e-07,
+      "loss": 0.0013,
+      "reward": 1.855645775794983,
+      "reward_std": 0.02895743027329445,
+      "rewards/accuracy_reward": 0.8580495715141296,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.21154022216797,
+      "epoch": 2.5735294117647056,
+      "grad_norm": 2.558018085056357,
+      "kl": 0.034423828125,
+      "learning_rate": 8.713235294117647e-07,
+      "loss": 0.0014,
+      "reward": 1.884694218635559,
+      "reward_std": 0.026144152507185936,
+      "rewards/accuracy_reward": 0.8858961462974548,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.8894271850586,
+      "epoch": 2.588235294117647,
+      "grad_norm": 4.093759859461043,
+      "kl": 0.0322265625,
+      "learning_rate": 8.705882352941177e-07,
+      "loss": 0.0013,
+      "reward": 1.859119176864624,
+      "reward_std": 0.023388409987092018,
+      "rewards/accuracy_reward": 0.859119176864624,
+      "rewards/format_reward": 1.0,
+      "step": 176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.76923370361328,
+      "epoch": 2.6029411764705883,
+      "grad_norm": 2.565372217350874,
+      "kl": 0.039794921875,
+      "learning_rate": 8.698529411764705e-07,
+      "loss": 0.0016,
+      "reward": 1.8814976215362549,
+      "reward_std": 0.024835962802171707,
+      "rewards/accuracy_reward": 0.8826994895935059,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 177
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.4158706665039,
+      "epoch": 2.6176470588235294,
+      "grad_norm": 1.546586759316415,
+      "kl": 0.03662109375,
+      "learning_rate": 8.691176470588235e-07,
+      "loss": 0.0015,
+      "reward": 1.8796207904815674,
+      "reward_std": 0.022139329463243484,
+      "rewards/accuracy_reward": 0.8796206712722778,
+      "rewards/format_reward": 1.0,
+      "step": 178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.92668914794922,
+      "epoch": 2.6323529411764706,
+      "grad_norm": 1.5303977681102032,
+      "kl": 0.03369140625,
+      "learning_rate": 8.683823529411765e-07,
+      "loss": 0.0014,
+      "reward": 1.8701624870300293,
+      "reward_std": 0.025447282940149307,
+      "rewards/accuracy_reward": 0.8701622486114502,
+      "rewards/format_reward": 1.0,
+      "step": 179
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.36659240722656,
+      "epoch": 2.6470588235294117,
+      "grad_norm": 1.518878529368709,
+      "kl": 0.033203125,
+      "learning_rate": 8.676470588235294e-07,
+      "loss": 0.0013,
+      "reward": 1.8654332160949707,
+      "reward_std": 0.026392469182610512,
+      "rewards/accuracy_reward": 0.8654330372810364,
+      "rewards/format_reward": 1.0,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.12620544433594,
+      "epoch": 2.661764705882353,
+      "grad_norm": 1.403191410442849,
+      "kl": 0.03173828125,
+      "learning_rate": 8.669117647058823e-07,
+      "loss": 0.0013,
+      "reward": 1.86957585811615,
+      "reward_std": 0.025720389559864998,
+      "rewards/accuracy_reward": 0.8695757985115051,
+      "rewards/format_reward": 1.0,
+      "step": 181
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.7283706665039,
+      "epoch": 2.6764705882352944,
+      "grad_norm": 2.022045986077511,
+      "kl": 0.031494140625,
+      "learning_rate": 8.661764705882353e-07,
+      "loss": 0.0013,
+      "reward": 1.8482249975204468,
+      "reward_std": 0.03184087574481964,
+      "rewards/accuracy_reward": 0.8494267463684082,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.31491088867188,
+      "epoch": 2.6911764705882355,
+      "grad_norm": 1.4526871175791842,
+      "kl": 0.043701171875,
+      "learning_rate": 8.654411764705882e-07,
+      "loss": 0.0017,
+      "reward": 1.8396512269973755,
+      "reward_std": 0.02649850584566593,
+      "rewards/accuracy_reward": 0.8396511673927307,
+      "rewards/format_reward": 1.0,
+      "step": 183
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.32933044433594,
+      "epoch": 2.7058823529411766,
+      "grad_norm": 1.962569471145186,
+      "kl": 0.034423828125,
+      "learning_rate": 8.647058823529412e-07,
+      "loss": 0.0014,
+      "reward": 1.864058017730713,
+      "reward_std": 0.026860656216740608,
+      "rewards/accuracy_reward": 0.8640578389167786,
+      "rewards/format_reward": 1.0,
+      "step": 184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.7331771850586,
+      "epoch": 2.7205882352941178,
+      "grad_norm": 2.2484467422992944,
+      "kl": 0.03369140625,
+      "learning_rate": 8.639705882352941e-07,
+      "loss": 0.0013,
+      "reward": 1.8754035234451294,
+      "reward_std": 0.024205412715673447,
+      "rewards/accuracy_reward": 0.8754035234451294,
+      "rewards/format_reward": 1.0,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.37860870361328,
+      "epoch": 2.735294117647059,
+      "grad_norm": 1.997975820505161,
+      "kl": 0.033935546875,
+      "learning_rate": 8.63235294117647e-07,
+      "loss": 0.0014,
+      "reward": 1.860775351524353,
+      "reward_std": 0.02340296283364296,
+      "rewards/accuracy_reward": 0.8607752919197083,
+      "rewards/format_reward": 1.0,
+      "step": 186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.65866088867188,
+      "epoch": 2.75,
+      "grad_norm": 1.7044143261510771,
+      "kl": 0.0341796875,
+      "learning_rate": 8.625e-07,
+      "loss": 0.0014,
+      "reward": 1.8710038661956787,
+      "reward_std": 0.026455307379364967,
+      "rewards/accuracy_reward": 0.8722056746482849,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 187
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.46514892578125,
+      "epoch": 2.764705882352941,
+      "grad_norm": 1.8000725865597647,
+      "kl": 0.038330078125,
+      "learning_rate": 8.61764705882353e-07,
+      "loss": 0.0015,
+      "reward": 1.8618617057800293,
+      "reward_std": 0.03181543946266174,
+      "rewards/accuracy_reward": 0.8642653822898865,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.53125762939453,
+      "epoch": 2.7794117647058822,
+      "grad_norm": 2.339206486710046,
+      "kl": 0.0322265625,
+      "learning_rate": 8.610294117647059e-07,
+      "loss": 0.0013,
+      "reward": 1.875985860824585,
+      "reward_std": 0.02267822064459324,
+      "rewards/accuracy_reward": 0.875985860824585,
+      "rewards/format_reward": 1.0,
+      "step": 189
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.21634674072266,
+      "epoch": 2.7941176470588234,
+      "grad_norm": 1.4965339815672982,
+      "kl": 0.033447265625,
+      "learning_rate": 8.602941176470588e-07,
+      "loss": 0.0013,
+      "reward": 1.8847665786743164,
+      "reward_std": 0.023552460595965385,
+      "rewards/accuracy_reward": 0.8847665190696716,
+      "rewards/format_reward": 1.0,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.6238021850586,
+      "epoch": 2.8088235294117645,
+      "grad_norm": 2.3107985557542774,
+      "kl": 0.03369140625,
+      "learning_rate": 8.595588235294118e-07,
+      "loss": 0.0014,
+      "reward": 1.8373372554779053,
+      "reward_std": 0.02436283603310585,
+      "rewards/accuracy_reward": 0.8373372554779053,
+      "rewards/format_reward": 1.0,
+      "step": 191
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.30409240722656,
+      "epoch": 2.8235294117647056,
+      "grad_norm": 1.92475811318894,
+      "kl": 0.0361328125,
+      "learning_rate": 8.588235294117646e-07,
+      "loss": 0.0014,
+      "reward": 1.8702090978622437,
+      "reward_std": 0.026847295463085175,
+      "rewards/accuracy_reward": 0.8714108467102051,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.48197174072266,
+      "epoch": 2.838235294117647,
+      "grad_norm": 2.0449651951501866,
+      "kl": 0.034423828125,
+      "learning_rate": 8.580882352941176e-07,
+      "loss": 0.0014,
+      "reward": 1.843615174293518,
+      "reward_std": 0.02285837009549141,
+      "rewards/accuracy_reward": 0.8436151742935181,
+      "rewards/format_reward": 1.0,
+      "step": 193
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.01202392578125,
+      "epoch": 2.8529411764705883,
+      "grad_norm": 1.2736084553332587,
+      "kl": 0.036865234375,
+      "learning_rate": 8.573529411764705e-07,
+      "loss": 0.0015,
+      "reward": 1.8695170879364014,
+      "reward_std": 0.022367505356669426,
+      "rewards/accuracy_reward": 0.8695171475410461,
+      "rewards/format_reward": 1.0,
+      "step": 194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.91827392578125,
+      "epoch": 2.8676470588235294,
+      "grad_norm": 1.9579889152211967,
+      "kl": 0.047119140625,
+      "learning_rate": 8.566176470588234e-07,
+      "loss": 0.0019,
+      "reward": 1.8573535680770874,
+      "reward_std": 0.026318097487092018,
+      "rewards/accuracy_reward": 0.8585554361343384,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 195
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.89543914794922,
+      "epoch": 2.8823529411764706,
+      "grad_norm": 1.6397435637270048,
+      "kl": 0.03173828125,
+      "learning_rate": 8.558823529411764e-07,
+      "loss": 0.0013,
+      "reward": 1.8664311170578003,
+      "reward_std": 0.02275273948907852,
+      "rewards/accuracy_reward": 0.8664309978485107,
+      "rewards/format_reward": 1.0,
+      "step": 196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.19351196289062,
+      "epoch": 2.8970588235294117,
+      "grad_norm": 1.3453321422064775,
+      "kl": 0.03271484375,
+      "learning_rate": 8.551470588235294e-07,
+      "loss": 0.0013,
+      "reward": 1.8688963651657104,
+      "reward_std": 0.02163490653038025,
+      "rewards/accuracy_reward": 0.8688963651657104,
+      "rewards/format_reward": 1.0,
+      "step": 197
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.34495544433594,
+      "epoch": 2.911764705882353,
+      "grad_norm": 1.6994019668289626,
+      "kl": 0.034423828125,
+      "learning_rate": 8.544117647058822e-07,
+      "loss": 0.0014,
+      "reward": 1.8594874143600464,
+      "reward_std": 0.025675183162093163,
+      "rewards/accuracy_reward": 0.8606892824172974,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.97956848144531,
+      "epoch": 2.9264705882352944,
+      "grad_norm": 1.443393412750695,
+      "kl": 0.033935546875,
+      "learning_rate": 8.536764705882352e-07,
+      "loss": 0.0014,
+      "reward": 1.8734104633331299,
+      "reward_std": 0.02485453523695469,
+      "rewards/accuracy_reward": 0.8734104633331299,
+      "rewards/format_reward": 1.0,
+      "step": 199
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.54447174072266,
+      "epoch": 2.9411764705882355,
+      "grad_norm": 1.3185192214107238,
+      "kl": 0.0322265625,
+      "learning_rate": 8.529411764705882e-07,
+      "loss": 0.0013,
+      "reward": 1.8638544082641602,
+      "reward_std": 0.024562494829297066,
+      "rewards/accuracy_reward": 0.8650562167167664,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.21995544433594,
+      "epoch": 2.9558823529411766,
+      "grad_norm": 1.9943222079885201,
+      "kl": 0.032470703125,
+      "learning_rate": 8.522058823529411e-07,
+      "loss": 0.0013,
+      "reward": 1.8713494539260864,
+      "reward_std": 0.027820177376270294,
+      "rewards/accuracy_reward": 0.8725512623786926,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 201
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.25000762939453,
+      "epoch": 2.9705882352941178,
+      "grad_norm": 1.3647685703194712,
+      "kl": 0.033203125,
+      "learning_rate": 8.514705882352941e-07,
+      "loss": 0.0013,
+      "reward": 1.8598443269729614,
+      "reward_std": 0.025986719876527786,
+      "rewards/accuracy_reward": 0.8598442673683167,
+      "rewards/format_reward": 1.0,
+      "step": 202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.90625762939453,
+      "epoch": 2.985294117647059,
+      "grad_norm": 1.6126487460912005,
+      "kl": 0.037841796875,
+      "learning_rate": 8.50735294117647e-07,
+      "loss": 0.0015,
+      "reward": 1.8783414363861084,
+      "reward_std": 0.023664260283112526,
+      "rewards/accuracy_reward": 0.8783413171768188,
+      "rewards/format_reward": 1.0,
+      "step": 203
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.15220642089844,
+      "epoch": 3.0,
+      "grad_norm": 1.780161633176048,
+      "kl": 0.034423828125,
+      "learning_rate": 8.499999999999999e-07,
+      "loss": 0.0014,
+      "reward": 1.8632652759552002,
+      "reward_std": 0.02758597768843174,
+      "rewards/accuracy_reward": 0.8647872805595398,
+      "rewards/format_reward": 0.9984779357910156,
+      "step": 204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.47956848144531,
+      "epoch": 3.014705882352941,
+      "grad_norm": 1.9805220919990834,
+      "kl": 0.03271484375,
+      "learning_rate": 8.492647058823529e-07,
+      "loss": 0.0013,
+      "reward": 1.8707077503204346,
+      "reward_std": 0.023023683577775955,
+      "rewards/accuracy_reward": 0.8707077503204346,
+      "rewards/format_reward": 1.0,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.53366088867188,
+      "epoch": 3.0294117647058822,
+      "grad_norm": 1.759835112928017,
+      "kl": 0.03271484375,
+      "learning_rate": 8.485294117647059e-07,
+      "loss": 0.0013,
+      "reward": 1.8758447170257568,
+      "reward_std": 0.02334214560687542,
+      "rewards/accuracy_reward": 0.8758447170257568,
+      "rewards/format_reward": 1.0,
+      "step": 206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.1346206665039,
+      "epoch": 3.0441176470588234,
+      "grad_norm": 1.9299225316519943,
+      "kl": 0.032958984375,
+      "learning_rate": 8.477941176470587e-07,
+      "loss": 0.0013,
+      "reward": 1.8904192447662354,
+      "reward_std": 0.021329201757907867,
+      "rewards/accuracy_reward": 0.8904191255569458,
+      "rewards/format_reward": 1.0,
+      "step": 207
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.8221206665039,
+      "epoch": 3.0588235294117645,
+      "grad_norm": 1.5162706953943,
+      "kl": 0.03759765625,
+      "learning_rate": 8.470588235294117e-07,
+      "loss": 0.0015,
+      "reward": 1.8666112422943115,
+      "reward_std": 0.02535596303641796,
+      "rewards/accuracy_reward": 0.8666113615036011,
+      "rewards/format_reward": 1.0,
+      "step": 208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.12019348144531,
+      "epoch": 3.073529411764706,
+      "grad_norm": 3.4426286843801193,
+      "kl": 0.08642578125,
+      "learning_rate": 8.463235294117647e-07,
+      "loss": 0.0035,
+      "reward": 1.8811273574829102,
+      "reward_std": 0.022378716617822647,
+      "rewards/accuracy_reward": 0.8811271786689758,
+      "rewards/format_reward": 1.0,
+      "step": 209
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.17909240722656,
+      "epoch": 3.088235294117647,
+      "grad_norm": 1.7571727846632814,
+      "kl": 0.0380859375,
+      "learning_rate": 8.455882352941176e-07,
+      "loss": 0.0015,
+      "reward": 1.8708646297454834,
+      "reward_std": 0.027770236134529114,
+      "rewards/accuracy_reward": 0.8732684254646301,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.16947174072266,
+      "epoch": 3.1029411764705883,
+      "grad_norm": 1.336840943007517,
+      "kl": 0.041259765625,
+      "learning_rate": 8.448529411764706e-07,
+      "loss": 0.0016,
+      "reward": 1.8811522722244263,
+      "reward_std": 0.02304122783243656,
+      "rewards/accuracy_reward": 0.8811522722244263,
+      "rewards/format_reward": 1.0,
+      "step": 211
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.03846740722656,
+      "epoch": 3.1176470588235294,
+      "grad_norm": 1.8533547410757114,
+      "kl": 0.0400390625,
+      "learning_rate": 8.441176470588235e-07,
+      "loss": 0.0016,
+      "reward": 1.8757656812667847,
+      "reward_std": 0.027794932946562767,
+      "rewards/accuracy_reward": 0.8769674897193909,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.97356414794922,
+      "epoch": 3.1323529411764706,
+      "grad_norm": 1.2600737914736126,
+      "kl": 0.0380859375,
+      "learning_rate": 8.433823529411764e-07,
+      "loss": 0.0015,
+      "reward": 1.8561041355133057,
+      "reward_std": 0.031202280893921852,
+      "rewards/accuracy_reward": 0.8585077524185181,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 213
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.35938262939453,
+      "epoch": 3.1470588235294117,
+      "grad_norm": 1.7979464745878548,
+      "kl": 0.043701171875,
+      "learning_rate": 8.426470588235294e-07,
+      "loss": 0.0017,
+      "reward": 1.8608158826828003,
+      "reward_std": 0.026735760271549225,
+      "rewards/accuracy_reward": 0.8608157634735107,
+      "rewards/format_reward": 1.0,
+      "step": 214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.00241088867188,
+      "epoch": 3.161764705882353,
+      "grad_norm": 5.398535880427292,
+      "kl": 0.05322265625,
+      "learning_rate": 8.419117647058824e-07,
+      "loss": 0.0021,
+      "reward": 1.8839178085327148,
+      "reward_std": 0.02646607905626297,
+      "rewards/accuracy_reward": 0.8851196765899658,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.87620544433594,
+      "epoch": 3.176470588235294,
+      "grad_norm": 1.6689071059712295,
+      "kl": 0.041748046875,
+      "learning_rate": 8.411764705882352e-07,
+      "loss": 0.0017,
+      "reward": 1.8544951677322388,
+      "reward_std": 0.02808697521686554,
+      "rewards/accuracy_reward": 0.854495108127594,
+      "rewards/format_reward": 1.0,
+      "step": 216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.43029022216797,
+      "epoch": 3.1911764705882355,
+      "grad_norm": 1.6258182876226766,
+      "kl": 0.04296875,
+      "learning_rate": 8.404411764705882e-07,
+      "loss": 0.0017,
+      "reward": 1.8663568496704102,
+      "reward_std": 0.029328130185604095,
+      "rewards/accuracy_reward": 0.8675586581230164,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 217
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.61418914794922,
+      "epoch": 3.2058823529411766,
+      "grad_norm": 1.4035611828729841,
+      "kl": 0.040771484375,
+      "learning_rate": 8.397058823529412e-07,
+      "loss": 0.0016,
+      "reward": 1.8575928211212158,
+      "reward_std": 0.02494001016020775,
+      "rewards/accuracy_reward": 0.8575928211212158,
+      "rewards/format_reward": 1.0,
+      "step": 218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.87620544433594,
+      "epoch": 3.2205882352941178,
+      "grad_norm": 1.5589742726291336,
+      "kl": 0.0419921875,
+      "learning_rate": 8.389705882352941e-07,
+      "loss": 0.0017,
+      "reward": 1.8885266780853271,
+      "reward_std": 0.023614982143044472,
+      "rewards/accuracy_reward": 0.8897286653518677,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 219
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.81851196289062,
+      "epoch": 3.235294117647059,
+      "grad_norm": 1.385282340462598,
+      "kl": 0.041259765625,
+      "learning_rate": 8.38235294117647e-07,
+      "loss": 0.0017,
+      "reward": 1.855526328086853,
+      "reward_std": 0.030946535989642143,
+      "rewards/accuracy_reward": 0.8567281365394592,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.7439956665039,
+      "epoch": 3.25,
+      "grad_norm": 2.962059838231098,
+      "kl": 0.044189453125,
+      "learning_rate": 8.375e-07,
+      "loss": 0.0018,
+      "reward": 1.855432152748108,
+      "reward_std": 0.02485704980790615,
+      "rewards/accuracy_reward": 0.8554319739341736,
+      "rewards/format_reward": 1.0,
+      "step": 221
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.8533706665039,
+      "epoch": 3.264705882352941,
+      "grad_norm": 1.3569137376665106,
+      "kl": 0.04296875,
+      "learning_rate": 8.367647058823529e-07,
+      "loss": 0.0017,
+      "reward": 1.8471739292144775,
+      "reward_std": 0.028255922719836235,
+      "rewards/accuracy_reward": 0.8483757972717285,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.22956848144531,
+      "epoch": 3.2794117647058822,
+      "grad_norm": 1.5185134752025757,
+      "kl": 0.04345703125,
+      "learning_rate": 8.360294117647059e-07,
+      "loss": 0.0017,
+      "reward": 1.8899589776992798,
+      "reward_std": 0.02284637838602066,
+      "rewards/accuracy_reward": 0.889958918094635,
+      "rewards/format_reward": 1.0,
+      "step": 223
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.96394348144531,
+      "epoch": 3.2941176470588234,
+      "grad_norm": 2.4678802175191117,
+      "kl": 0.0458984375,
+      "learning_rate": 8.352941176470589e-07,
+      "loss": 0.0018,
+      "reward": 1.8883994817733765,
+      "reward_std": 0.021819431334733963,
+      "rewards/accuracy_reward": 0.8883993625640869,
+      "rewards/format_reward": 1.0,
+      "step": 224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.47356414794922,
+      "epoch": 3.3088235294117645,
+      "grad_norm": 1.7983286178946878,
+      "kl": 0.04736328125,
+      "learning_rate": 8.345588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.8640559911727905,
+      "reward_std": 0.023397305980324745,
+      "rewards/accuracy_reward": 0.8640559315681458,
+      "rewards/format_reward": 1.0,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.4675521850586,
+      "epoch": 3.323529411764706,
+      "grad_norm": 1.8613936498025156,
+      "kl": 0.04736328125,
+      "learning_rate": 8.338235294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.8887497186660767,
+      "reward_std": 0.022856805473566055,
+      "rewards/accuracy_reward": 0.8887495994567871,
+      "rewards/format_reward": 1.0,
+      "step": 226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.93029022216797,
+      "epoch": 3.338235294117647,
+      "grad_norm": 1.5411506435438795,
+      "kl": 0.045654296875,
+      "learning_rate": 8.330882352941177e-07,
+      "loss": 0.0018,
+      "reward": 1.8804612159729004,
+      "reward_std": 0.02065611444413662,
+      "rewards/accuracy_reward": 0.8804610371589661,
+      "rewards/format_reward": 1.0,
+      "step": 227
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.00601196289062,
+      "epoch": 3.3529411764705883,
+      "grad_norm": 2.851058373606194,
+      "kl": 0.048583984375,
+      "learning_rate": 8.323529411764706e-07,
+      "loss": 0.0019,
+      "reward": 1.8770747184753418,
+      "reward_std": 0.0234384723007679,
+      "rewards/accuracy_reward": 0.8770747184753418,
+      "rewards/format_reward": 1.0,
+      "step": 228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.92909240722656,
+      "epoch": 3.3676470588235294,
+      "grad_norm": 1.437225684861633,
+      "kl": 0.04345703125,
+      "learning_rate": 8.316176470588235e-07,
+      "loss": 0.0017,
+      "reward": 1.8713281154632568,
+      "reward_std": 0.02329498529434204,
+      "rewards/accuracy_reward": 0.8713279962539673,
+      "rewards/format_reward": 1.0,
+      "step": 229
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.06610870361328,
+      "epoch": 3.3823529411764706,
+      "grad_norm": 1.9896299970328106,
+      "kl": 0.04296875,
+      "learning_rate": 8.308823529411765e-07,
+      "loss": 0.0017,
+      "reward": 1.878764271736145,
+      "reward_std": 0.025371139869093895,
+      "rewards/accuracy_reward": 0.8787642121315002,
+      "rewards/format_reward": 1.0,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.38341522216797,
+      "epoch": 3.3970588235294117,
+      "grad_norm": 2.234055405089306,
+      "kl": 0.04345703125,
+      "learning_rate": 8.301470588235294e-07,
+      "loss": 0.0017,
+      "reward": 1.898535132408142,
+      "reward_std": 0.02297838404774666,
+      "rewards/accuracy_reward": 0.8985350131988525,
+      "rewards/format_reward": 1.0,
+      "step": 231
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.45072174072266,
+      "epoch": 3.411764705882353,
+      "grad_norm": 1.776208061156291,
+      "kl": 0.04052734375,
+      "learning_rate": 8.294117647058824e-07,
+      "loss": 0.0016,
+      "reward": 1.873674750328064,
+      "reward_std": 0.022656356915831566,
+      "rewards/accuracy_reward": 0.8736746907234192,
+      "rewards/format_reward": 1.0,
+      "step": 232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.99038696289062,
+      "epoch": 3.426470588235294,
+      "grad_norm": 1.5187065914939428,
+      "kl": 0.0390625,
+      "learning_rate": 8.286764705882354e-07,
+      "loss": 0.0016,
+      "reward": 1.8770731687545776,
+      "reward_std": 0.02269612066447735,
+      "rewards/accuracy_reward": 0.8770731091499329,
+      "rewards/format_reward": 1.0,
+      "step": 233
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.18389892578125,
+      "epoch": 3.4411764705882355,
+      "grad_norm": 1.6681947750734276,
+      "kl": 0.040771484375,
+      "learning_rate": 8.279411764705881e-07,
+      "loss": 0.0016,
+      "reward": 1.8845463991165161,
+      "reward_std": 0.02028636820614338,
+      "rewards/accuracy_reward": 0.8845462203025818,
+      "rewards/format_reward": 1.0,
+      "step": 234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.5613021850586,
+      "epoch": 3.4558823529411766,
+      "grad_norm": 1.4760305455453029,
+      "kl": 0.03759765625,
+      "learning_rate": 8.272058823529411e-07,
+      "loss": 0.0015,
+      "reward": 1.890356183052063,
+      "reward_std": 0.02070949226617813,
+      "rewards/accuracy_reward": 0.8903560042381287,
+      "rewards/format_reward": 1.0,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.21394348144531,
+      "epoch": 3.4705882352941178,
+      "grad_norm": 16.169977810311337,
+      "kl": 0.03759765625,
+      "learning_rate": 8.264705882352941e-07,
+      "loss": 0.0015,
+      "reward": 1.8766230344772339,
+      "reward_std": 0.02773961052298546,
+      "rewards/accuracy_reward": 0.8778248429298401,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.24759674072266,
+      "epoch": 3.485294117647059,
+      "grad_norm": 6.784504465196233,
+      "kl": 0.046630859375,
+      "learning_rate": 8.25735294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.8772554397583008,
+      "reward_std": 0.022827137261629105,
+      "rewards/accuracy_reward": 0.8772553205490112,
+      "rewards/format_reward": 1.0,
+      "step": 237
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.28485870361328,
+      "epoch": 3.5,
+      "grad_norm": 1.5048421325605899,
+      "kl": 0.040283203125,
+      "learning_rate": 8.249999999999999e-07,
+      "loss": 0.0016,
+      "reward": 1.8868188858032227,
+      "reward_std": 0.022883420810103416,
+      "rewards/accuracy_reward": 0.8868188261985779,
+      "rewards/format_reward": 1.0,
+      "step": 238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.67428588867188,
+      "epoch": 3.514705882352941,
+      "grad_norm": 1.9797082947474498,
+      "kl": 0.041259765625,
+      "learning_rate": 8.242647058823528e-07,
+      "loss": 0.0017,
+      "reward": 1.8824166059494019,
+      "reward_std": 0.025011856108903885,
+      "rewards/accuracy_reward": 0.8824164867401123,
+      "rewards/format_reward": 1.0,
+      "step": 239
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.68991088867188,
+      "epoch": 3.5294117647058822,
+      "grad_norm": 1.3808876653722937,
+      "kl": 0.038818359375,
+      "learning_rate": 8.235294117647058e-07,
+      "loss": 0.0016,
+      "reward": 1.8804829120635986,
+      "reward_std": 0.019605407491326332,
+      "rewards/accuracy_reward": 0.8804829716682434,
+      "rewards/format_reward": 1.0,
+      "step": 240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.4675521850586,
+      "epoch": 3.5441176470588234,
+      "grad_norm": 2.2656927922991352,
+      "kl": 0.03955078125,
+      "learning_rate": 8.227941176470588e-07,
+      "loss": 0.0016,
+      "reward": 1.8745205402374268,
+      "reward_std": 0.019595161080360413,
+      "rewards/accuracy_reward": 0.8745204210281372,
+      "rewards/format_reward": 1.0,
+      "step": 241
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.08293914794922,
+      "epoch": 3.5588235294117645,
+      "grad_norm": 2.2969538457560796,
+      "kl": 0.040771484375,
+      "learning_rate": 8.220588235294116e-07,
+      "loss": 0.0016,
+      "reward": 1.8878697156906128,
+      "reward_std": 0.02017536200582981,
+      "rewards/accuracy_reward": 0.8878695964813232,
+      "rewards/format_reward": 1.0,
+      "step": 242
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.17788696289062,
+      "epoch": 3.5735294117647056,
+      "grad_norm": 1.3250386411482078,
+      "kl": 0.036865234375,
+      "learning_rate": 8.213235294117646e-07,
+      "loss": 0.0015,
+      "reward": 1.882023572921753,
+      "reward_std": 0.019443001598119736,
+      "rewards/accuracy_reward": 0.8820234537124634,
+      "rewards/format_reward": 1.0,
+      "step": 243
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.6863021850586,
+      "epoch": 3.588235294117647,
+      "grad_norm": 1.666209048217596,
+      "kl": 0.041259765625,
+      "learning_rate": 8.205882352941176e-07,
+      "loss": 0.0016,
+      "reward": 1.8930561542510986,
+      "reward_std": 0.023166730999946594,
+      "rewards/accuracy_reward": 0.8942580223083496,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 244
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.91947174072266,
+      "epoch": 3.6029411764705883,
+      "grad_norm": 8.010516964998228,
+      "kl": 0.0419921875,
+      "learning_rate": 8.198529411764705e-07,
+      "loss": 0.0017,
+      "reward": 1.8905775547027588,
+      "reward_std": 0.01961654983460903,
+      "rewards/accuracy_reward": 0.890577495098114,
+      "rewards/format_reward": 1.0,
+      "step": 245
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.99759674072266,
+      "epoch": 3.6176470588235294,
+      "grad_norm": 2.0494021896720236,
+      "kl": 0.07470703125,
+      "learning_rate": 8.191176470588235e-07,
+      "loss": 0.003,
+      "reward": 1.8533637523651123,
+      "reward_std": 0.022241346538066864,
+      "rewards/accuracy_reward": 0.8533636331558228,
+      "rewards/format_reward": 1.0,
+      "step": 246
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.3377456665039,
+      "epoch": 3.6323529411764706,
+      "grad_norm": 2.0730725351705477,
+      "kl": 0.041748046875,
+      "learning_rate": 8.183823529411764e-07,
+      "loss": 0.0017,
+      "reward": 1.8775808811187744,
+      "reward_std": 0.02502569369971752,
+      "rewards/accuracy_reward": 0.8787827491760254,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 247
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.89663696289062,
+      "epoch": 3.6470588235294117,
+      "grad_norm": 2.0409255064434504,
+      "kl": 0.04931640625,
+      "learning_rate": 8.176470588235293e-07,
+      "loss": 0.002,
+      "reward": 1.887112021446228,
+      "reward_std": 0.02062002196907997,
+      "rewards/accuracy_reward": 0.8871120810508728,
+      "rewards/format_reward": 1.0,
+      "step": 248
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.95913696289062,
+      "epoch": 3.661764705882353,
+      "grad_norm": 1.4309091507183769,
+      "kl": 0.04638671875,
+      "learning_rate": 8.169117647058823e-07,
+      "loss": 0.0019,
+      "reward": 1.8621455430984497,
+      "reward_std": 0.02185475081205368,
+      "rewards/accuracy_reward": 0.8621454834938049,
+      "rewards/format_reward": 1.0,
+      "step": 249
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.15144348144531,
+      "epoch": 3.6764705882352944,
+      "grad_norm": 2.5697762019528216,
+      "kl": 0.0390625,
+      "learning_rate": 8.161764705882353e-07,
+      "loss": 0.0016,
+      "reward": 1.885780692100525,
+      "reward_std": 0.020246701315045357,
+      "rewards/accuracy_reward": 0.8857808113098145,
+      "rewards/format_reward": 1.0,
+      "step": 250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.49159240722656,
+      "epoch": 3.6911764705882355,
+      "grad_norm": 1.879132846743946,
+      "kl": 0.040283203125,
+      "learning_rate": 8.154411764705881e-07,
+      "loss": 0.0016,
+      "reward": 1.8713446855545044,
+      "reward_std": 0.024630680680274963,
+      "rewards/accuracy_reward": 0.8725464940071106,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 251
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.16226196289062,
+      "epoch": 3.7058823529411766,
+      "grad_norm": 2.1971208610905966,
+      "kl": 0.0400390625,
+      "learning_rate": 8.147058823529411e-07,
+      "loss": 0.0016,
+      "reward": 1.8803523778915405,
+      "reward_std": 0.02186383679509163,
+      "rewards/accuracy_reward": 0.8803524971008301,
+      "rewards/format_reward": 1.0,
+      "step": 252
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.24279022216797,
+      "epoch": 3.7205882352941178,
+      "grad_norm": 1.8334229890682787,
+      "kl": 0.0380859375,
+      "learning_rate": 8.139705882352941e-07,
+      "loss": 0.0015,
+      "reward": 1.8805620670318604,
+      "reward_std": 0.021345175802707672,
+      "rewards/accuracy_reward": 0.880561888217926,
+      "rewards/format_reward": 1.0,
+      "step": 253
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.02764892578125,
+      "epoch": 3.735294117647059,
+      "grad_norm": 3.7258956882443703,
+      "kl": 0.045654296875,
+      "learning_rate": 8.13235294117647e-07,
+      "loss": 0.0018,
+      "reward": 1.9009848833084106,
+      "reward_std": 0.02094642072916031,
+      "rewards/accuracy_reward": 0.9009847044944763,
+      "rewards/format_reward": 1.0,
+      "step": 254
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.55769348144531,
+      "epoch": 3.75,
+      "grad_norm": 1.678601436345225,
+      "kl": 0.045166015625,
+      "learning_rate": 8.125e-07,
+      "loss": 0.0018,
+      "reward": 1.8814560174942017,
+      "reward_std": 0.021061250939965248,
+      "rewards/accuracy_reward": 0.8814561367034912,
+      "rewards/format_reward": 1.0,
+      "step": 255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.65264892578125,
+      "epoch": 3.764705882352941,
+      "grad_norm": 1.745259980031886,
+      "kl": 0.04052734375,
+      "learning_rate": 8.117647058823529e-07,
+      "loss": 0.0016,
+      "reward": 1.8852721452713013,
+      "reward_std": 0.020116155967116356,
+      "rewards/accuracy_reward": 0.8852721452713013,
+      "rewards/format_reward": 1.0,
+      "step": 256
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.62019348144531,
+      "epoch": 3.7794117647058822,
+      "grad_norm": 2.461830018990095,
+      "kl": 0.041015625,
+      "learning_rate": 8.110294117647058e-07,
+      "loss": 0.0016,
+      "reward": 1.8814154863357544,
+      "reward_std": 0.02218296006321907,
+      "rewards/accuracy_reward": 0.8814153075218201,
+      "rewards/format_reward": 1.0,
+      "step": 257
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.7175521850586,
+      "epoch": 3.7941176470588234,
+      "grad_norm": 1.68704697360333,
+      "kl": 0.03955078125,
+      "learning_rate": 8.102941176470588e-07,
+      "loss": 0.0016,
+      "reward": 1.8427629470825195,
+      "reward_std": 0.023804400116205215,
+      "rewards/accuracy_reward": 0.8427630066871643,
+      "rewards/format_reward": 1.0,
+      "step": 258
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.58894348144531,
+      "epoch": 3.8088235294117645,
+      "grad_norm": 1.709855378144107,
+      "kl": 0.04052734375,
+      "learning_rate": 8.095588235294118e-07,
+      "loss": 0.0016,
+      "reward": 1.882066011428833,
+      "reward_std": 0.024021144956350327,
+      "rewards/accuracy_reward": 0.8832679986953735,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 259
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.78245544433594,
+      "epoch": 3.8235294117647056,
+      "grad_norm": 1.8255695685831124,
+      "kl": 0.04150390625,
+      "learning_rate": 8.088235294117646e-07,
+      "loss": 0.0016,
+      "reward": 1.8689602613449097,
+      "reward_std": 0.019115259870886803,
+      "rewards/accuracy_reward": 0.8689602017402649,
+      "rewards/format_reward": 1.0,
+      "step": 260
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.55409240722656,
+      "epoch": 3.838235294117647,
+      "grad_norm": 3.3416809660726403,
+      "kl": 0.04052734375,
+      "learning_rate": 8.080882352941176e-07,
+      "loss": 0.0017,
+      "reward": 1.8896381855010986,
+      "reward_std": 0.019461615011096,
+      "rewards/accuracy_reward": 0.8896381855010986,
+      "rewards/format_reward": 1.0,
+      "step": 261
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.46634674072266,
+      "epoch": 3.8529411764705883,
+      "grad_norm": 2.630911489725864,
+      "kl": 0.0380859375,
+      "learning_rate": 8.073529411764706e-07,
+      "loss": 0.0015,
+      "reward": 1.87398362159729,
+      "reward_std": 0.023873545229434967,
+      "rewards/accuracy_reward": 0.8739834427833557,
+      "rewards/format_reward": 1.0,
+      "step": 262
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.01683044433594,
+      "epoch": 3.8676470588235294,
+      "grad_norm": 1.843021207465952,
+      "kl": 0.040283203125,
+      "learning_rate": 8.066176470588235e-07,
+      "loss": 0.0016,
+      "reward": 1.8928405046463013,
+      "reward_std": 0.019875595346093178,
+      "rewards/accuracy_reward": 0.892840564250946,
+      "rewards/format_reward": 1.0,
+      "step": 263
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.4627456665039,
+      "epoch": 3.8823529411764706,
+      "grad_norm": 2.191986984722032,
+      "kl": 0.040283203125,
+      "learning_rate": 8.058823529411764e-07,
+      "loss": 0.0016,
+      "reward": 1.868396282196045,
+      "reward_std": 0.025705352425575256,
+      "rewards/accuracy_reward": 0.8683962225914001,
+      "rewards/format_reward": 1.0,
+      "step": 264
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.62139892578125,
+      "epoch": 3.8970588235294117,
+      "grad_norm": 1.9987217362414245,
+      "kl": 0.037353515625,
+      "learning_rate": 8.051470588235294e-07,
+      "loss": 0.0015,
+      "reward": 1.864815592765808,
+      "reward_std": 0.02225598692893982,
+      "rewards/accuracy_reward": 0.8648155331611633,
+      "rewards/format_reward": 1.0,
+      "step": 265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.4988021850586,
+      "epoch": 3.911764705882353,
+      "grad_norm": 2.050016597438298,
+      "kl": 0.0400390625,
+      "learning_rate": 8.044117647058823e-07,
+      "loss": 0.0016,
+      "reward": 1.8612146377563477,
+      "reward_std": 0.02399851381778717,
+      "rewards/accuracy_reward": 0.8612145781517029,
+      "rewards/format_reward": 1.0,
+      "step": 266
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.59375762939453,
+      "epoch": 3.9264705882352944,
+      "grad_norm": 1.6627115812158326,
+      "kl": 0.043701171875,
+      "learning_rate": 8.036764705882353e-07,
+      "loss": 0.0017,
+      "reward": 1.8737601041793823,
+      "reward_std": 0.024240681901574135,
+      "rewards/accuracy_reward": 0.8737599849700928,
+      "rewards/format_reward": 1.0,
+      "step": 267
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.94351196289062,
+      "epoch": 3.9411764705882355,
+      "grad_norm": 1.9515217712056225,
+      "kl": 0.0439453125,
+      "learning_rate": 8.029411764705883e-07,
+      "loss": 0.0018,
+      "reward": 1.866271734237671,
+      "reward_std": 0.023485863581299782,
+      "rewards/accuracy_reward": 0.8662715554237366,
+      "rewards/format_reward": 1.0,
+      "step": 268
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.50721740722656,
+      "epoch": 3.9558823529411766,
+      "grad_norm": 1.741853556080663,
+      "kl": 0.05029296875,
+      "learning_rate": 8.022058823529411e-07,
+      "loss": 0.002,
+      "reward": 1.88307523727417,
+      "reward_std": 0.021244022995233536,
+      "rewards/accuracy_reward": 0.8830749988555908,
+      "rewards/format_reward": 1.0,
+      "step": 269
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.47356414794922,
+      "epoch": 3.9705882352941178,
+      "grad_norm": 4.485332406331801,
+      "kl": 0.05712890625,
+      "learning_rate": 8.014705882352941e-07,
+      "loss": 0.0023,
+      "reward": 1.8939332962036133,
+      "reward_std": 0.025166498497128487,
+      "rewards/accuracy_reward": 0.8939332365989685,
+      "rewards/format_reward": 1.0,
+      "step": 270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.5613021850586,
+      "epoch": 3.985294117647059,
+      "grad_norm": 1.6003108364548544,
+      "kl": 0.060302734375,
+      "learning_rate": 8.007352941176471e-07,
+      "loss": 0.0024,
+      "reward": 1.883571982383728,
+      "reward_std": 0.025911405682563782,
+      "rewards/accuracy_reward": 0.8847737312316895,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 271
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.46575164794922,
+      "epoch": 4.0,
+      "grad_norm": 1.5983921955450255,
+      "kl": 0.0537109375,
+      "learning_rate": 8e-07,
+      "loss": 0.0022,
+      "reward": 1.8933547735214233,
+      "reward_std": 0.02186509408056736,
+      "rewards/accuracy_reward": 0.8933547735214233,
+      "rewards/format_reward": 1.0,
+      "step": 272
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.33413696289062,
+      "epoch": 4.014705882352941,
+      "grad_norm": 2.727093502391197,
+      "kl": 0.053466796875,
+      "learning_rate": 7.992647058823529e-07,
+      "loss": 0.0021,
+      "reward": 1.8750649690628052,
+      "reward_std": 0.019625522196292877,
+      "rewards/accuracy_reward": 0.8750649094581604,
+      "rewards/format_reward": 1.0,
+      "step": 273
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.62019348144531,
+      "epoch": 4.029411764705882,
+      "grad_norm": 1.8871571408393943,
+      "kl": 0.06103515625,
+      "learning_rate": 7.985294117647059e-07,
+      "loss": 0.0024,
+      "reward": 1.8743054866790771,
+      "reward_std": 0.024585077539086342,
+      "rewards/accuracy_reward": 0.8743056058883667,
+      "rewards/format_reward": 1.0,
+      "step": 274
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.87019348144531,
+      "epoch": 4.044117647058823,
+      "grad_norm": 2.102514327126164,
+      "kl": 0.05810546875,
+      "learning_rate": 7.977941176470588e-07,
+      "loss": 0.0023,
+      "reward": 1.89711594581604,
+      "reward_std": 0.021427426487207413,
+      "rewards/accuracy_reward": 0.8971157670021057,
+      "rewards/format_reward": 1.0,
+      "step": 275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.05529022216797,
+      "epoch": 4.0588235294117645,
+      "grad_norm": 1.5123356737294997,
+      "kl": 0.05810546875,
+      "learning_rate": 7.970588235294118e-07,
+      "loss": 0.0023,
+      "reward": 1.8750731945037842,
+      "reward_std": 0.022188987582921982,
+      "rewards/accuracy_reward": 0.875073254108429,
+      "rewards/format_reward": 1.0,
+      "step": 276
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.95072174072266,
+      "epoch": 4.073529411764706,
+      "grad_norm": 1.4897094983434982,
+      "kl": 0.060791015625,
+      "learning_rate": 7.963235294117646e-07,
+      "loss": 0.0024,
+      "reward": 1.886474847793579,
+      "reward_std": 0.021508391946554184,
+      "rewards/accuracy_reward": 0.8864747881889343,
+      "rewards/format_reward": 1.0,
+      "step": 277
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.2800521850586,
+      "epoch": 4.088235294117647,
+      "grad_norm": 3.202169591157568,
+      "kl": 0.05517578125,
+      "learning_rate": 7.955882352941175e-07,
+      "loss": 0.0022,
+      "reward": 1.8837499618530273,
+      "reward_std": 0.02077949419617653,
+      "rewards/accuracy_reward": 0.8849518299102783,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 278
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.98798370361328,
+      "epoch": 4.102941176470588,
+      "grad_norm": 2.3522518543200115,
+      "kl": 0.0498046875,
+      "learning_rate": 7.948529411764705e-07,
+      "loss": 0.002,
+      "reward": 1.8832675218582153,
+      "reward_std": 0.02067939005792141,
+      "rewards/accuracy_reward": 0.883267343044281,
+      "rewards/format_reward": 1.0,
+      "step": 279
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.70072174072266,
+      "epoch": 4.117647058823529,
+      "grad_norm": 1.4424360808940544,
+      "kl": 0.0478515625,
+      "learning_rate": 7.941176470588235e-07,
+      "loss": 0.0019,
+      "reward": 1.8915950059890747,
+      "reward_std": 0.018967358395457268,
+      "rewards/accuracy_reward": 0.8915950059890747,
+      "rewards/format_reward": 1.0,
+      "step": 280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.0096206665039,
+      "epoch": 4.132352941176471,
+      "grad_norm": 1.6320380993577588,
+      "kl": 0.057373046875,
+      "learning_rate": 7.933823529411764e-07,
+      "loss": 0.0023,
+      "reward": 1.8841755390167236,
+      "reward_std": 0.019956490024924278,
+      "rewards/accuracy_reward": 0.8841755390167236,
+      "rewards/format_reward": 1.0,
+      "step": 281
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.82452392578125,
+      "epoch": 4.147058823529412,
+      "grad_norm": 1.6748728201505565,
+      "kl": 0.0478515625,
+      "learning_rate": 7.926470588235293e-07,
+      "loss": 0.0019,
+      "reward": 1.881099820137024,
+      "reward_std": 0.01961349882185459,
+      "rewards/accuracy_reward": 0.8810995817184448,
+      "rewards/format_reward": 1.0,
+      "step": 282
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.52404022216797,
+      "epoch": 4.161764705882353,
+      "grad_norm": 3.540072890075679,
+      "kl": 0.04736328125,
+      "learning_rate": 7.919117647058823e-07,
+      "loss": 0.0019,
+      "reward": 1.8972245454788208,
+      "reward_std": 0.020341571420431137,
+      "rewards/accuracy_reward": 0.897224485874176,
+      "rewards/format_reward": 1.0,
+      "step": 283
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.55288696289062,
+      "epoch": 4.176470588235294,
+      "grad_norm": 1.9653905551543096,
+      "kl": 0.045654296875,
+      "learning_rate": 7.911764705882352e-07,
+      "loss": 0.0018,
+      "reward": 1.8803445100784302,
+      "reward_std": 0.022631661966443062,
+      "rewards/accuracy_reward": 0.8803444504737854,
+      "rewards/format_reward": 1.0,
+      "step": 284
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.54688262939453,
+      "epoch": 4.1911764705882355,
+      "grad_norm": 3.95007121019553,
+      "kl": 0.046875,
+      "learning_rate": 7.904411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.8853232860565186,
+      "reward_std": 0.02392488718032837,
+      "rewards/accuracy_reward": 0.8865249752998352,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.4627456665039,
+      "epoch": 4.205882352941177,
+      "grad_norm": 1.6574614891623431,
+      "kl": 0.05078125,
+      "learning_rate": 7.897058823529411e-07,
+      "loss": 0.002,
+      "reward": 1.8902332782745361,
+      "reward_std": 0.024903936311602592,
+      "rewards/accuracy_reward": 0.8914350867271423,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 286
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.29688262939453,
+      "epoch": 4.220588235294118,
+      "grad_norm": 3.256913991460478,
+      "kl": 0.046142578125,
+      "learning_rate": 7.88970588235294e-07,
+      "loss": 0.0018,
+      "reward": 1.8858234882354736,
+      "reward_std": 0.024734003469347954,
+      "rewards/accuracy_reward": 0.8882272243499756,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 287
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.60697174072266,
+      "epoch": 4.235294117647059,
+      "grad_norm": 1.3114281230029299,
+      "kl": 0.0458984375,
+      "learning_rate": 7.88235294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.890177607536316,
+      "reward_std": 0.02220279537141323,
+      "rewards/accuracy_reward": 0.8901773691177368,
+      "rewards/format_reward": 1.0,
+      "step": 288
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.62741088867188,
+      "epoch": 4.25,
+      "grad_norm": 1.4493002218382003,
+      "kl": 0.049560546875,
+      "learning_rate": 7.875e-07,
+      "loss": 0.002,
+      "reward": 1.8985555171966553,
+      "reward_std": 0.020094698294997215,
+      "rewards/accuracy_reward": 0.8985555768013,
+      "rewards/format_reward": 1.0,
+      "step": 289
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.68389892578125,
+      "epoch": 4.264705882352941,
+      "grad_norm": 4.352470640135198,
+      "kl": 0.04931640625,
+      "learning_rate": 7.867647058823529e-07,
+      "loss": 0.002,
+      "reward": 1.8766732215881348,
+      "reward_std": 0.027829263359308243,
+      "rewards/accuracy_reward": 0.8790771961212158,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.04688262939453,
+      "epoch": 4.279411764705882,
+      "grad_norm": 3.874276314718884,
+      "kl": 0.0830078125,
+      "learning_rate": 7.860294117647058e-07,
+      "loss": 0.0033,
+      "reward": 1.8896328210830688,
+      "reward_std": 0.021389471367001534,
+      "rewards/accuracy_reward": 0.8896327614784241,
+      "rewards/format_reward": 1.0,
+      "step": 291
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.61178588867188,
+      "epoch": 4.294117647058823,
+      "grad_norm": 4.875295082608794,
+      "kl": 0.04541015625,
+      "learning_rate": 7.852941176470588e-07,
+      "loss": 0.0018,
+      "reward": 1.8860790729522705,
+      "reward_std": 0.021392209455370903,
+      "rewards/accuracy_reward": 0.8872808814048767,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 292
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.39543914794922,
+      "epoch": 4.3088235294117645,
+      "grad_norm": 1.5633011375678962,
+      "kl": 0.04345703125,
+      "learning_rate": 7.845588235294117e-07,
+      "loss": 0.0017,
+      "reward": 1.8786873817443848,
+      "reward_std": 0.021603846922516823,
+      "rewards/accuracy_reward": 0.8786874413490295,
+      "rewards/format_reward": 1.0,
+      "step": 293
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.50360870361328,
+      "epoch": 4.323529411764706,
+      "grad_norm": 1.4773757778833134,
+      "kl": 0.047119140625,
+      "learning_rate": 7.838235294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.8730394840240479,
+      "reward_std": 0.02080570161342621,
+      "rewards/accuracy_reward": 0.8730394244194031,
+      "rewards/format_reward": 1.0,
+      "step": 294
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.4363021850586,
+      "epoch": 4.338235294117647,
+      "grad_norm": 1.8801422889509678,
+      "kl": 0.04638671875,
+      "learning_rate": 7.830882352941176e-07,
+      "loss": 0.0019,
+      "reward": 1.8871146440505981,
+      "reward_std": 0.020325137302279472,
+      "rewards/accuracy_reward": 0.8871145844459534,
+      "rewards/format_reward": 1.0,
+      "step": 295
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.09014892578125,
+      "epoch": 4.352941176470588,
+      "grad_norm": 2.5097065769254594,
+      "kl": 0.0498046875,
+      "learning_rate": 7.823529411764705e-07,
+      "loss": 0.002,
+      "reward": 1.868382453918457,
+      "reward_std": 0.02320130541920662,
+      "rewards/accuracy_reward": 0.8695842623710632,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 296
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.3377456665039,
+      "epoch": 4.367647058823529,
+      "grad_norm": 2.553942778131699,
+      "kl": 0.044189453125,
+      "learning_rate": 7.816176470588235e-07,
+      "loss": 0.0018,
+      "reward": 1.9048007726669312,
+      "reward_std": 0.019152075052261353,
+      "rewards/accuracy_reward": 0.9048006534576416,
+      "rewards/format_reward": 1.0,
+      "step": 297
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.56610870361328,
+      "epoch": 4.382352941176471,
+      "grad_norm": 1.2815769671843946,
+      "kl": 0.049560546875,
+      "learning_rate": 7.808823529411765e-07,
+      "loss": 0.002,
+      "reward": 1.8882514238357544,
+      "reward_std": 0.023224936798214912,
+      "rewards/accuracy_reward": 0.8906552791595459,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 298
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.27043914794922,
+      "epoch": 4.397058823529412,
+      "grad_norm": 1.4236109919766304,
+      "kl": 0.0439453125,
+      "learning_rate": 7.801470588235293e-07,
+      "loss": 0.0018,
+      "reward": 1.8794277906417847,
+      "reward_std": 0.022136587649583817,
+      "rewards/accuracy_reward": 0.8818315863609314,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 299
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.25000762939453,
+      "epoch": 4.411764705882353,
+      "grad_norm": 1.4092988693440858,
+      "kl": 0.044189453125,
+      "learning_rate": 7.794117647058823e-07,
+      "loss": 0.0018,
+      "reward": 1.8663372993469238,
+      "reward_std": 0.02092735283076763,
+      "rewards/accuracy_reward": 0.8663372993469238,
+      "rewards/format_reward": 1.0,
+      "step": 300
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.10577392578125,
+      "epoch": 4.426470588235294,
+      "grad_norm": 2.054063342384727,
+      "kl": 0.0517578125,
+      "learning_rate": 7.786764705882353e-07,
+      "loss": 0.0021,
+      "reward": 1.8976588249206543,
+      "reward_std": 0.01807982660830021,
+      "rewards/accuracy_reward": 0.8976587653160095,
+      "rewards/format_reward": 1.0,
+      "step": 301
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.3377456665039,
+      "epoch": 4.4411764705882355,
+      "grad_norm": 1.9093857237441312,
+      "kl": 0.047607421875,
+      "learning_rate": 7.779411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.8833731412887573,
+      "reward_std": 0.01974165067076683,
+      "rewards/accuracy_reward": 0.8833730816841125,
+      "rewards/format_reward": 1.0,
+      "step": 302
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.30409240722656,
+      "epoch": 4.455882352941177,
+      "grad_norm": 2.6735018202140988,
+      "kl": 0.04541015625,
+      "learning_rate": 7.772058823529412e-07,
+      "loss": 0.0018,
+      "reward": 1.8876824378967285,
+      "reward_std": 0.019306330010294914,
+      "rewards/accuracy_reward": 0.8876824975013733,
+      "rewards/format_reward": 1.0,
+      "step": 303
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.91106414794922,
+      "epoch": 4.470588235294118,
+      "grad_norm": 1.4970895120586163,
+      "kl": 0.0439453125,
+      "learning_rate": 7.764705882352941e-07,
+      "loss": 0.0018,
+      "reward": 1.8828318119049072,
+      "reward_std": 0.020149705931544304,
+      "rewards/accuracy_reward": 0.8840336799621582,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 304
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.24519348144531,
+      "epoch": 4.485294117647059,
+      "grad_norm": 5.117496046280143,
+      "kl": 0.04296875,
+      "learning_rate": 7.75735294117647e-07,
+      "loss": 0.0017,
+      "reward": 1.8947540521621704,
+      "reward_std": 0.024016601964831352,
+      "rewards/accuracy_reward": 0.8971578478813171,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 305
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.50360870361328,
+      "epoch": 4.5,
+      "grad_norm": 2.5813219086011028,
+      "kl": 0.052001953125,
+      "learning_rate": 7.75e-07,
+      "loss": 0.0021,
+      "reward": 1.8869359493255615,
+      "reward_std": 0.02114337682723999,
+      "rewards/accuracy_reward": 0.8869357705116272,
+      "rewards/format_reward": 1.0,
+      "step": 306
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.6550521850586,
+      "epoch": 4.514705882352941,
+      "grad_norm": 2.5546889887402235,
+      "kl": 0.0458984375,
+      "learning_rate": 7.74264705882353e-07,
+      "loss": 0.0018,
+      "reward": 1.8768377304077148,
+      "reward_std": 0.020170308649539948,
+      "rewards/accuracy_reward": 0.8768375515937805,
+      "rewards/format_reward": 1.0,
+      "step": 307
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.10216522216797,
+      "epoch": 4.529411764705882,
+      "grad_norm": 2.0444532213674775,
+      "kl": 0.048828125,
+      "learning_rate": 7.735294117647058e-07,
+      "loss": 0.002,
+      "reward": 1.8887766599655151,
+      "reward_std": 0.019136058166623116,
+      "rewards/accuracy_reward": 0.8887766599655151,
+      "rewards/format_reward": 1.0,
+      "step": 308
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.73918914794922,
+      "epoch": 4.544117647058823,
+      "grad_norm": 1.4865300788991052,
+      "kl": 0.05419921875,
+      "learning_rate": 7.727941176470588e-07,
+      "loss": 0.0022,
+      "reward": 1.8884928226470947,
+      "reward_std": 0.018670309334993362,
+      "rewards/accuracy_reward": 0.88849276304245,
+      "rewards/format_reward": 1.0,
+      "step": 309
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.81610870361328,
+      "epoch": 4.5588235294117645,
+      "grad_norm": 1.8317615570289676,
+      "kl": 0.0498046875,
+      "learning_rate": 7.720588235294118e-07,
+      "loss": 0.002,
+      "reward": 1.8673958778381348,
+      "reward_std": 0.022479576990008354,
+      "rewards/accuracy_reward": 0.8673959374427795,
+      "rewards/format_reward": 1.0,
+      "step": 310
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.82933044433594,
+      "epoch": 4.573529411764706,
+      "grad_norm": 1.3908315302560892,
+      "kl": 0.047119140625,
+      "learning_rate": 7.713235294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.8690990209579468,
+      "reward_std": 0.01971607096493244,
+      "rewards/accuracy_reward": 0.8690990805625916,
+      "rewards/format_reward": 1.0,
+      "step": 311
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.58053588867188,
+      "epoch": 4.588235294117647,
+      "grad_norm": 2.2483220834096223,
+      "kl": 0.0458984375,
+      "learning_rate": 7.705882352941177e-07,
+      "loss": 0.0018,
+      "reward": 1.8699548244476318,
+      "reward_std": 0.02238030731678009,
+      "rewards/accuracy_reward": 0.8699548840522766,
+      "rewards/format_reward": 1.0,
+      "step": 312
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.37741088867188,
+      "epoch": 4.602941176470588,
+      "grad_norm": 1.9027749884419907,
+      "kl": 0.049560546875,
+      "learning_rate": 7.698529411764706e-07,
+      "loss": 0.002,
+      "reward": 1.8906946182250977,
+      "reward_std": 0.02228682115674019,
+      "rewards/accuracy_reward": 0.8918964266777039,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 313
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.70673370361328,
+      "epoch": 4.617647058823529,
+      "grad_norm": 1.2672783381025527,
+      "kl": 0.045654296875,
+      "learning_rate": 7.691176470588235e-07,
+      "loss": 0.0018,
+      "reward": 1.8969817161560059,
+      "reward_std": 0.01983834058046341,
+      "rewards/accuracy_reward": 0.8969816565513611,
+      "rewards/format_reward": 1.0,
+      "step": 314
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.74279022216797,
+      "epoch": 4.632352941176471,
+      "grad_norm": 1.3268803478175482,
+      "kl": 0.0458984375,
+      "learning_rate": 7.683823529411765e-07,
+      "loss": 0.0018,
+      "reward": 1.884127140045166,
+      "reward_std": 0.02158300392329693,
+      "rewards/accuracy_reward": 0.884127140045166,
+      "rewards/format_reward": 1.0,
+      "step": 315
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.26683044433594,
+      "epoch": 4.647058823529412,
+      "grad_norm": 4.254278045186561,
+      "kl": 0.045654296875,
+      "learning_rate": 7.676470588235295e-07,
+      "loss": 0.0018,
+      "reward": 1.8847973346710205,
+      "reward_std": 0.023614434525370598,
+      "rewards/accuracy_reward": 0.887201189994812,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 316
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.02644348144531,
+      "epoch": 4.661764705882353,
+      "grad_norm": 1.4138173515058006,
+      "kl": 0.045654296875,
+      "learning_rate": 7.669117647058823e-07,
+      "loss": 0.0018,
+      "reward": 1.8916139602661133,
+      "reward_std": 0.021671609953045845,
+      "rewards/accuracy_reward": 0.8916139006614685,
+      "rewards/format_reward": 1.0,
+      "step": 317
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.03485870361328,
+      "epoch": 4.676470588235294,
+      "grad_norm": 1.4297077613699323,
+      "kl": 0.041748046875,
+      "learning_rate": 7.661764705882353e-07,
+      "loss": 0.0017,
+      "reward": 1.8865143060684204,
+      "reward_std": 0.022390324622392654,
+      "rewards/accuracy_reward": 0.8877161145210266,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 318
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.99759674072266,
+      "epoch": 4.6911764705882355,
+      "grad_norm": 1.2033391337835728,
+      "kl": 0.041259765625,
+      "learning_rate": 7.654411764705882e-07,
+      "loss": 0.0017,
+      "reward": 1.8856085538864136,
+      "reward_std": 0.024989847093820572,
+      "rewards/accuracy_reward": 0.8868104815483093,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 319
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.29447174072266,
+      "epoch": 4.705882352941177,
+      "grad_norm": 1.3733454266831897,
+      "kl": 0.044189453125,
+      "learning_rate": 7.647058823529411e-07,
+      "loss": 0.0018,
+      "reward": 1.878577709197998,
+      "reward_std": 0.021778374910354614,
+      "rewards/accuracy_reward": 0.8785776495933533,
+      "rewards/format_reward": 1.0,
+      "step": 320
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.90745544433594,
+      "epoch": 4.720588235294118,
+      "grad_norm": 1.5274339339441825,
+      "kl": 0.039794921875,
+      "learning_rate": 7.63970588235294e-07,
+      "loss": 0.0016,
+      "reward": 1.872501015663147,
+      "reward_std": 0.01973523572087288,
+      "rewards/accuracy_reward": 0.8725009560585022,
+      "rewards/format_reward": 1.0,
+      "step": 321
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.89063262939453,
+      "epoch": 4.735294117647059,
+      "grad_norm": 1.5962482033199081,
+      "kl": 0.041015625,
+      "learning_rate": 7.63235294117647e-07,
+      "loss": 0.0016,
+      "reward": 1.8869467973709106,
+      "reward_std": 0.022963199764490128,
+      "rewards/accuracy_reward": 0.8881486058235168,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 322
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.35938262939453,
+      "epoch": 4.75,
+      "grad_norm": 2.017481820546224,
+      "kl": 0.039794921875,
+      "learning_rate": 7.624999999999999e-07,
+      "loss": 0.0016,
+      "reward": 1.8841745853424072,
+      "reward_std": 0.025449533015489578,
+      "rewards/accuracy_reward": 0.886578381061554,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 323
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.48918914794922,
+      "epoch": 4.764705882352941,
+      "grad_norm": 5.893277558364017,
+      "kl": 0.04052734375,
+      "learning_rate": 7.617647058823529e-07,
+      "loss": 0.0016,
+      "reward": 1.8966243267059326,
+      "reward_std": 0.017657946795225143,
+      "rewards/accuracy_reward": 0.8966242074966431,
+      "rewards/format_reward": 1.0,
+      "step": 324
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.59495544433594,
+      "epoch": 4.779411764705882,
+      "grad_norm": 1.9364427240337212,
+      "kl": 0.03857421875,
+      "learning_rate": 7.610294117647059e-07,
+      "loss": 0.0015,
+      "reward": 1.8959040641784668,
+      "reward_std": 0.01980137825012207,
+      "rewards/accuracy_reward": 0.8959041237831116,
+      "rewards/format_reward": 1.0,
+      "step": 325
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.75000762939453,
+      "epoch": 4.794117647058823,
+      "grad_norm": 1.8831495019330111,
+      "kl": 0.042724609375,
+      "learning_rate": 7.602941176470587e-07,
+      "loss": 0.0017,
+      "reward": 1.8923227787017822,
+      "reward_std": 0.021822385489940643,
+      "rewards/accuracy_reward": 0.8935245871543884,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 326
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.97476196289062,
+      "epoch": 4.8088235294117645,
+      "grad_norm": 3.7686300331111924,
+      "kl": 0.0478515625,
+      "learning_rate": 7.595588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.9007014036178589,
+      "reward_std": 0.020531214773654938,
+      "rewards/accuracy_reward": 0.9031053781509399,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 327
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.31610870361328,
+      "epoch": 4.823529411764706,
+      "grad_norm": 2.3075627417580566,
+      "kl": 0.043212890625,
+      "learning_rate": 7.588235294117647e-07,
+      "loss": 0.0017,
+      "reward": 1.8857510089874268,
+      "reward_std": 0.019290894269943237,
+      "rewards/accuracy_reward": 0.8857510089874268,
+      "rewards/format_reward": 1.0,
+      "step": 328
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.52284240722656,
+      "epoch": 4.838235294117647,
+      "grad_norm": 2.4505241717663218,
+      "kl": 0.044677734375,
+      "learning_rate": 7.580882352941176e-07,
+      "loss": 0.0018,
+      "reward": 1.8795123100280762,
+      "reward_std": 0.017949938774108887,
+      "rewards/accuracy_reward": 0.8795123100280762,
+      "rewards/format_reward": 1.0,
+      "step": 329
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.17548370361328,
+      "epoch": 4.852941176470588,
+      "grad_norm": 1.715533751803751,
+      "kl": 0.04443359375,
+      "learning_rate": 7.573529411764705e-07,
+      "loss": 0.0018,
+      "reward": 1.8877590894699097,
+      "reward_std": 0.01760478876531124,
+      "rewards/accuracy_reward": 0.8877590298652649,
+      "rewards/format_reward": 1.0,
+      "step": 330
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.52644348144531,
+      "epoch": 4.867647058823529,
+      "grad_norm": 1.6176970942840496,
+      "kl": 0.049560546875,
+      "learning_rate": 7.566176470588235e-07,
+      "loss": 0.002,
+      "reward": 1.8978863954544067,
+      "reward_std": 0.01671546697616577,
+      "rewards/accuracy_reward": 0.8978862166404724,
+      "rewards/format_reward": 1.0,
+      "step": 331
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.68870544433594,
+      "epoch": 4.882352941176471,
+      "grad_norm": 1.7008680805466612,
+      "kl": 0.04541015625,
+      "learning_rate": 7.558823529411764e-07,
+      "loss": 0.0018,
+      "reward": 1.8799728155136108,
+      "reward_std": 0.017163129523396492,
+      "rewards/accuracy_reward": 0.8799727559089661,
+      "rewards/format_reward": 1.0,
+      "step": 332
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.3064956665039,
+      "epoch": 4.897058823529412,
+      "grad_norm": 5.121313120929418,
+      "kl": 0.04443359375,
+      "learning_rate": 7.551470588235294e-07,
+      "loss": 0.0018,
+      "reward": 1.8954551219940186,
+      "reward_std": 0.01760135032236576,
+      "rewards/accuracy_reward": 0.8954549431800842,
+      "rewards/format_reward": 1.0,
+      "step": 333
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.14784240722656,
+      "epoch": 4.911764705882353,
+      "grad_norm": 1.3552009766911794,
+      "kl": 0.046142578125,
+      "learning_rate": 7.544117647058824e-07,
+      "loss": 0.0018,
+      "reward": 1.8577083349227905,
+      "reward_std": 0.018814481794834137,
+      "rewards/accuracy_reward": 0.8577081561088562,
+      "rewards/format_reward": 1.0,
+      "step": 334
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.55889892578125,
+      "epoch": 4.926470588235294,
+      "grad_norm": 2.140682303302643,
+      "kl": 0.05078125,
+      "learning_rate": 7.536764705882352e-07,
+      "loss": 0.002,
+      "reward": 1.881057620048523,
+      "reward_std": 0.019572759047150612,
+      "rewards/accuracy_reward": 0.881057620048523,
+      "rewards/format_reward": 1.0,
+      "step": 335
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.62620544433594,
+      "epoch": 4.9411764705882355,
+      "grad_norm": 3.272070631910461,
+      "kl": 0.046630859375,
+      "learning_rate": 7.529411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.8901574611663818,
+      "reward_std": 0.015406808815896511,
+      "rewards/accuracy_reward": 0.8901574611663818,
+      "rewards/format_reward": 1.0,
+      "step": 336
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.64784240722656,
+      "epoch": 4.955882352941177,
+      "grad_norm": 3.6448062675563726,
+      "kl": 0.053955078125,
+      "learning_rate": 7.522058823529412e-07,
+      "loss": 0.0022,
+      "reward": 1.875773310661316,
+      "reward_std": 0.02223069965839386,
+      "rewards/accuracy_reward": 0.8769751191139221,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 337
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.64904022216797,
+      "epoch": 4.970588235294118,
+      "grad_norm": 1.4354791062562573,
+      "kl": 0.06103515625,
+      "learning_rate": 7.514705882352941e-07,
+      "loss": 0.0024,
+      "reward": 1.8930749893188477,
+      "reward_std": 0.018565351143479347,
+      "rewards/accuracy_reward": 0.8930751085281372,
+      "rewards/format_reward": 1.0,
+      "step": 338
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.71995544433594,
+      "epoch": 4.985294117647059,
+      "grad_norm": 1.8660094880654952,
+      "kl": 0.05810546875,
+      "learning_rate": 7.50735294117647e-07,
+      "loss": 0.0023,
+      "reward": 1.8842915296554565,
+      "reward_std": 0.02008167654275894,
+      "rewards/accuracy_reward": 0.8842914700508118,
+      "rewards/format_reward": 1.0,
+      "step": 339
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.37290954589844,
+      "epoch": 5.0,
+      "grad_norm": 1.508014879105866,
+      "kl": 0.05078125,
+      "learning_rate": 7.5e-07,
+      "loss": 0.002,
+      "reward": 1.8771898746490479,
+      "reward_std": 0.02064666524529457,
+      "rewards/accuracy_reward": 0.8771898150444031,
+      "rewards/format_reward": 1.0,
+      "step": 340
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.55889892578125,
+      "epoch": 5.014705882352941,
+      "grad_norm": 1.6330775894781793,
+      "kl": 0.0478515625,
+      "learning_rate": 7.492647058823529e-07,
+      "loss": 0.0019,
+      "reward": 1.8954782485961914,
+      "reward_std": 0.017391271889209747,
+      "rewards/accuracy_reward": 0.8954780697822571,
+      "rewards/format_reward": 1.0,
+      "step": 341
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.3738021850586,
+      "epoch": 5.029411764705882,
+      "grad_norm": 1.5308902611838158,
+      "kl": 0.04833984375,
+      "learning_rate": 7.485294117647059e-07,
+      "loss": 0.0019,
+      "reward": 1.9155678749084473,
+      "reward_std": 0.018119707703590393,
+      "rewards/accuracy_reward": 0.9167699217796326,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 342
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.8269271850586,
+      "epoch": 5.044117647058823,
+      "grad_norm": 3.6296477320844893,
+      "kl": 0.06298828125,
+      "learning_rate": 7.477941176470587e-07,
+      "loss": 0.0025,
+      "reward": 1.8854138851165771,
+      "reward_std": 0.02051103487610817,
+      "rewards/accuracy_reward": 0.8854138851165771,
+      "rewards/format_reward": 1.0,
+      "step": 343
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.56370544433594,
+      "epoch": 5.0588235294117645,
+      "grad_norm": 3.0646702385899562,
+      "kl": 0.047119140625,
+      "learning_rate": 7.470588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.9035041332244873,
+      "reward_std": 0.0216043870896101,
+      "rewards/accuracy_reward": 0.9047059416770935,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 344
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.50241088867188,
+      "epoch": 5.073529411764706,
+      "grad_norm": 1.7545084393443575,
+      "kl": 0.049560546875,
+      "learning_rate": 7.463235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.879274606704712,
+      "reward_std": 0.020033851265907288,
+      "rewards/accuracy_reward": 0.8792744278907776,
+      "rewards/format_reward": 1.0,
+      "step": 345
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.07572174072266,
+      "epoch": 5.088235294117647,
+      "grad_norm": 2.265810746797048,
+      "kl": 0.0458984375,
+      "learning_rate": 7.455882352941176e-07,
+      "loss": 0.0018,
+      "reward": 1.8713810443878174,
+      "reward_std": 0.019391771405935287,
+      "rewards/accuracy_reward": 0.8713809251785278,
+      "rewards/format_reward": 1.0,
+      "step": 346
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.33053588867188,
+      "epoch": 5.102941176470588,
+      "grad_norm": 4.678653380466847,
+      "kl": 0.058349609375,
+      "learning_rate": 7.448529411764706e-07,
+      "loss": 0.0023,
+      "reward": 1.8980027437210083,
+      "reward_std": 0.019409772008657455,
+      "rewards/accuracy_reward": 0.8992045521736145,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 347
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.4519271850586,
+      "epoch": 5.117647058823529,
+      "grad_norm": 5.587657670792633,
+      "kl": 0.044921875,
+      "learning_rate": 7.441176470588235e-07,
+      "loss": 0.0018,
+      "reward": 1.8735015392303467,
+      "reward_std": 0.019884636625647545,
+      "rewards/accuracy_reward": 0.8735013604164124,
+      "rewards/format_reward": 1.0,
+      "step": 348
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.25000762939453,
+      "epoch": 5.132352941176471,
+      "grad_norm": 1.4732972699207496,
+      "kl": 0.050048828125,
+      "learning_rate": 7.433823529411764e-07,
+      "loss": 0.002,
+      "reward": 1.8633267879486084,
+      "reward_std": 0.020572712644934654,
+      "rewards/accuracy_reward": 0.8633267879486084,
+      "rewards/format_reward": 1.0,
+      "step": 349
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.14303588867188,
+      "epoch": 5.147058823529412,
+      "grad_norm": 1.4434229984530293,
+      "kl": 0.051025390625,
+      "learning_rate": 7.426470588235294e-07,
+      "loss": 0.002,
+      "reward": 1.8769383430480957,
+      "reward_std": 0.02142389491200447,
+      "rewards/accuracy_reward": 0.8781403303146362,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 350
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.86418914794922,
+      "epoch": 5.161764705882353,
+      "grad_norm": 2.15469914353684,
+      "kl": 0.04541015625,
+      "learning_rate": 7.419117647058824e-07,
+      "loss": 0.0018,
+      "reward": 1.882927417755127,
+      "reward_std": 0.019372547045350075,
+      "rewards/accuracy_reward": 0.882927417755127,
+      "rewards/format_reward": 1.0,
+      "step": 351
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.43389892578125,
+      "epoch": 5.176470588235294,
+      "grad_norm": 1.7010413230089305,
+      "kl": 0.04345703125,
+      "learning_rate": 7.411764705882352e-07,
+      "loss": 0.0017,
+      "reward": 1.8917574882507324,
+      "reward_std": 0.021130278706550598,
+      "rewards/accuracy_reward": 0.8929594159126282,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 352
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.03726196289062,
+      "epoch": 5.1911764705882355,
+      "grad_norm": 1.4178252033931626,
+      "kl": 0.0458984375,
+      "learning_rate": 7.404411764705882e-07,
+      "loss": 0.0018,
+      "reward": 1.902689814567566,
+      "reward_std": 0.020760996267199516,
+      "rewards/accuracy_reward": 0.9038916826248169,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 353
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.5721206665039,
+      "epoch": 5.205882352941177,
+      "grad_norm": 3.030997635370593,
+      "kl": 0.049072265625,
+      "learning_rate": 7.397058823529412e-07,
+      "loss": 0.002,
+      "reward": 1.8927603960037231,
+      "reward_std": 0.017168840393424034,
+      "rewards/accuracy_reward": 0.8927604556083679,
+      "rewards/format_reward": 1.0,
+      "step": 354
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.91106414794922,
+      "epoch": 5.220588235294118,
+      "grad_norm": 1.680026683397978,
+      "kl": 0.046630859375,
+      "learning_rate": 7.389705882352941e-07,
+      "loss": 0.0019,
+      "reward": 1.9062864780426025,
+      "reward_std": 0.021335626021027565,
+      "rewards/accuracy_reward": 0.9074884653091431,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 355
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.44231414794922,
+      "epoch": 5.235294117647059,
+      "grad_norm": 1.9153566616206694,
+      "kl": 0.044189453125,
+      "learning_rate": 7.382352941176471e-07,
+      "loss": 0.0018,
+      "reward": 1.8809791803359985,
+      "reward_std": 0.02271600253880024,
+      "rewards/accuracy_reward": 0.8821810483932495,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 356
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.19231414794922,
+      "epoch": 5.25,
+      "grad_norm": 3.558694171062752,
+      "kl": 0.048583984375,
+      "learning_rate": 7.375e-07,
+      "loss": 0.002,
+      "reward": 1.9085917472839355,
+      "reward_std": 0.01498875766992569,
+      "rewards/accuracy_reward": 0.9085916876792908,
+      "rewards/format_reward": 1.0,
+      "step": 357
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.37019348144531,
+      "epoch": 5.264705882352941,
+      "grad_norm": 2.0945595152655208,
+      "kl": 0.04443359375,
+      "learning_rate": 7.367647058823529e-07,
+      "loss": 0.0018,
+      "reward": 1.8866630792617798,
+      "reward_std": 0.021569877862930298,
+      "rewards/accuracy_reward": 0.8890668749809265,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 358
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.05769348144531,
+      "epoch": 5.279411764705882,
+      "grad_norm": 1.8644120471415602,
+      "kl": 0.048095703125,
+      "learning_rate": 7.360294117647059e-07,
+      "loss": 0.0019,
+      "reward": 1.890414834022522,
+      "reward_std": 0.021897265687584877,
+      "rewards/accuracy_reward": 0.891616702079773,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 359
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.48438262939453,
+      "epoch": 5.294117647058823,
+      "grad_norm": 2.9245490886556236,
+      "kl": 0.044921875,
+      "learning_rate": 7.352941176470589e-07,
+      "loss": 0.0018,
+      "reward": 1.8733696937561035,
+      "reward_std": 0.019373353570699692,
+      "rewards/accuracy_reward": 0.8745716214179993,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 360
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.13101196289062,
+      "epoch": 5.3088235294117645,
+      "grad_norm": 2.4171192429296933,
+      "kl": 0.04833984375,
+      "learning_rate": 7.345588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.8899723291397095,
+      "reward_std": 0.020467419177293777,
+      "rewards/accuracy_reward": 0.8911741971969604,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 361
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.25000762939453,
+      "epoch": 5.323529411764706,
+      "grad_norm": 2.1361138297509035,
+      "kl": 0.0439453125,
+      "learning_rate": 7.338235294117646e-07,
+      "loss": 0.0018,
+      "reward": 1.892791509628296,
+      "reward_std": 0.015906570479273796,
+      "rewards/accuracy_reward": 0.8927913308143616,
+      "rewards/format_reward": 1.0,
+      "step": 362
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.56370544433594,
+      "epoch": 5.338235294117647,
+      "grad_norm": 1.92362426869984,
+      "kl": 0.045166015625,
+      "learning_rate": 7.330882352941176e-07,
+      "loss": 0.0018,
+      "reward": 1.8666787147521973,
+      "reward_std": 0.019119028002023697,
+      "rewards/accuracy_reward": 0.8666786551475525,
+      "rewards/format_reward": 1.0,
+      "step": 363
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.6971206665039,
+      "epoch": 5.352941176470588,
+      "grad_norm": 1.6581703712574578,
+      "kl": 0.049072265625,
+      "learning_rate": 7.323529411764705e-07,
+      "loss": 0.002,
+      "reward": 1.874783992767334,
+      "reward_std": 0.01916446164250374,
+      "rewards/accuracy_reward": 0.8759857416152954,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 364
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.17428588867188,
+      "epoch": 5.367647058823529,
+      "grad_norm": 1.316540945229491,
+      "kl": 0.0478515625,
+      "learning_rate": 7.316176470588234e-07,
+      "loss": 0.0019,
+      "reward": 1.8962125778198242,
+      "reward_std": 0.01739468425512314,
+      "rewards/accuracy_reward": 0.8962125182151794,
+      "rewards/format_reward": 1.0,
+      "step": 365
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.37620544433594,
+      "epoch": 5.382352941176471,
+      "grad_norm": 8.565938358324495,
+      "kl": 0.046630859375,
+      "learning_rate": 7.308823529411764e-07,
+      "loss": 0.0019,
+      "reward": 1.8905795812606812,
+      "reward_std": 0.020930087193846703,
+      "rewards/accuracy_reward": 0.890579342842102,
+      "rewards/format_reward": 1.0,
+      "step": 366
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.68991088867188,
+      "epoch": 5.397058823529412,
+      "grad_norm": 3.469819197381053,
+      "kl": 0.05419921875,
+      "learning_rate": 7.301470588235293e-07,
+      "loss": 0.0022,
+      "reward": 1.8995338678359985,
+      "reward_std": 0.02121318317949772,
+      "rewards/accuracy_reward": 0.9007357358932495,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 367
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.24759674072266,
+      "epoch": 5.411764705882353,
+      "grad_norm": 3.6183994362346517,
+      "kl": 0.04248046875,
+      "learning_rate": 7.294117647058823e-07,
+      "loss": 0.0017,
+      "reward": 1.8944286108016968,
+      "reward_std": 0.019341690465807915,
+      "rewards/accuracy_reward": 0.894428551197052,
+      "rewards/format_reward": 1.0,
+      "step": 368
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.42188262939453,
+      "epoch": 5.426470588235294,
+      "grad_norm": 1.5985396075036147,
+      "kl": 0.046630859375,
+      "learning_rate": 7.286764705882353e-07,
+      "loss": 0.0019,
+      "reward": 1.9031792879104614,
+      "reward_std": 0.01647799275815487,
+      "rewards/accuracy_reward": 0.9031792283058167,
+      "rewards/format_reward": 1.0,
+      "step": 369
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.56851196289062,
+      "epoch": 5.4411764705882355,
+      "grad_norm": 1.2316701809022303,
+      "kl": 0.0439453125,
+      "learning_rate": 7.279411764705881e-07,
+      "loss": 0.0018,
+      "reward": 1.874247670173645,
+      "reward_std": 0.021124592050909996,
+      "rewards/accuracy_reward": 0.8754495978355408,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 370
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.75360870361328,
+      "epoch": 5.455882352941177,
+      "grad_norm": 2.02457072563307,
+      "kl": 0.046142578125,
+      "learning_rate": 7.272058823529411e-07,
+      "loss": 0.0018,
+      "reward": 1.8876434564590454,
+      "reward_std": 0.018463727086782455,
+      "rewards/accuracy_reward": 0.887643575668335,
+      "rewards/format_reward": 1.0,
+      "step": 371
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.8894271850586,
+      "epoch": 5.470588235294118,
+      "grad_norm": 1.9670592101281013,
+      "kl": 0.04296875,
+      "learning_rate": 7.264705882352941e-07,
+      "loss": 0.0017,
+      "reward": 1.8814630508422852,
+      "reward_std": 0.016402965411543846,
+      "rewards/accuracy_reward": 0.8814628720283508,
+      "rewards/format_reward": 1.0,
+      "step": 372
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.15625762939453,
+      "epoch": 5.485294117647059,
+      "grad_norm": 1.4520387331980766,
+      "kl": 0.044189453125,
+      "learning_rate": 7.25735294117647e-07,
+      "loss": 0.0018,
+      "reward": 1.8984284400939941,
+      "reward_std": 0.015249278396368027,
+      "rewards/accuracy_reward": 0.8984284400939941,
+      "rewards/format_reward": 1.0,
+      "step": 373
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.3738021850586,
+      "epoch": 5.5,
+      "grad_norm": 1.5706724220082209,
+      "kl": 0.04833984375,
+      "learning_rate": 7.249999999999999e-07,
+      "loss": 0.0019,
+      "reward": 1.8929314613342285,
+      "reward_std": 0.018274588510394096,
+      "rewards/accuracy_reward": 0.8929312825202942,
+      "rewards/format_reward": 1.0,
+      "step": 374
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.56370544433594,
+      "epoch": 5.514705882352941,
+      "grad_norm": 3.532450016398957,
+      "kl": 0.0439453125,
+      "learning_rate": 7.242647058823529e-07,
+      "loss": 0.0018,
+      "reward": 1.892069935798645,
+      "reward_std": 0.019071701914072037,
+      "rewards/accuracy_reward": 0.8920699954032898,
+      "rewards/format_reward": 1.0,
+      "step": 375
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.27284240722656,
+      "epoch": 5.529411764705882,
+      "grad_norm": 2.9126439595526112,
+      "kl": 0.052734375,
+      "learning_rate": 7.235294117647058e-07,
+      "loss": 0.0021,
+      "reward": 1.8809665441513062,
+      "reward_std": 0.01572202891111374,
+      "rewards/accuracy_reward": 0.8809664845466614,
+      "rewards/format_reward": 1.0,
+      "step": 376
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.28846740722656,
+      "epoch": 5.544117647058823,
+      "grad_norm": 2.3143306804592423,
+      "kl": 0.048095703125,
+      "learning_rate": 7.227941176470588e-07,
+      "loss": 0.0019,
+      "reward": 1.905410885810852,
+      "reward_std": 0.015785330906510353,
+      "rewards/accuracy_reward": 0.9054108262062073,
+      "rewards/format_reward": 1.0,
+      "step": 377
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.16947174072266,
+      "epoch": 5.5588235294117645,
+      "grad_norm": 2.6475439861754095,
+      "kl": 0.04638671875,
+      "learning_rate": 7.220588235294118e-07,
+      "loss": 0.0019,
+      "reward": 1.8877785205841064,
+      "reward_std": 0.01584857515990734,
+      "rewards/accuracy_reward": 0.8877785205841064,
+      "rewards/format_reward": 1.0,
+      "step": 378
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.41346740722656,
+      "epoch": 5.573529411764706,
+      "grad_norm": 2.025239158313011,
+      "kl": 0.04736328125,
+      "learning_rate": 7.213235294117646e-07,
+      "loss": 0.0019,
+      "reward": 1.9032666683197021,
+      "reward_std": 0.016153108328580856,
+      "rewards/accuracy_reward": 0.9032665491104126,
+      "rewards/format_reward": 1.0,
+      "step": 379
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.09616088867188,
+      "epoch": 5.588235294117647,
+      "grad_norm": 4.81037382465536,
+      "kl": 0.049072265625,
+      "learning_rate": 7.205882352941176e-07,
+      "loss": 0.002,
+      "reward": 1.8973829746246338,
+      "reward_std": 0.018959591165184975,
+      "rewards/accuracy_reward": 0.8985849618911743,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 380
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.60456848144531,
+      "epoch": 5.602941176470588,
+      "grad_norm": 2.0424310708762543,
+      "kl": 0.0478515625,
+      "learning_rate": 7.198529411764706e-07,
+      "loss": 0.0019,
+      "reward": 1.891837239265442,
+      "reward_std": 0.017265740782022476,
+      "rewards/accuracy_reward": 0.8918372392654419,
+      "rewards/format_reward": 1.0,
+      "step": 381
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.49639892578125,
+      "epoch": 5.617647058823529,
+      "grad_norm": 7.450414291777024,
+      "kl": 0.040283203125,
+      "learning_rate": 7.191176470588235e-07,
+      "loss": 0.0016,
+      "reward": 1.8869714736938477,
+      "reward_std": 0.014449415728449821,
+      "rewards/accuracy_reward": 0.8869713544845581,
+      "rewards/format_reward": 1.0,
+      "step": 382
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.50841522216797,
+      "epoch": 5.632352941176471,
+      "grad_norm": 2.035166070284095,
+      "kl": 0.05029296875,
+      "learning_rate": 7.183823529411764e-07,
+      "loss": 0.002,
+      "reward": 1.8704235553741455,
+      "reward_std": 0.01924438774585724,
+      "rewards/accuracy_reward": 0.8704236149787903,
+      "rewards/format_reward": 1.0,
+      "step": 383
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.93269348144531,
+      "epoch": 5.647058823529412,
+      "grad_norm": 1.5304768166279001,
+      "kl": 0.04638671875,
+      "learning_rate": 7.176470588235294e-07,
+      "loss": 0.0019,
+      "reward": 1.889725923538208,
+      "reward_std": 0.020155081525444984,
+      "rewards/accuracy_reward": 0.890927791595459,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 384
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.50481414794922,
+      "epoch": 5.661764705882353,
+      "grad_norm": 3.3086513126052437,
+      "kl": 0.046875,
+      "learning_rate": 7.169117647058823e-07,
+      "loss": 0.0019,
+      "reward": 1.9017019271850586,
+      "reward_std": 0.02102070115506649,
+      "rewards/accuracy_reward": 0.9029037952423096,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 385
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.59856414794922,
+      "epoch": 5.676470588235294,
+      "grad_norm": 1.9961992837605382,
+      "kl": 0.04443359375,
+      "learning_rate": 7.161764705882353e-07,
+      "loss": 0.0018,
+      "reward": 1.898403286933899,
+      "reward_std": 0.01933371089398861,
+      "rewards/accuracy_reward": 0.8996050953865051,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 386
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.00360870361328,
+      "epoch": 5.6911764705882355,
+      "grad_norm": 5.861203308278023,
+      "kl": 0.044189453125,
+      "learning_rate": 7.154411764705882e-07,
+      "loss": 0.0018,
+      "reward": 1.9097044467926025,
+      "reward_std": 0.017147168517112732,
+      "rewards/accuracy_reward": 0.9097043871879578,
+      "rewards/format_reward": 1.0,
+      "step": 387
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.95433044433594,
+      "epoch": 5.705882352941177,
+      "grad_norm": 1.8272044694889817,
+      "kl": 0.046142578125,
+      "learning_rate": 7.147058823529411e-07,
+      "loss": 0.0018,
+      "reward": 1.897651195526123,
+      "reward_std": 0.019564587622880936,
+      "rewards/accuracy_reward": 0.897650957107544,
+      "rewards/format_reward": 1.0,
+      "step": 388
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.26202392578125,
+      "epoch": 5.720588235294118,
+      "grad_norm": 1.945047556411728,
+      "kl": 0.053955078125,
+      "learning_rate": 7.139705882352941e-07,
+      "loss": 0.0022,
+      "reward": 1.8845726251602173,
+      "reward_std": 0.018275026232004166,
+      "rewards/accuracy_reward": 0.8845725655555725,
+      "rewards/format_reward": 1.0,
+      "step": 389
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.27764892578125,
+      "epoch": 5.735294117647059,
+      "grad_norm": 1.872210801006099,
+      "kl": 0.0419921875,
+      "learning_rate": 7.132352941176471e-07,
+      "loss": 0.0017,
+      "reward": 1.8972781896591187,
+      "reward_std": 0.019161643460392952,
+      "rewards/accuracy_reward": 0.8972780704498291,
+      "rewards/format_reward": 1.0,
+      "step": 390
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.4206771850586,
+      "epoch": 5.75,
+      "grad_norm": 1.628706031121031,
+      "kl": 0.044921875,
+      "learning_rate": 7.125e-07,
+      "loss": 0.0018,
+      "reward": 1.8868787288665771,
+      "reward_std": 0.017812801524996758,
+      "rewards/accuracy_reward": 0.8868787884712219,
+      "rewards/format_reward": 1.0,
+      "step": 391
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.36538696289062,
+      "epoch": 5.764705882352941,
+      "grad_norm": 2.025154646985711,
+      "kl": 0.043701171875,
+      "learning_rate": 7.117647058823529e-07,
+      "loss": 0.0017,
+      "reward": 1.8947402238845825,
+      "reward_std": 0.016441920772194862,
+      "rewards/accuracy_reward": 0.8947402834892273,
+      "rewards/format_reward": 1.0,
+      "step": 392
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.59375762939453,
+      "epoch": 5.779411764705882,
+      "grad_norm": 3.1223862690686466,
+      "kl": 0.04443359375,
+      "learning_rate": 7.110294117647059e-07,
+      "loss": 0.0018,
+      "reward": 1.884828805923462,
+      "reward_std": 0.02042386308312416,
+      "rewards/accuracy_reward": 0.8848287463188171,
+      "rewards/format_reward": 1.0,
+      "step": 393
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.52163696289062,
+      "epoch": 5.794117647058823,
+      "grad_norm": 1.2474938760026444,
+      "kl": 0.0439453125,
+      "learning_rate": 7.102941176470588e-07,
+      "loss": 0.0018,
+      "reward": 1.894531488418579,
+      "reward_std": 0.017643319442868233,
+      "rewards/accuracy_reward": 0.8945314288139343,
+      "rewards/format_reward": 1.0,
+      "step": 394
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.53966522216797,
+      "epoch": 5.8088235294117645,
+      "grad_norm": 2.4031954689781556,
+      "kl": 0.04443359375,
+      "learning_rate": 7.095588235294118e-07,
+      "loss": 0.0018,
+      "reward": 1.906731367111206,
+      "reward_std": 0.019448986276984215,
+      "rewards/accuracy_reward": 0.9079332947731018,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 395
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.2752456665039,
+      "epoch": 5.823529411764706,
+      "grad_norm": 2.3766132625090495,
+      "kl": 0.06005859375,
+      "learning_rate": 7.088235294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.9001885652542114,
+      "reward_std": 0.018285507336258888,
+      "rewards/accuracy_reward": 0.9001885056495667,
+      "rewards/format_reward": 1.0,
+      "step": 396
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.21034240722656,
+      "epoch": 5.838235294117647,
+      "grad_norm": 1.3410708995130616,
+      "kl": 0.049560546875,
+      "learning_rate": 7.080882352941176e-07,
+      "loss": 0.002,
+      "reward": 1.8895319700241089,
+      "reward_std": 0.019128086045384407,
+      "rewards/accuracy_reward": 0.8907338976860046,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 397
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.0300521850586,
+      "epoch": 5.852941176470588,
+      "grad_norm": 2.1806221304312188,
+      "kl": 0.047607421875,
+      "learning_rate": 7.073529411764706e-07,
+      "loss": 0.0019,
+      "reward": 1.8873209953308105,
+      "reward_std": 0.015813617035746574,
+      "rewards/accuracy_reward": 0.8873209357261658,
+      "rewards/format_reward": 1.0,
+      "step": 398
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.05889892578125,
+      "epoch": 5.867647058823529,
+      "grad_norm": 2.381149520093015,
+      "kl": 0.05126953125,
+      "learning_rate": 7.066176470588236e-07,
+      "loss": 0.0021,
+      "reward": 1.9061532020568848,
+      "reward_std": 0.017387444153428078,
+      "rewards/accuracy_reward": 0.9061532020568848,
+      "rewards/format_reward": 1.0,
+      "step": 399
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.60216522216797,
+      "epoch": 5.882352941176471,
+      "grad_norm": 2.0072217926486235,
+      "kl": 0.048583984375,
+      "learning_rate": 7.058823529411765e-07,
+      "loss": 0.0019,
+      "reward": 1.8860886096954346,
+      "reward_std": 0.01758231595158577,
+      "rewards/accuracy_reward": 0.8860886096954346,
+      "rewards/format_reward": 1.0,
+      "step": 400
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.2908706665039,
+      "epoch": 5.897058823529412,
+      "grad_norm": 3.7976903995467244,
+      "kl": 0.043701171875,
+      "learning_rate": 7.051470588235294e-07,
+      "loss": 0.0017,
+      "reward": 1.9081175327301025,
+      "reward_std": 0.016382873058319092,
+      "rewards/accuracy_reward": 0.9081174731254578,
+      "rewards/format_reward": 1.0,
+      "step": 401
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.97235870361328,
+      "epoch": 5.911764705882353,
+      "grad_norm": 1.6832136746582174,
+      "kl": 0.05322265625,
+      "learning_rate": 7.044117647058824e-07,
+      "loss": 0.0021,
+      "reward": 1.8958063125610352,
+      "reward_std": 0.014682255685329437,
+      "rewards/accuracy_reward": 0.8958064317703247,
+      "rewards/format_reward": 1.0,
+      "step": 402
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.90384674072266,
+      "epoch": 5.926470588235294,
+      "grad_norm": 2.1815402582773618,
+      "kl": 0.0498046875,
+      "learning_rate": 7.036764705882353e-07,
+      "loss": 0.002,
+      "reward": 1.8909062147140503,
+      "reward_std": 0.018609097227454185,
+      "rewards/accuracy_reward": 0.8909061551094055,
+      "rewards/format_reward": 1.0,
+      "step": 403
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.67788696289062,
+      "epoch": 5.9411764705882355,
+      "grad_norm": 1.2061130128497455,
+      "kl": 0.047119140625,
+      "learning_rate": 7.029411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.8931101560592651,
+      "reward_std": 0.01808551885187626,
+      "rewards/accuracy_reward": 0.8931100368499756,
+      "rewards/format_reward": 1.0,
+      "step": 404
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.68870544433594,
+      "epoch": 5.955882352941177,
+      "grad_norm": 1.9433195416220157,
+      "kl": 0.046630859375,
+      "learning_rate": 7.022058823529411e-07,
+      "loss": 0.0019,
+      "reward": 1.9073554277420044,
+      "reward_std": 0.018337463960051537,
+      "rewards/accuracy_reward": 0.9073554277420044,
+      "rewards/format_reward": 1.0,
+      "step": 405
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.47476196289062,
+      "epoch": 5.970588235294118,
+      "grad_norm": 1.8820064437816768,
+      "kl": 0.060546875,
+      "learning_rate": 7.01470588235294e-07,
+      "loss": 0.0024,
+      "reward": 1.894400715827942,
+      "reward_std": 0.021061552688479424,
+      "rewards/accuracy_reward": 0.8944007158279419,
+      "rewards/format_reward": 1.0,
+      "step": 406
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.52884674072266,
+      "epoch": 5.985294117647059,
+      "grad_norm": 1.4623484883692877,
+      "kl": 0.05859375,
+      "learning_rate": 7.00735294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.9013231992721558,
+      "reward_std": 0.02373763732612133,
+      "rewards/accuracy_reward": 0.9037269949913025,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 407
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.86758422851562,
+      "epoch": 6.0,
+      "grad_norm": 2.1711922114519124,
+      "kl": 0.048828125,
+      "learning_rate": 7e-07,
+      "loss": 0.002,
+      "reward": 1.9016249179840088,
+      "reward_std": 0.023924626410007477,
+      "rewards/accuracy_reward": 0.9046690464019775,
+      "rewards/format_reward": 0.9969558715820312,
+      "step": 408
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.19110870361328,
+      "epoch": 6.014705882352941,
+      "grad_norm": 1.7310411415632332,
+      "kl": 0.052490234375,
+      "learning_rate": 6.992647058823528e-07,
+      "loss": 0.0021,
+      "reward": 1.8854323625564575,
+      "reward_std": 0.023845864459872246,
+      "rewards/accuracy_reward": 0.8866342902183533,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 409
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.46875762939453,
+      "epoch": 6.029411764705882,
+      "grad_norm": 5.687861204362904,
+      "kl": 0.0908203125,
+      "learning_rate": 6.985294117647058e-07,
+      "loss": 0.0036,
+      "reward": 1.9103180170059204,
+      "reward_std": 0.017014116048812866,
+      "rewards/accuracy_reward": 0.9103179574012756,
+      "rewards/format_reward": 1.0,
+      "step": 410
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.84134674072266,
+      "epoch": 6.044117647058823,
+      "grad_norm": 1.8355942786177852,
+      "kl": 0.048583984375,
+      "learning_rate": 6.977941176470588e-07,
+      "loss": 0.002,
+      "reward": 1.8937817811965942,
+      "reward_std": 0.018505675718188286,
+      "rewards/accuracy_reward": 0.8937817215919495,
+      "rewards/format_reward": 1.0,
+      "step": 411
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.79688262939453,
+      "epoch": 6.0588235294117645,
+      "grad_norm": 1.9759679376426085,
+      "kl": 0.052001953125,
+      "learning_rate": 6.970588235294117e-07,
+      "loss": 0.0021,
+      "reward": 1.8942158222198486,
+      "reward_std": 0.02279289998114109,
+      "rewards/accuracy_reward": 0.8954177498817444,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 412
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.37860870361328,
+      "epoch": 6.073529411764706,
+      "grad_norm": 2.300855046609347,
+      "kl": 0.059814453125,
+      "learning_rate": 6.963235294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.8962349891662598,
+      "reward_std": 0.020851602777838707,
+      "rewards/accuracy_reward": 0.8962349891662598,
+      "rewards/format_reward": 1.0,
+      "step": 413
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.5252456665039,
+      "epoch": 6.088235294117647,
+      "grad_norm": 12.310047514562172,
+      "kl": 0.047607421875,
+      "learning_rate": 6.955882352941176e-07,
+      "loss": 0.0019,
+      "reward": 1.899317979812622,
+      "reward_std": 0.020527588203549385,
+      "rewards/accuracy_reward": 0.9005199074745178,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 414
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.1238021850586,
+      "epoch": 6.102941176470588,
+      "grad_norm": 1.5229186035110283,
+      "kl": 0.05224609375,
+      "learning_rate": 6.948529411764705e-07,
+      "loss": 0.0021,
+      "reward": 1.8821978569030762,
+      "reward_std": 0.022272054105997086,
+      "rewards/accuracy_reward": 0.8833997249603271,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 415
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.74279022216797,
+      "epoch": 6.117647058823529,
+      "grad_norm": 3.2878165148453236,
+      "kl": 0.053955078125,
+      "learning_rate": 6.941176470588235e-07,
+      "loss": 0.0022,
+      "reward": 1.86842679977417,
+      "reward_std": 0.024334656074643135,
+      "rewards/accuracy_reward": 0.8684268593788147,
+      "rewards/format_reward": 1.0,
+      "step": 416
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.48918914794922,
+      "epoch": 6.132352941176471,
+      "grad_norm": 1.2525615714604925,
+      "kl": 0.05029296875,
+      "learning_rate": 6.933823529411765e-07,
+      "loss": 0.002,
+      "reward": 1.914664387702942,
+      "reward_std": 0.01958000287413597,
+      "rewards/accuracy_reward": 0.9146643877029419,
+      "rewards/format_reward": 1.0,
+      "step": 417
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.93269348144531,
+      "epoch": 6.147058823529412,
+      "grad_norm": 2.561139147224278,
+      "kl": 0.052734375,
+      "learning_rate": 6.926470588235293e-07,
+      "loss": 0.0021,
+      "reward": 1.9154963493347168,
+      "reward_std": 0.016698814928531647,
+      "rewards/accuracy_reward": 0.9154961109161377,
+      "rewards/format_reward": 1.0,
+      "step": 418
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.98077392578125,
+      "epoch": 6.161764705882353,
+      "grad_norm": 2.1752639111351666,
+      "kl": 0.058349609375,
+      "learning_rate": 6.919117647058823e-07,
+      "loss": 0.0023,
+      "reward": 1.9047746658325195,
+      "reward_std": 0.018907994031906128,
+      "rewards/accuracy_reward": 0.9047747254371643,
+      "rewards/format_reward": 1.0,
+      "step": 419
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.78606414794922,
+      "epoch": 6.176470588235294,
+      "grad_norm": 2.0292217325556416,
+      "kl": 0.0517578125,
+      "learning_rate": 6.911764705882353e-07,
+      "loss": 0.0021,
+      "reward": 1.9034497737884521,
+      "reward_std": 0.024114146828651428,
+      "rewards/accuracy_reward": 0.9034495949745178,
+      "rewards/format_reward": 1.0,
+      "step": 420
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.4519271850586,
+      "epoch": 6.1911764705882355,
+      "grad_norm": 6.699279802691781,
+      "kl": 0.055419921875,
+      "learning_rate": 6.904411764705882e-07,
+      "loss": 0.0022,
+      "reward": 1.887842059135437,
+      "reward_std": 0.020647389814257622,
+      "rewards/accuracy_reward": 0.887842059135437,
+      "rewards/format_reward": 1.0,
+      "step": 421
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.42188262939453,
+      "epoch": 6.205882352941177,
+      "grad_norm": 1.534235488134726,
+      "kl": 0.050537109375,
+      "learning_rate": 6.897058823529411e-07,
+      "loss": 0.002,
+      "reward": 1.896329402923584,
+      "reward_std": 0.018744876608252525,
+      "rewards/accuracy_reward": 0.8963293433189392,
+      "rewards/format_reward": 1.0,
+      "step": 422
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.9831771850586,
+      "epoch": 6.220588235294118,
+      "grad_norm": 1.5766135493091737,
+      "kl": 0.0478515625,
+      "learning_rate": 6.889705882352941e-07,
+      "loss": 0.0019,
+      "reward": 1.8838109970092773,
+      "reward_std": 0.02198025956749916,
+      "rewards/accuracy_reward": 0.8850129246711731,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 423
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.84616088867188,
+      "epoch": 6.235294117647059,
+      "grad_norm": 2.0708203706815853,
+      "kl": 0.078125,
+      "learning_rate": 6.88235294117647e-07,
+      "loss": 0.0031,
+      "reward": 1.8984771966934204,
+      "reward_std": 0.021190447732806206,
+      "rewards/accuracy_reward": 0.8984770178794861,
+      "rewards/format_reward": 1.0,
+      "step": 424
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.41226196289062,
+      "epoch": 6.25,
+      "grad_norm": 1.7348463789648598,
+      "kl": 0.050048828125,
+      "learning_rate": 6.875e-07,
+      "loss": 0.002,
+      "reward": 1.9097307920455933,
+      "reward_std": 0.019229857251048088,
+      "rewards/accuracy_reward": 0.9097307324409485,
+      "rewards/format_reward": 1.0,
+      "step": 425
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.37500762939453,
+      "epoch": 6.264705882352941,
+      "grad_norm": 15.785742386708213,
+      "kl": 0.05224609375,
+      "learning_rate": 6.86764705882353e-07,
+      "loss": 0.0021,
+      "reward": 1.8983619213104248,
+      "reward_std": 0.018431352451443672,
+      "rewards/accuracy_reward": 0.8983619213104248,
+      "rewards/format_reward": 1.0,
+      "step": 426
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.81009674072266,
+      "epoch": 6.279411764705882,
+      "grad_norm": 2.6324515777384017,
+      "kl": 0.056396484375,
+      "learning_rate": 6.860294117647058e-07,
+      "loss": 0.0023,
+      "reward": 1.8895386457443237,
+      "reward_std": 0.02556595578789711,
+      "rewards/accuracy_reward": 0.889538586139679,
+      "rewards/format_reward": 1.0,
+      "step": 427
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.29808044433594,
+      "epoch": 6.294117647058823,
+      "grad_norm": 1.6012975476940077,
+      "kl": 0.04931640625,
+      "learning_rate": 6.852941176470588e-07,
+      "loss": 0.002,
+      "reward": 1.9019221067428589,
+      "reward_std": 0.023045556619763374,
+      "rewards/accuracy_reward": 0.9031239748001099,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 428
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.78125762939453,
+      "epoch": 6.3088235294117645,
+      "grad_norm": 2.761466006204092,
+      "kl": 0.0732421875,
+      "learning_rate": 6.845588235294118e-07,
+      "loss": 0.0029,
+      "reward": 1.8907809257507324,
+      "reward_std": 0.021768517792224884,
+      "rewards/accuracy_reward": 0.8907808661460876,
+      "rewards/format_reward": 1.0,
+      "step": 429
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.83413696289062,
+      "epoch": 6.323529411764706,
+      "grad_norm": 1.9867024053824853,
+      "kl": 0.05517578125,
+      "learning_rate": 6.838235294117647e-07,
+      "loss": 0.0022,
+      "reward": 1.8989307880401611,
+      "reward_std": 0.02491842210292816,
+      "rewards/accuracy_reward": 0.9001325964927673,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 430
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.92548370361328,
+      "epoch": 6.338235294117647,
+      "grad_norm": 2.506151416718587,
+      "kl": 0.060546875,
+      "learning_rate": 6.830882352941176e-07,
+      "loss": 0.0024,
+      "reward": 1.8981561660766602,
+      "reward_std": 0.022075144574046135,
+      "rewards/accuracy_reward": 0.8981561064720154,
+      "rewards/format_reward": 1.0,
+      "step": 431
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.0300521850586,
+      "epoch": 6.352941176470588,
+      "grad_norm": 3.453658386620713,
+      "kl": 0.052734375,
+      "learning_rate": 6.823529411764706e-07,
+      "loss": 0.0021,
+      "reward": 1.8945103883743286,
+      "reward_std": 0.02551359124481678,
+      "rewards/accuracy_reward": 0.8969140648841858,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 432
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.82933044433594,
+      "epoch": 6.367647058823529,
+      "grad_norm": 1.8291376872404717,
+      "kl": 0.049072265625,
+      "learning_rate": 6.816176470588235e-07,
+      "loss": 0.002,
+      "reward": 1.8951138257980347,
+      "reward_std": 0.02293815091252327,
+      "rewards/accuracy_reward": 0.8951138257980347,
+      "rewards/format_reward": 1.0,
+      "step": 433
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.97356414794922,
+      "epoch": 6.382352941176471,
+      "grad_norm": 4.4657449515144965,
+      "kl": 0.051025390625,
+      "learning_rate": 6.808823529411765e-07,
+      "loss": 0.002,
+      "reward": 1.8871821165084839,
+      "reward_std": 0.02225620485842228,
+      "rewards/accuracy_reward": 0.8871821165084839,
+      "rewards/format_reward": 1.0,
+      "step": 434
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.1033706665039,
+      "epoch": 6.397058823529412,
+      "grad_norm": 1.0719524860130538,
+      "kl": 0.05029296875,
+      "learning_rate": 6.801470588235295e-07,
+      "loss": 0.002,
+      "reward": 1.8991506099700928,
+      "reward_std": 0.024187587201595306,
+      "rewards/accuracy_reward": 0.9003525376319885,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 435
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.64063262939453,
+      "epoch": 6.411764705882353,
+      "grad_norm": 1.3740348079978744,
+      "kl": 0.04736328125,
+      "learning_rate": 6.794117647058823e-07,
+      "loss": 0.0019,
+      "reward": 1.8954823017120361,
+      "reward_std": 0.024401018396019936,
+      "rewards/accuracy_reward": 0.8954823017120361,
+      "rewards/format_reward": 1.0,
+      "step": 436
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.45433044433594,
+      "epoch": 6.426470588235294,
+      "grad_norm": 2.5851908946821287,
+      "kl": 0.05126953125,
+      "learning_rate": 6.786764705882353e-07,
+      "loss": 0.002,
+      "reward": 1.9073567390441895,
+      "reward_std": 0.020375726744532585,
+      "rewards/accuracy_reward": 0.9073567390441895,
+      "rewards/format_reward": 1.0,
+      "step": 437
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.93389892578125,
+      "epoch": 6.4411764705882355,
+      "grad_norm": 1.167566426650332,
+      "kl": 0.048828125,
+      "learning_rate": 6.779411764705883e-07,
+      "loss": 0.0019,
+      "reward": 1.8960213661193848,
+      "reward_std": 0.024139244109392166,
+      "rewards/accuracy_reward": 0.8960214257240295,
+      "rewards/format_reward": 1.0,
+      "step": 438
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.06610870361328,
+      "epoch": 6.455882352941177,
+      "grad_norm": 1.1659335106544957,
+      "kl": 0.047607421875,
+      "learning_rate": 6.772058823529412e-07,
+      "loss": 0.0019,
+      "reward": 1.8937411308288574,
+      "reward_std": 0.02366737276315689,
+      "rewards/accuracy_reward": 0.8949430584907532,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 439
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.38341522216797,
+      "epoch": 6.470588235294118,
+      "grad_norm": 3.3358776773497176,
+      "kl": 0.048095703125,
+      "learning_rate": 6.764705882352941e-07,
+      "loss": 0.0019,
+      "reward": 1.8928595781326294,
+      "reward_std": 0.021879835054278374,
+      "rewards/accuracy_reward": 0.8928595781326294,
+      "rewards/format_reward": 1.0,
+      "step": 440
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.35216522216797,
+      "epoch": 6.485294117647059,
+      "grad_norm": 1.3906028161781399,
+      "kl": 0.052490234375,
+      "learning_rate": 6.757352941176471e-07,
+      "loss": 0.0021,
+      "reward": 1.8927258253097534,
+      "reward_std": 0.020552638918161392,
+      "rewards/accuracy_reward": 0.8927258849143982,
+      "rewards/format_reward": 1.0,
+      "step": 441
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.82091522216797,
+      "epoch": 6.5,
+      "grad_norm": 3.8548633882230567,
+      "kl": 0.059814453125,
+      "learning_rate": 6.75e-07,
+      "loss": 0.0024,
+      "reward": 1.9043219089508057,
+      "reward_std": 0.02147606387734413,
+      "rewards/accuracy_reward": 0.9055235981941223,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 442
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.60456848144531,
+      "epoch": 6.514705882352941,
+      "grad_norm": 2.723744256999097,
+      "kl": 0.05419921875,
+      "learning_rate": 6.74264705882353e-07,
+      "loss": 0.0022,
+      "reward": 1.8985867500305176,
+      "reward_std": 0.017505373805761337,
+      "rewards/accuracy_reward": 0.8985867500305176,
+      "rewards/format_reward": 1.0,
+      "step": 443
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.91466522216797,
+      "epoch": 6.529411764705882,
+      "grad_norm": 2.467770138712217,
+      "kl": 0.05126953125,
+      "learning_rate": 6.735294117647058e-07,
+      "loss": 0.0021,
+      "reward": 1.8991594314575195,
+      "reward_std": 0.0224330872297287,
+      "rewards/accuracy_reward": 0.9003612995147705,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 444
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.12981414794922,
+      "epoch": 6.544117647058823,
+      "grad_norm": 1.235445619018083,
+      "kl": 0.056396484375,
+      "learning_rate": 6.727941176470588e-07,
+      "loss": 0.0023,
+      "reward": 1.8991564512252808,
+      "reward_std": 0.01990494504570961,
+      "rewards/accuracy_reward": 0.900358259677887,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 445
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.72716522216797,
+      "epoch": 6.5588235294117645,
+      "grad_norm": 4.57392237252639,
+      "kl": 0.0576171875,
+      "learning_rate": 6.720588235294118e-07,
+      "loss": 0.0023,
+      "reward": 1.8918448686599731,
+      "reward_std": 0.018118849024176598,
+      "rewards/accuracy_reward": 0.8918446898460388,
+      "rewards/format_reward": 1.0,
+      "step": 446
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.4519271850586,
+      "epoch": 6.573529411764706,
+      "grad_norm": 1.3515959803768587,
+      "kl": 0.05908203125,
+      "learning_rate": 6.713235294117646e-07,
+      "loss": 0.0024,
+      "reward": 1.8976571559906006,
+      "reward_std": 0.019342336803674698,
+      "rewards/accuracy_reward": 0.897657036781311,
+      "rewards/format_reward": 1.0,
+      "step": 447
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.30889892578125,
+      "epoch": 6.588235294117647,
+      "grad_norm": 2.000337423528591,
+      "kl": 0.0654296875,
+      "learning_rate": 6.705882352941176e-07,
+      "loss": 0.0026,
+      "reward": 1.9029864072799683,
+      "reward_std": 0.01870027370750904,
+      "rewards/accuracy_reward": 0.9029862880706787,
+      "rewards/format_reward": 1.0,
+      "step": 448
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.02764892578125,
+      "epoch": 6.602941176470588,
+      "grad_norm": 1.5831041809663697,
+      "kl": 0.05908203125,
+      "learning_rate": 6.698529411764705e-07,
+      "loss": 0.0024,
+      "reward": 1.8981215953826904,
+      "reward_std": 0.023059705272316933,
+      "rewards/accuracy_reward": 0.8993234038352966,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 449
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.4471206665039,
+      "epoch": 6.617647058823529,
+      "grad_norm": 1.7615653292492308,
+      "kl": 0.061279296875,
+      "learning_rate": 6.691176470588234e-07,
+      "loss": 0.0025,
+      "reward": 1.8908604383468628,
+      "reward_std": 0.02007007598876953,
+      "rewards/accuracy_reward": 0.890860378742218,
+      "rewards/format_reward": 1.0,
+      "step": 450
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.1346206665039,
+      "epoch": 6.632352941176471,
+      "grad_norm": 1.6994254050705504,
+      "kl": 0.06298828125,
+      "learning_rate": 6.683823529411764e-07,
+      "loss": 0.0025,
+      "reward": 1.9058598279953003,
+      "reward_std": 0.017709294334053993,
+      "rewards/accuracy_reward": 0.9058597683906555,
+      "rewards/format_reward": 1.0,
+      "step": 451
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.54688262939453,
+      "epoch": 6.647058823529412,
+      "grad_norm": 2.143476953028863,
+      "kl": 0.051513671875,
+      "learning_rate": 6.676470588235294e-07,
+      "loss": 0.0021,
+      "reward": 1.9037673473358154,
+      "reward_std": 0.01912897825241089,
+      "rewards/accuracy_reward": 0.9037670493125916,
+      "rewards/format_reward": 1.0,
+      "step": 452
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.27644348144531,
+      "epoch": 6.661764705882353,
+      "grad_norm": 2.01478887608093,
+      "kl": 0.05029296875,
+      "learning_rate": 6.669117647058822e-07,
+      "loss": 0.002,
+      "reward": 1.8817764520645142,
+      "reward_std": 0.0232061930000782,
+      "rewards/accuracy_reward": 0.8829783201217651,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 453
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.56731414794922,
+      "epoch": 6.676470588235294,
+      "grad_norm": 2.690844672468002,
+      "kl": 0.052978515625,
+      "learning_rate": 6.661764705882352e-07,
+      "loss": 0.0021,
+      "reward": 1.923115849494934,
+      "reward_std": 0.014602301642298698,
+      "rewards/accuracy_reward": 0.9231158494949341,
+      "rewards/format_reward": 1.0,
+      "step": 454
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.59495544433594,
+      "epoch": 6.6911764705882355,
+      "grad_norm": 1.7310069495348166,
+      "kl": 0.048095703125,
+      "learning_rate": 6.654411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.901757836341858,
+      "reward_std": 0.019638314843177795,
+      "rewards/accuracy_reward": 0.9017577767372131,
+      "rewards/format_reward": 1.0,
+      "step": 455
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.0877456665039,
+      "epoch": 6.705882352941177,
+      "grad_norm": 4.401458379456843,
+      "kl": 0.052978515625,
+      "learning_rate": 6.647058823529411e-07,
+      "loss": 0.0021,
+      "reward": 1.9064425230026245,
+      "reward_std": 0.018589207902550697,
+      "rewards/accuracy_reward": 0.9076444506645203,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 456
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.69231414794922,
+      "epoch": 6.720588235294118,
+      "grad_norm": 2.0195398931401867,
+      "kl": 0.04736328125,
+      "learning_rate": 6.63970588235294e-07,
+      "loss": 0.0019,
+      "reward": 1.8995909690856934,
+      "reward_std": 0.015642177313566208,
+      "rewards/accuracy_reward": 0.8995909094810486,
+      "rewards/format_reward": 1.0,
+      "step": 457
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.81731414794922,
+      "epoch": 6.735294117647059,
+      "grad_norm": 1.5989334696664628,
+      "kl": 0.05224609375,
+      "learning_rate": 6.63235294117647e-07,
+      "loss": 0.0021,
+      "reward": 1.8981314897537231,
+      "reward_std": 0.020380795001983643,
+      "rewards/accuracy_reward": 0.8981314897537231,
+      "rewards/format_reward": 1.0,
+      "step": 458
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.00841522216797,
+      "epoch": 6.75,
+      "grad_norm": 3.993916385712523,
+      "kl": 0.045166015625,
+      "learning_rate": 6.624999999999999e-07,
+      "loss": 0.0018,
+      "reward": 1.9087324142456055,
+      "reward_std": 0.018654441460967064,
+      "rewards/accuracy_reward": 0.9087323546409607,
+      "rewards/format_reward": 1.0,
+      "step": 459
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.8689956665039,
+      "epoch": 6.764705882352941,
+      "grad_norm": 5.917555869011424,
+      "kl": 0.0458984375,
+      "learning_rate": 6.617647058823529e-07,
+      "loss": 0.0018,
+      "reward": 1.9015119075775146,
+      "reward_std": 0.01625223271548748,
+      "rewards/accuracy_reward": 0.9015120267868042,
+      "rewards/format_reward": 1.0,
+      "step": 460
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.99639892578125,
+      "epoch": 6.779411764705882,
+      "grad_norm": 2.4437828911603607,
+      "kl": 0.047607421875,
+      "learning_rate": 6.610294117647059e-07,
+      "loss": 0.0019,
+      "reward": 1.8872531652450562,
+      "reward_std": 0.020600629970431328,
+      "rewards/accuracy_reward": 0.8872530460357666,
+      "rewards/format_reward": 1.0,
+      "step": 461
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.85456848144531,
+      "epoch": 6.794117647058823,
+      "grad_norm": 1.220750465871548,
+      "kl": 0.054931640625,
+      "learning_rate": 6.602941176470587e-07,
+      "loss": 0.0022,
+      "reward": 1.902260661125183,
+      "reward_std": 0.020439889281988144,
+      "rewards/accuracy_reward": 0.9034625291824341,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 462
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.22116088867188,
+      "epoch": 6.8088235294117645,
+      "grad_norm": 2.3522146924761804,
+      "kl": 0.048583984375,
+      "learning_rate": 6.595588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.8962863683700562,
+      "reward_std": 0.0212849248200655,
+      "rewards/accuracy_reward": 0.8962863087654114,
+      "rewards/format_reward": 1.0,
+      "step": 463
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.49038696289062,
+      "epoch": 6.823529411764706,
+      "grad_norm": 2.653649647480221,
+      "kl": 0.054931640625,
+      "learning_rate": 6.588235294117647e-07,
+      "loss": 0.0022,
+      "reward": 1.907329797744751,
+      "reward_std": 0.01719941571354866,
+      "rewards/accuracy_reward": 0.9073297381401062,
+      "rewards/format_reward": 1.0,
+      "step": 464
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.78606414794922,
+      "epoch": 6.838235294117647,
+      "grad_norm": 1.5113941947142104,
+      "kl": 0.07421875,
+      "learning_rate": 6.580882352941176e-07,
+      "loss": 0.003,
+      "reward": 1.905678153038025,
+      "reward_std": 0.027055159211158752,
+      "rewards/accuracy_reward": 0.9092837572097778,
+      "rewards/format_reward": 0.9963942766189575,
+      "step": 465
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.69831848144531,
+      "epoch": 6.852941176470588,
+      "grad_norm": 2.845621962915759,
+      "kl": 0.054443359375,
+      "learning_rate": 6.573529411764705e-07,
+      "loss": 0.0022,
+      "reward": 1.900158405303955,
+      "reward_std": 0.0176254715770483,
+      "rewards/accuracy_reward": 0.9001582264900208,
+      "rewards/format_reward": 1.0,
+      "step": 466
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.25721740722656,
+      "epoch": 6.867647058823529,
+      "grad_norm": 1.7461784259949287,
+      "kl": 0.060546875,
+      "learning_rate": 6.566176470588235e-07,
+      "loss": 0.0024,
+      "reward": 1.9130123853683472,
+      "reward_std": 0.018750464543700218,
+      "rewards/accuracy_reward": 0.9130123257637024,
+      "rewards/format_reward": 1.0,
+      "step": 467
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.20433044433594,
+      "epoch": 6.882352941176471,
+      "grad_norm": 2.1022701427819346,
+      "kl": 0.051025390625,
+      "learning_rate": 6.558823529411764e-07,
+      "loss": 0.002,
+      "reward": 1.9070162773132324,
+      "reward_std": 0.01999017596244812,
+      "rewards/accuracy_reward": 0.9070160388946533,
+      "rewards/format_reward": 1.0,
+      "step": 468
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.2596206665039,
+      "epoch": 6.897058823529412,
+      "grad_norm": 1.9884301824289226,
+      "kl": 0.05322265625,
+      "learning_rate": 6.551470588235294e-07,
+      "loss": 0.0021,
+      "reward": 1.8830878734588623,
+      "reward_std": 0.025470944121479988,
+      "rewards/accuracy_reward": 0.884289562702179,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 469
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.24639892578125,
+      "epoch": 6.911764705882353,
+      "grad_norm": 2.363006531050007,
+      "kl": 0.060791015625,
+      "learning_rate": 6.544117647058824e-07,
+      "loss": 0.0024,
+      "reward": 1.899465799331665,
+      "reward_std": 0.018820466473698616,
+      "rewards/accuracy_reward": 0.8994657397270203,
+      "rewards/format_reward": 1.0,
+      "step": 470
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.1189956665039,
+      "epoch": 6.926470588235294,
+      "grad_norm": 1.955011720116878,
+      "kl": 0.053466796875,
+      "learning_rate": 6.536764705882352e-07,
+      "loss": 0.0021,
+      "reward": 1.9155514240264893,
+      "reward_std": 0.017393523827195168,
+      "rewards/accuracy_reward": 0.9155513048171997,
+      "rewards/format_reward": 1.0,
+      "step": 471
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.6971206665039,
+      "epoch": 6.9411764705882355,
+      "grad_norm": 1.9269477392518999,
+      "kl": 0.050048828125,
+      "learning_rate": 6.529411764705882e-07,
+      "loss": 0.002,
+      "reward": 1.8984498977661133,
+      "reward_std": 0.019148575142025948,
+      "rewards/accuracy_reward": 0.8996517658233643,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 472
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.8221206665039,
+      "epoch": 6.955882352941177,
+      "grad_norm": 2.9581301695718514,
+      "kl": 0.0537109375,
+      "learning_rate": 6.522058823529412e-07,
+      "loss": 0.0021,
+      "reward": 1.8932065963745117,
+      "reward_std": 0.0210660882294178,
+      "rewards/accuracy_reward": 0.8932065367698669,
+      "rewards/format_reward": 1.0,
+      "step": 473
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.91947174072266,
+      "epoch": 6.970588235294118,
+      "grad_norm": 2.111290033794273,
+      "kl": 0.059326171875,
+      "learning_rate": 6.514705882352941e-07,
+      "loss": 0.0024,
+      "reward": 1.901350498199463,
+      "reward_std": 0.021692074835300446,
+      "rewards/accuracy_reward": 0.9025523662567139,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 474
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.16947174072266,
+      "epoch": 6.985294117647059,
+      "grad_norm": 2.2057304416622108,
+      "kl": 0.060791015625,
+      "learning_rate": 6.50735294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.9076955318450928,
+      "reward_std": 0.021920649334788322,
+      "rewards/accuracy_reward": 0.9088975191116333,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 475
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.17960357666016,
+      "epoch": 7.0,
+      "grad_norm": 1.617597906920766,
+      "kl": 0.056396484375,
+      "learning_rate": 6.5e-07,
+      "loss": 0.0023,
+      "reward": 1.9078389406204224,
+      "reward_std": 0.023063138127326965,
+      "rewards/accuracy_reward": 0.9078390598297119,
+      "rewards/format_reward": 1.0,
+      "step": 476
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.60216522216797,
+      "epoch": 7.014705882352941,
+      "grad_norm": 1.6817279506404212,
+      "kl": 0.0546875,
+      "learning_rate": 6.492647058823529e-07,
+      "loss": 0.0022,
+      "reward": 1.9104756116867065,
+      "reward_std": 0.020941542461514473,
+      "rewards/accuracy_reward": 0.9116773009300232,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 477
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.2752456665039,
+      "epoch": 7.029411764705882,
+      "grad_norm": 2.3781369604321836,
+      "kl": 0.052490234375,
+      "learning_rate": 6.485294117647059e-07,
+      "loss": 0.0021,
+      "reward": 1.8896749019622803,
+      "reward_std": 0.020192930474877357,
+      "rewards/accuracy_reward": 0.8896746635437012,
+      "rewards/format_reward": 1.0,
+      "step": 478
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.77163696289062,
+      "epoch": 7.044117647058823,
+      "grad_norm": 1.276439509093276,
+      "kl": 0.059326171875,
+      "learning_rate": 6.477941176470589e-07,
+      "loss": 0.0024,
+      "reward": 1.8859210014343262,
+      "reward_std": 0.022135620936751366,
+      "rewards/accuracy_reward": 0.8859207630157471,
+      "rewards/format_reward": 1.0,
+      "step": 479
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.27163696289062,
+      "epoch": 7.0588235294117645,
+      "grad_norm": 2.4358423376131815,
+      "kl": 0.047607421875,
+      "learning_rate": 6.470588235294117e-07,
+      "loss": 0.0019,
+      "reward": 1.8972910642623901,
+      "reward_std": 0.020838702097535133,
+      "rewards/accuracy_reward": 0.8972910642623901,
+      "rewards/format_reward": 1.0,
+      "step": 480
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.93991088867188,
+      "epoch": 7.073529411764706,
+      "grad_norm": 2.4539112139463777,
+      "kl": 0.05078125,
+      "learning_rate": 6.463235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.8896875381469727,
+      "reward_std": 0.0205745380371809,
+      "rewards/accuracy_reward": 0.8896873593330383,
+      "rewards/format_reward": 1.0,
+      "step": 481
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.85697174072266,
+      "epoch": 7.088235294117647,
+      "grad_norm": 1.5146047739430706,
+      "kl": 0.054931640625,
+      "learning_rate": 6.455882352941177e-07,
+      "loss": 0.0022,
+      "reward": 1.9029262065887451,
+      "reward_std": 0.020332474261522293,
+      "rewards/accuracy_reward": 0.9029260873794556,
+      "rewards/format_reward": 1.0,
+      "step": 482
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.64543914794922,
+      "epoch": 7.102941176470588,
+      "grad_norm": 1.6263090300209622,
+      "kl": 0.049560546875,
+      "learning_rate": 6.448529411764706e-07,
+      "loss": 0.002,
+      "reward": 1.9050016403198242,
+      "reward_std": 0.020645473152399063,
+      "rewards/accuracy_reward": 0.9050015807151794,
+      "rewards/format_reward": 1.0,
+      "step": 483
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.32452392578125,
+      "epoch": 7.117647058823529,
+      "grad_norm": 2.386969995018498,
+      "kl": 0.04833984375,
+      "learning_rate": 6.441176470588235e-07,
+      "loss": 0.0019,
+      "reward": 1.8980265855789185,
+      "reward_std": 0.02636340633034706,
+      "rewards/accuracy_reward": 0.9016323685646057,
+      "rewards/format_reward": 0.9963942766189575,
+      "step": 484
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.62741088867188,
+      "epoch": 7.132352941176471,
+      "grad_norm": 2.488441055097933,
+      "kl": 0.060302734375,
+      "learning_rate": 6.433823529411765e-07,
+      "loss": 0.0024,
+      "reward": 1.8940205574035645,
+      "reward_std": 0.021218711510300636,
+      "rewards/accuracy_reward": 0.8940205574035645,
+      "rewards/format_reward": 1.0,
+      "step": 485
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.08173370361328,
+      "epoch": 7.147058823529412,
+      "grad_norm": 1.4885294609323232,
+      "kl": 0.055419921875,
+      "learning_rate": 6.426470588235294e-07,
+      "loss": 0.0022,
+      "reward": 1.9081767797470093,
+      "reward_std": 0.021031972020864487,
+      "rewards/accuracy_reward": 0.9093785881996155,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 486
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.18029022216797,
+      "epoch": 7.161764705882353,
+      "grad_norm": 2.3558483279167257,
+      "kl": 0.050048828125,
+      "learning_rate": 6.419117647058824e-07,
+      "loss": 0.002,
+      "reward": 1.9089596271514893,
+      "reward_std": 0.021060261875391006,
+      "rewards/accuracy_reward": 0.910161554813385,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 487
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.5300521850586,
+      "epoch": 7.176470588235294,
+      "grad_norm": 2.185868565772857,
+      "kl": 0.051513671875,
+      "learning_rate": 6.411764705882354e-07,
+      "loss": 0.002,
+      "reward": 1.895540475845337,
+      "reward_std": 0.02588125877082348,
+      "rewards/accuracy_reward": 0.8991460204124451,
+      "rewards/format_reward": 0.9963942766189575,
+      "step": 488
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.70313262939453,
+      "epoch": 7.1911764705882355,
+      "grad_norm": 13.781167768252143,
+      "kl": 0.0556640625,
+      "learning_rate": 6.404411764705881e-07,
+      "loss": 0.0022,
+      "reward": 1.900436520576477,
+      "reward_std": 0.018973568454384804,
+      "rewards/accuracy_reward": 0.9004363417625427,
+      "rewards/format_reward": 1.0,
+      "step": 489
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.69231414794922,
+      "epoch": 7.205882352941177,
+      "grad_norm": 1.787139009281532,
+      "kl": 0.055908203125,
+      "learning_rate": 6.397058823529411e-07,
+      "loss": 0.0022,
+      "reward": 1.914228916168213,
+      "reward_std": 0.025157246738672256,
+      "rewards/accuracy_reward": 0.9166327714920044,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 490
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.53606414794922,
+      "epoch": 7.220588235294118,
+      "grad_norm": 1.3383135392600143,
+      "kl": 0.05322265625,
+      "learning_rate": 6.389705882352941e-07,
+      "loss": 0.0021,
+      "reward": 1.9147520065307617,
+      "reward_std": 0.01828881911933422,
+      "rewards/accuracy_reward": 0.9147518277168274,
+      "rewards/format_reward": 1.0,
+      "step": 491
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.80769348144531,
+      "epoch": 7.235294117647059,
+      "grad_norm": 1.499587887382779,
+      "kl": 0.049560546875,
+      "learning_rate": 6.38235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.9006352424621582,
+      "reward_std": 0.021384483203291893,
+      "rewards/accuracy_reward": 0.901837170124054,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 492
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.85216522216797,
+      "epoch": 7.25,
+      "grad_norm": 9.51747929867123,
+      "kl": 0.052978515625,
+      "learning_rate": 6.374999999999999e-07,
+      "loss": 0.0021,
+      "reward": 1.9037039279937744,
+      "reward_std": 0.017083317041397095,
+      "rewards/accuracy_reward": 0.9037038683891296,
+      "rewards/format_reward": 1.0,
+      "step": 493
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.1863021850586,
+      "epoch": 7.264705882352941,
+      "grad_norm": 2.190318540866678,
+      "kl": 0.052978515625,
+      "learning_rate": 6.367647058823529e-07,
+      "loss": 0.0021,
+      "reward": 1.9090691804885864,
+      "reward_std": 0.016836341470479965,
+      "rewards/accuracy_reward": 0.9090691804885864,
+      "rewards/format_reward": 1.0,
+      "step": 494
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.6658706665039,
+      "epoch": 7.279411764705882,
+      "grad_norm": 16.171603319545223,
+      "kl": 0.0576171875,
+      "learning_rate": 6.360294117647058e-07,
+      "loss": 0.0023,
+      "reward": 1.8776507377624512,
+      "reward_std": 0.018403498455882072,
+      "rewards/accuracy_reward": 0.8776505589485168,
+      "rewards/format_reward": 1.0,
+      "step": 495
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.74519348144531,
+      "epoch": 7.294117647058823,
+      "grad_norm": 1.9202223901107958,
+      "kl": 0.054931640625,
+      "learning_rate": 6.352941176470588e-07,
+      "loss": 0.0022,
+      "reward": 1.883994698524475,
+      "reward_std": 0.018577104434370995,
+      "rewards/accuracy_reward": 0.8839945197105408,
+      "rewards/format_reward": 1.0,
+      "step": 496
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.7752456665039,
+      "epoch": 7.3088235294117645,
+      "grad_norm": 2.7194904114682545,
+      "kl": 0.05908203125,
+      "learning_rate": 6.345588235294117e-07,
+      "loss": 0.0024,
+      "reward": 1.9002341032028198,
+      "reward_std": 0.02195299230515957,
+      "rewards/accuracy_reward": 0.9026379585266113,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 497
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.41106414794922,
+      "epoch": 7.323529411764706,
+      "grad_norm": 2.818503081927554,
+      "kl": 0.0546875,
+      "learning_rate": 6.338235294117646e-07,
+      "loss": 0.0022,
+      "reward": 1.9031240940093994,
+      "reward_std": 0.021893031895160675,
+      "rewards/accuracy_reward": 0.9055278897285461,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 498
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.1658706665039,
+      "epoch": 7.338235294117647,
+      "grad_norm": 2.2904375345608683,
+      "kl": 0.054443359375,
+      "learning_rate": 6.330882352941176e-07,
+      "loss": 0.0022,
+      "reward": 1.8997858762741089,
+      "reward_std": 0.01683872379362583,
+      "rewards/accuracy_reward": 0.8997858762741089,
+      "rewards/format_reward": 1.0,
+      "step": 499
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.5877456665039,
+      "epoch": 7.352941176470588,
+      "grad_norm": 1.9874226919695666,
+      "kl": 0.05712890625,
+      "learning_rate": 6.323529411764706e-07,
+      "loss": 0.0023,
+      "reward": 1.9213850498199463,
+      "reward_std": 0.016001444309949875,
+      "rewards/accuracy_reward": 0.9213849306106567,
+      "rewards/format_reward": 1.0,
+      "step": 500
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.45433044433594,
+      "epoch": 7.367647058823529,
+      "grad_norm": 1.4810061564924804,
+      "kl": 0.06005859375,
+      "learning_rate": 6.316176470588234e-07,
+      "loss": 0.0024,
+      "reward": 1.9132663011550903,
+      "reward_std": 0.016456911340355873,
+      "rewards/accuracy_reward": 0.9132663011550903,
+      "rewards/format_reward": 1.0,
+      "step": 501
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.87259674072266,
+      "epoch": 7.382352941176471,
+      "grad_norm": 2.354230850080614,
+      "kl": 0.0537109375,
+      "learning_rate": 6.308823529411764e-07,
+      "loss": 0.0021,
+      "reward": 1.9061925411224365,
+      "reward_std": 0.015117881819605827,
+      "rewards/accuracy_reward": 0.906192421913147,
+      "rewards/format_reward": 1.0,
+      "step": 502
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.88221740722656,
+      "epoch": 7.397058823529412,
+      "grad_norm": 2.702198636610281,
+      "kl": 0.055908203125,
+      "learning_rate": 6.301470588235294e-07,
+      "loss": 0.0023,
+      "reward": 1.8956823348999023,
+      "reward_std": 0.017957165837287903,
+      "rewards/accuracy_reward": 0.8956822752952576,
+      "rewards/format_reward": 1.0,
+      "step": 503
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.91827392578125,
+      "epoch": 7.411764705882353,
+      "grad_norm": 2.707910090355551,
+      "kl": 0.05517578125,
+      "learning_rate": 6.294117647058823e-07,
+      "loss": 0.0022,
+      "reward": 1.8958656787872314,
+      "reward_std": 0.01692720130085945,
+      "rewards/accuracy_reward": 0.8958656787872314,
+      "rewards/format_reward": 1.0,
+      "step": 504
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.41106414794922,
+      "epoch": 7.426470588235294,
+      "grad_norm": 2.732018255754022,
+      "kl": 0.06884765625,
+      "learning_rate": 6.286764705882353e-07,
+      "loss": 0.0028,
+      "reward": 1.907556176185608,
+      "reward_std": 0.016339732334017754,
+      "rewards/accuracy_reward": 0.9075562953948975,
+      "rewards/format_reward": 1.0,
+      "step": 505
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.5252456665039,
+      "epoch": 7.4411764705882355,
+      "grad_norm": 1.9561974612826654,
+      "kl": 0.061279296875,
+      "learning_rate": 6.279411764705882e-07,
+      "loss": 0.0025,
+      "reward": 1.9172346591949463,
+      "reward_std": 0.017858320847153664,
+      "rewards/accuracy_reward": 0.9184365272521973,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 506
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.40745544433594,
+      "epoch": 7.455882352941177,
+      "grad_norm": 4.587975830736582,
+      "kl": 0.07080078125,
+      "learning_rate": 6.272058823529411e-07,
+      "loss": 0.0028,
+      "reward": 1.9027947187423706,
+      "reward_std": 0.014803332276642323,
+      "rewards/accuracy_reward": 0.9027946591377258,
+      "rewards/format_reward": 1.0,
+      "step": 507
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.92909240722656,
+      "epoch": 7.470588235294118,
+      "grad_norm": 3.0606242226177085,
+      "kl": 0.05859375,
+      "learning_rate": 6.264705882352941e-07,
+      "loss": 0.0023,
+      "reward": 1.9170668125152588,
+      "reward_std": 0.01804046705365181,
+      "rewards/accuracy_reward": 0.9182686805725098,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 508
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.75000762939453,
+      "epoch": 7.485294117647059,
+      "grad_norm": 1.565766574508637,
+      "kl": 0.06005859375,
+      "learning_rate": 6.257352941176471e-07,
+      "loss": 0.0024,
+      "reward": 1.9129410982131958,
+      "reward_std": 0.016586054116487503,
+      "rewards/accuracy_reward": 0.914142906665802,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 509
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.11659240722656,
+      "epoch": 7.5,
+      "grad_norm": 2.188031838473879,
+      "kl": 0.06494140625,
+      "learning_rate": 6.249999999999999e-07,
+      "loss": 0.0026,
+      "reward": 1.9109857082366943,
+      "reward_std": 0.01680620200932026,
+      "rewards/accuracy_reward": 0.9109857082366943,
+      "rewards/format_reward": 1.0,
+      "step": 510
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.4206771850586,
+      "epoch": 7.514705882352941,
+      "grad_norm": 2.301709034695236,
+      "kl": 0.05419921875,
+      "learning_rate": 6.242647058823529e-07,
+      "loss": 0.0022,
+      "reward": 1.8962676525115967,
+      "reward_std": 0.01672891154885292,
+      "rewards/accuracy_reward": 0.8962674736976624,
+      "rewards/format_reward": 1.0,
+      "step": 511
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.53726196289062,
+      "epoch": 7.529411764705882,
+      "grad_norm": 2.3427553949456135,
+      "kl": 0.057373046875,
+      "learning_rate": 6.235294117647059e-07,
+      "loss": 0.0023,
+      "reward": 1.9040287733078003,
+      "reward_std": 0.017709048464894295,
+      "rewards/accuracy_reward": 0.9040287137031555,
+      "rewards/format_reward": 1.0,
+      "step": 512
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.12019348144531,
+      "epoch": 7.544117647058823,
+      "grad_norm": 1.665426870174403,
+      "kl": 0.0537109375,
+      "learning_rate": 6.227941176470588e-07,
+      "loss": 0.0022,
+      "reward": 1.88170325756073,
+      "reward_std": 0.017501268535852432,
+      "rewards/accuracy_reward": 0.8817031979560852,
+      "rewards/format_reward": 1.0,
+      "step": 513
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.15144348144531,
+      "epoch": 7.5588235294117645,
+      "grad_norm": 1.6488965895634309,
+      "kl": 0.06591796875,
+      "learning_rate": 6.220588235294118e-07,
+      "loss": 0.0026,
+      "reward": 1.8925837278366089,
+      "reward_std": 0.015000036917626858,
+      "rewards/accuracy_reward": 0.8925836682319641,
+      "rewards/format_reward": 1.0,
+      "step": 514
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.16106414794922,
+      "epoch": 7.573529411764706,
+      "grad_norm": 1.5981543745714246,
+      "kl": 0.05517578125,
+      "learning_rate": 6.213235294117647e-07,
+      "loss": 0.0022,
+      "reward": 1.920210599899292,
+      "reward_std": 0.01567223109304905,
+      "rewards/accuracy_reward": 0.9202104806900024,
+      "rewards/format_reward": 1.0,
+      "step": 515
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.27764892578125,
+      "epoch": 7.588235294117647,
+      "grad_norm": 1.914576473194542,
+      "kl": 0.054931640625,
+      "learning_rate": 6.205882352941176e-07,
+      "loss": 0.0022,
+      "reward": 1.9109389781951904,
+      "reward_std": 0.01758970133960247,
+      "rewards/accuracy_reward": 0.9121409058570862,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 516
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.74639892578125,
+      "epoch": 7.602941176470588,
+      "grad_norm": 2.2056419921026444,
+      "kl": 0.06591796875,
+      "learning_rate": 6.198529411764706e-07,
+      "loss": 0.0026,
+      "reward": 1.909173846244812,
+      "reward_std": 0.0163656584918499,
+      "rewards/accuracy_reward": 0.909173846244812,
+      "rewards/format_reward": 1.0,
+      "step": 517
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.97956848144531,
+      "epoch": 7.617647058823529,
+      "grad_norm": 2.3280532068329505,
+      "kl": 0.0517578125,
+      "learning_rate": 6.191176470588236e-07,
+      "loss": 0.0021,
+      "reward": 1.8901021480560303,
+      "reward_std": 0.01763245277106762,
+      "rewards/accuracy_reward": 0.890101969242096,
+      "rewards/format_reward": 1.0,
+      "step": 518
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.45793914794922,
+      "epoch": 7.632352941176471,
+      "grad_norm": 1.402338629602267,
+      "kl": 0.05712890625,
+      "learning_rate": 6.183823529411764e-07,
+      "loss": 0.0023,
+      "reward": 1.9023847579956055,
+      "reward_std": 0.016608012840151787,
+      "rewards/accuracy_reward": 0.9023846387863159,
+      "rewards/format_reward": 1.0,
+      "step": 519
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.38341522216797,
+      "epoch": 7.647058823529412,
+      "grad_norm": 1.6835653034818958,
+      "kl": 0.058837890625,
+      "learning_rate": 6.176470588235294e-07,
+      "loss": 0.0024,
+      "reward": 1.8985044956207275,
+      "reward_std": 0.01695878617465496,
+      "rewards/accuracy_reward": 0.8985044360160828,
+      "rewards/format_reward": 1.0,
+      "step": 520
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.1346206665039,
+      "epoch": 7.661764705882353,
+      "grad_norm": 9.541516490873528,
+      "kl": 0.05419921875,
+      "learning_rate": 6.169117647058824e-07,
+      "loss": 0.0022,
+      "reward": 1.903184175491333,
+      "reward_std": 0.01812390610575676,
+      "rewards/accuracy_reward": 0.9031841158866882,
+      "rewards/format_reward": 1.0,
+      "step": 521
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.78125762939453,
+      "epoch": 7.676470588235294,
+      "grad_norm": 2.907795347878127,
+      "kl": 0.056396484375,
+      "learning_rate": 6.161764705882353e-07,
+      "loss": 0.0022,
+      "reward": 1.9058642387390137,
+      "reward_std": 0.022106477990746498,
+      "rewards/accuracy_reward": 0.9070661067962646,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 522
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.3425521850586,
+      "epoch": 7.6911764705882355,
+      "grad_norm": 2.122588959449984,
+      "kl": 0.047607421875,
+      "learning_rate": 6.154411764705883e-07,
+      "loss": 0.0019,
+      "reward": 1.892153024673462,
+      "reward_std": 0.017479056492447853,
+      "rewards/accuracy_reward": 0.8921528458595276,
+      "rewards/format_reward": 1.0,
+      "step": 523
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.2019271850586,
+      "epoch": 7.705882352941177,
+      "grad_norm": 2.3564295353752986,
+      "kl": 0.05615234375,
+      "learning_rate": 6.147058823529412e-07,
+      "loss": 0.0022,
+      "reward": 1.9090198278427124,
+      "reward_std": 0.01690274477005005,
+      "rewards/accuracy_reward": 0.9090198874473572,
+      "rewards/format_reward": 1.0,
+      "step": 524
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.92188262939453,
+      "epoch": 7.720588235294118,
+      "grad_norm": 1.574872602292037,
+      "kl": 0.055419921875,
+      "learning_rate": 6.139705882352941e-07,
+      "loss": 0.0022,
+      "reward": 1.901806354522705,
+      "reward_std": 0.020534634590148926,
+      "rewards/accuracy_reward": 0.9030082821846008,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 525
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.71634674072266,
+      "epoch": 7.735294117647059,
+      "grad_norm": 1.4642975468665929,
+      "kl": 0.05615234375,
+      "learning_rate": 6.132352941176471e-07,
+      "loss": 0.0022,
+      "reward": 1.8870961666107178,
+      "reward_std": 0.0190273430198431,
+      "rewards/accuracy_reward": 0.8870960474014282,
+      "rewards/format_reward": 1.0,
+      "step": 526
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.76683044433594,
+      "epoch": 7.75,
+      "grad_norm": 1.64957668506362,
+      "kl": 0.055908203125,
+      "learning_rate": 6.125000000000001e-07,
+      "loss": 0.0022,
+      "reward": 1.888306736946106,
+      "reward_std": 0.018305247649550438,
+      "rewards/accuracy_reward": 0.888306736946106,
+      "rewards/format_reward": 1.0,
+      "step": 527
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.36659240722656,
+      "epoch": 7.764705882352941,
+      "grad_norm": 1.8470414599040466,
+      "kl": 0.054443359375,
+      "learning_rate": 6.117647058823529e-07,
+      "loss": 0.0022,
+      "reward": 1.914186716079712,
+      "reward_std": 0.018072694540023804,
+      "rewards/accuracy_reward": 0.9141866564750671,
+      "rewards/format_reward": 1.0,
+      "step": 528
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.49279022216797,
+      "epoch": 7.779411764705882,
+      "grad_norm": 1.4757794429532862,
+      "kl": 0.05419921875,
+      "learning_rate": 6.110294117647059e-07,
+      "loss": 0.0022,
+      "reward": 1.8795055150985718,
+      "reward_std": 0.022909091785550117,
+      "rewards/accuracy_reward": 0.8807073831558228,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 529
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.40384674072266,
+      "epoch": 7.794117647058823,
+      "grad_norm": 2.0338636064103333,
+      "kl": 0.059326171875,
+      "learning_rate": 6.102941176470589e-07,
+      "loss": 0.0024,
+      "reward": 1.9119820594787598,
+      "reward_std": 0.01673622988164425,
+      "rewards/accuracy_reward": 0.9119820594787598,
+      "rewards/format_reward": 1.0,
+      "step": 530
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.04808044433594,
+      "epoch": 7.8088235294117645,
+      "grad_norm": 1.5646667587468694,
+      "kl": 0.0498046875,
+      "learning_rate": 6.095588235294118e-07,
+      "loss": 0.002,
+      "reward": 1.9045634269714355,
+      "reward_std": 0.019505947828292847,
+      "rewards/accuracy_reward": 0.9045633673667908,
+      "rewards/format_reward": 1.0,
+      "step": 531
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.84495544433594,
+      "epoch": 7.823529411764706,
+      "grad_norm": 1.9910418315918015,
+      "kl": 0.04736328125,
+      "learning_rate": 6.088235294117646e-07,
+      "loss": 0.0019,
+      "reward": 1.9242442846298218,
+      "reward_std": 0.015269236639142036,
+      "rewards/accuracy_reward": 0.9242441058158875,
+      "rewards/format_reward": 1.0,
+      "step": 532
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.67909240722656,
+      "epoch": 7.838235294117647,
+      "grad_norm": 2.1921636670867213,
+      "kl": 0.05126953125,
+      "learning_rate": 6.080882352941175e-07,
+      "loss": 0.0021,
+      "reward": 1.9174097776412964,
+      "reward_std": 0.016234705224633217,
+      "rewards/accuracy_reward": 0.9174097776412964,
+      "rewards/format_reward": 1.0,
+      "step": 533
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.95433044433594,
+      "epoch": 7.852941176470588,
+      "grad_norm": 5.773358773291786,
+      "kl": 0.052734375,
+      "learning_rate": 6.073529411764705e-07,
+      "loss": 0.0021,
+      "reward": 1.904146671295166,
+      "reward_std": 0.018309401348233223,
+      "rewards/accuracy_reward": 0.904146671295166,
+      "rewards/format_reward": 1.0,
+      "step": 534
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.10216522216797,
+      "epoch": 7.867647058823529,
+      "grad_norm": 3.1463228619890646,
+      "kl": 0.0498046875,
+      "learning_rate": 6.066176470588235e-07,
+      "loss": 0.002,
+      "reward": 1.9100762605667114,
+      "reward_std": 0.0184779055416584,
+      "rewards/accuracy_reward": 0.9100762009620667,
+      "rewards/format_reward": 1.0,
+      "step": 535
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.77644348144531,
+      "epoch": 7.882352941176471,
+      "grad_norm": 4.843243565505217,
+      "kl": 0.04736328125,
+      "learning_rate": 6.058823529411763e-07,
+      "loss": 0.0019,
+      "reward": 1.9202395677566528,
+      "reward_std": 0.0161683801561594,
+      "rewards/accuracy_reward": 0.9202395677566528,
+      "rewards/format_reward": 1.0,
+      "step": 536
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.99159240722656,
+      "epoch": 7.897058823529412,
+      "grad_norm": 1.5367043322463625,
+      "kl": 0.0546875,
+      "learning_rate": 6.051470588235293e-07,
+      "loss": 0.0022,
+      "reward": 1.9017425775527954,
+      "reward_std": 0.017361411824822426,
+      "rewards/accuracy_reward": 0.901742696762085,
+      "rewards/format_reward": 1.0,
+      "step": 537
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.72716522216797,
+      "epoch": 7.911764705882353,
+      "grad_norm": 1.6647667695224042,
+      "kl": 0.04736328125,
+      "learning_rate": 6.044117647058823e-07,
+      "loss": 0.0019,
+      "reward": 1.9160679578781128,
+      "reward_std": 0.01760725863277912,
+      "rewards/accuracy_reward": 0.9160678386688232,
+      "rewards/format_reward": 1.0,
+      "step": 538
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.06971740722656,
+      "epoch": 7.926470588235294,
+      "grad_norm": 1.5196339741547849,
+      "kl": 0.04931640625,
+      "learning_rate": 6.036764705882352e-07,
+      "loss": 0.002,
+      "reward": 1.9151060581207275,
+      "reward_std": 0.017051585018634796,
+      "rewards/accuracy_reward": 0.9151060581207275,
+      "rewards/format_reward": 1.0,
+      "step": 539
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.31971740722656,
+      "epoch": 7.9411764705882355,
+      "grad_norm": 2.094866774315946,
+      "kl": 0.04736328125,
+      "learning_rate": 6.029411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.8982237577438354,
+      "reward_std": 0.017682263627648354,
+      "rewards/accuracy_reward": 0.8982236981391907,
+      "rewards/format_reward": 1.0,
+      "step": 540
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.39543914794922,
+      "epoch": 7.955882352941177,
+      "grad_norm": 3.260878136220598,
+      "kl": 0.04638671875,
+      "learning_rate": 6.022058823529411e-07,
+      "loss": 0.0019,
+      "reward": 1.8989781141281128,
+      "reward_std": 0.016665568575263023,
+      "rewards/accuracy_reward": 0.8989779949188232,
+      "rewards/format_reward": 1.0,
+      "step": 541
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.65625762939453,
+      "epoch": 7.970588235294118,
+      "grad_norm": 2.6615474581638856,
+      "kl": 0.05126953125,
+      "learning_rate": 6.01470588235294e-07,
+      "loss": 0.002,
+      "reward": 1.9191153049468994,
+      "reward_std": 0.016191896051168442,
+      "rewards/accuracy_reward": 0.9191152453422546,
+      "rewards/format_reward": 1.0,
+      "step": 542
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.72116088867188,
+      "epoch": 7.985294117647059,
+      "grad_norm": 2.0899244271104815,
+      "kl": 0.05322265625,
+      "learning_rate": 6.00735294117647e-07,
+      "loss": 0.0021,
+      "reward": 1.9136680364608765,
+      "reward_std": 0.017099877819418907,
+      "rewards/accuracy_reward": 0.9136679768562317,
+      "rewards/format_reward": 1.0,
+      "step": 543
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.3500747680664,
+      "epoch": 8.0,
+      "grad_norm": 3.3285479491467647,
+      "kl": 0.050048828125,
+      "learning_rate": 6e-07,
+      "loss": 0.002,
+      "reward": 1.921190857887268,
+      "reward_std": 0.019811777397990227,
+      "rewards/accuracy_reward": 0.9211909174919128,
+      "rewards/format_reward": 1.0,
+      "step": 544
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.65625762939453,
+      "epoch": 8.014705882352942,
+      "grad_norm": 2.8131573068708047,
+      "kl": 0.0478515625,
+      "learning_rate": 5.992647058823528e-07,
+      "loss": 0.0019,
+      "reward": 1.9018555879592896,
+      "reward_std": 0.017298076301813126,
+      "rewards/accuracy_reward": 0.9018555879592896,
+      "rewards/format_reward": 1.0,
+      "step": 545
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.14423370361328,
+      "epoch": 8.029411764705882,
+      "grad_norm": 1.7926326341266048,
+      "kl": 0.048583984375,
+      "learning_rate": 5.985294117647058e-07,
+      "loss": 0.0019,
+      "reward": 1.902543544769287,
+      "reward_std": 0.018634334206581116,
+      "rewards/accuracy_reward": 0.9025435447692871,
+      "rewards/format_reward": 1.0,
+      "step": 546
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.56370544433594,
+      "epoch": 8.044117647058824,
+      "grad_norm": 1.4449179097965148,
+      "kl": 0.048583984375,
+      "learning_rate": 5.977941176470588e-07,
+      "loss": 0.0019,
+      "reward": 1.916272759437561,
+      "reward_std": 0.01657208427786827,
+      "rewards/accuracy_reward": 0.916272759437561,
+      "rewards/format_reward": 1.0,
+      "step": 547
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.97356414794922,
+      "epoch": 8.058823529411764,
+      "grad_norm": 6.756394605908286,
+      "kl": 0.05126953125,
+      "learning_rate": 5.970588235294117e-07,
+      "loss": 0.0021,
+      "reward": 1.9202964305877686,
+      "reward_std": 0.018207618966698647,
+      "rewards/accuracy_reward": 0.920296311378479,
+      "rewards/format_reward": 1.0,
+      "step": 548
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.40745544433594,
+      "epoch": 8.073529411764707,
+      "grad_norm": 3.5136091347598986,
+      "kl": 0.050537109375,
+      "learning_rate": 5.963235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.9019232988357544,
+      "reward_std": 0.0198956411331892,
+      "rewards/accuracy_reward": 0.9019232392311096,
+      "rewards/format_reward": 1.0,
+      "step": 549
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.43991088867188,
+      "epoch": 8.088235294117647,
+      "grad_norm": 2.2839304418519792,
+      "kl": 0.052001953125,
+      "learning_rate": 5.955882352941176e-07,
+      "loss": 0.0021,
+      "reward": 1.9206691980361938,
+      "reward_std": 0.014682717621326447,
+      "rewards/accuracy_reward": 0.9206691384315491,
+      "rewards/format_reward": 1.0,
+      "step": 550
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.02043914794922,
+      "epoch": 8.102941176470589,
+      "grad_norm": 2.484567856847274,
+      "kl": 0.055419921875,
+      "learning_rate": 5.948529411764705e-07,
+      "loss": 0.0022,
+      "reward": 1.9100967645645142,
+      "reward_std": 0.018312010914087296,
+      "rewards/accuracy_reward": 0.9100965857505798,
+      "rewards/format_reward": 1.0,
+      "step": 551
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.34375762939453,
+      "epoch": 8.117647058823529,
+      "grad_norm": 1.6432899488733537,
+      "kl": 0.05419921875,
+      "learning_rate": 5.941176470588235e-07,
+      "loss": 0.0022,
+      "reward": 1.8991518020629883,
+      "reward_std": 0.021620534360408783,
+      "rewards/accuracy_reward": 0.900353729724884,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 552
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.63581848144531,
+      "epoch": 8.132352941176471,
+      "grad_norm": 2.0271204887130416,
+      "kl": 0.05712890625,
+      "learning_rate": 5.933823529411765e-07,
+      "loss": 0.0023,
+      "reward": 1.900107502937317,
+      "reward_std": 0.019040938466787338,
+      "rewards/accuracy_reward": 0.9001075029373169,
+      "rewards/format_reward": 1.0,
+      "step": 553
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.09616088867188,
+      "epoch": 8.147058823529411,
+      "grad_norm": 1.5813240400193374,
+      "kl": 0.060791015625,
+      "learning_rate": 5.926470588235293e-07,
+      "loss": 0.0024,
+      "reward": 1.915557622909546,
+      "reward_std": 0.020517727360129356,
+      "rewards/accuracy_reward": 0.9167595505714417,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 554
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.71394348144531,
+      "epoch": 8.161764705882353,
+      "grad_norm": 4.637148825036712,
+      "kl": 0.0576171875,
+      "learning_rate": 5.919117647058823e-07,
+      "loss": 0.0023,
+      "reward": 1.9027869701385498,
+      "reward_std": 0.019659575074911118,
+      "rewards/accuracy_reward": 0.9027867317199707,
+      "rewards/format_reward": 1.0,
+      "step": 555
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.27404022216797,
+      "epoch": 8.176470588235293,
+      "grad_norm": 1.6509059104235309,
+      "kl": 0.0556640625,
+      "learning_rate": 5.911764705882353e-07,
+      "loss": 0.0022,
+      "reward": 1.898353934288025,
+      "reward_std": 0.019414007663726807,
+      "rewards/accuracy_reward": 0.8983538746833801,
+      "rewards/format_reward": 1.0,
+      "step": 556
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.88101196289062,
+      "epoch": 8.191176470588236,
+      "grad_norm": 1.2731867179847607,
+      "kl": 0.0576171875,
+      "learning_rate": 5.904411764705882e-07,
+      "loss": 0.0023,
+      "reward": 1.8892360925674438,
+      "reward_std": 0.025776617228984833,
+      "rewards/accuracy_reward": 0.8916398882865906,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 557
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.42188262939453,
+      "epoch": 8.205882352941176,
+      "grad_norm": 2.6655315489926967,
+      "kl": 0.059326171875,
+      "learning_rate": 5.897058823529412e-07,
+      "loss": 0.0024,
+      "reward": 1.9022037982940674,
+      "reward_std": 0.01893787272274494,
+      "rewards/accuracy_reward": 0.9034057855606079,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 558
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.95793914794922,
+      "epoch": 8.220588235294118,
+      "grad_norm": 1.7443518844175923,
+      "kl": 0.053955078125,
+      "learning_rate": 5.889705882352941e-07,
+      "loss": 0.0022,
+      "reward": 1.9148838520050049,
+      "reward_std": 0.016492504626512527,
+      "rewards/accuracy_reward": 0.9148839116096497,
+      "rewards/format_reward": 1.0,
+      "step": 559
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.52043914794922,
+      "epoch": 8.235294117647058,
+      "grad_norm": 2.0265041434888853,
+      "kl": 0.050048828125,
+      "learning_rate": 5.88235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.90397047996521,
+      "reward_std": 0.020762760192155838,
+      "rewards/accuracy_reward": 0.90397047996521,
+      "rewards/format_reward": 1.0,
+      "step": 560
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.28966522216797,
+      "epoch": 8.25,
+      "grad_norm": 1.5715106635469254,
+      "kl": 0.055419921875,
+      "learning_rate": 5.875e-07,
+      "loss": 0.0022,
+      "reward": 1.911298394203186,
+      "reward_std": 0.019145453348755836,
+      "rewards/accuracy_reward": 0.9112983345985413,
+      "rewards/format_reward": 1.0,
+      "step": 561
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.29808044433594,
+      "epoch": 8.264705882352942,
+      "grad_norm": 1.4932683274744363,
+      "kl": 0.057373046875,
+      "learning_rate": 5.86764705882353e-07,
+      "loss": 0.0023,
+      "reward": 1.9099814891815186,
+      "reward_std": 0.025356175377964973,
+      "rewards/accuracy_reward": 0.9123852252960205,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 562
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.23197174072266,
+      "epoch": 8.279411764705882,
+      "grad_norm": 7.280740226777266,
+      "kl": 0.055419921875,
+      "learning_rate": 5.860294117647058e-07,
+      "loss": 0.0022,
+      "reward": 1.9085097312927246,
+      "reward_std": 0.018148187547922134,
+      "rewards/accuracy_reward": 0.9085096716880798,
+      "rewards/format_reward": 1.0,
+      "step": 563
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.80048370361328,
+      "epoch": 8.294117647058824,
+      "grad_norm": 2.9076060047905163,
+      "kl": 0.05419921875,
+      "learning_rate": 5.852941176470588e-07,
+      "loss": 0.0022,
+      "reward": 1.909620761871338,
+      "reward_std": 0.018346594646573067,
+      "rewards/accuracy_reward": 0.9096207618713379,
+      "rewards/format_reward": 1.0,
+      "step": 564
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.26081848144531,
+      "epoch": 8.308823529411764,
+      "grad_norm": 3.2289837306279168,
+      "kl": 0.060791015625,
+      "learning_rate": 5.845588235294118e-07,
+      "loss": 0.0024,
+      "reward": 1.9069983959197998,
+      "reward_std": 0.0201736968010664,
+      "rewards/accuracy_reward": 0.9082001447677612,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 565
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.25120544433594,
+      "epoch": 8.323529411764707,
+      "grad_norm": 1.7199349309033736,
+      "kl": 0.05517578125,
+      "learning_rate": 5.838235294117647e-07,
+      "loss": 0.0022,
+      "reward": 1.887500286102295,
+      "reward_std": 0.020135780796408653,
+      "rewards/accuracy_reward": 0.8887021541595459,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 566
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.41226196289062,
+      "epoch": 8.338235294117647,
+      "grad_norm": 6.802575033519044,
+      "kl": 0.0576171875,
+      "learning_rate": 5.830882352941177e-07,
+      "loss": 0.0023,
+      "reward": 1.8974536657333374,
+      "reward_std": 0.020134514197707176,
+      "rewards/accuracy_reward": 0.8974535465240479,
+      "rewards/format_reward": 1.0,
+      "step": 567
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.51683044433594,
+      "epoch": 8.352941176470589,
+      "grad_norm": 36.18405306346145,
+      "kl": 0.054931640625,
+      "learning_rate": 5.823529411764706e-07,
+      "loss": 0.0022,
+      "reward": 1.8917019367218018,
+      "reward_std": 0.01857663132250309,
+      "rewards/accuracy_reward": 0.8929038643836975,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 568
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.5769271850586,
+      "epoch": 8.367647058823529,
+      "grad_norm": 2.3459848810177975,
+      "kl": 0.052734375,
+      "learning_rate": 5.816176470588235e-07,
+      "loss": 0.0021,
+      "reward": 1.9242991209030151,
+      "reward_std": 0.018164673820137978,
+      "rewards/accuracy_reward": 0.9255009293556213,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 569
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.03485870361328,
+      "epoch": 8.382352941176471,
+      "grad_norm": 1.6032225663505273,
+      "kl": 0.06103515625,
+      "learning_rate": 5.808823529411765e-07,
+      "loss": 0.0025,
+      "reward": 1.9081476926803589,
+      "reward_std": 0.019294267520308495,
+      "rewards/accuracy_reward": 0.9081474542617798,
+      "rewards/format_reward": 1.0,
+      "step": 570
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.40866088867188,
+      "epoch": 8.397058823529411,
+      "grad_norm": 2.653883841862032,
+      "kl": 0.05908203125,
+      "learning_rate": 5.801470588235295e-07,
+      "loss": 0.0024,
+      "reward": 1.904270052909851,
+      "reward_std": 0.017766019329428673,
+      "rewards/accuracy_reward": 0.9042699933052063,
+      "rewards/format_reward": 1.0,
+      "step": 571
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.58173370361328,
+      "epoch": 8.411764705882353,
+      "grad_norm": 1.8098644267187556,
+      "kl": 0.05908203125,
+      "learning_rate": 5.794117647058823e-07,
+      "loss": 0.0024,
+      "reward": 1.9107073545455933,
+      "reward_std": 0.017658641561865807,
+      "rewards/accuracy_reward": 0.9107073545455933,
+      "rewards/format_reward": 1.0,
+      "step": 572
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.56250762939453,
+      "epoch": 8.426470588235293,
+      "grad_norm": 3.2259185749098784,
+      "kl": 0.06005859375,
+      "learning_rate": 5.786764705882353e-07,
+      "loss": 0.0024,
+      "reward": 1.899427056312561,
+      "reward_std": 0.016428910195827484,
+      "rewards/accuracy_reward": 0.8994268774986267,
+      "rewards/format_reward": 1.0,
+      "step": 573
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.02764892578125,
+      "epoch": 8.441176470588236,
+      "grad_norm": 3.282308261215477,
+      "kl": 0.05859375,
+      "learning_rate": 5.779411764705882e-07,
+      "loss": 0.0023,
+      "reward": 1.9036839008331299,
+      "reward_std": 0.020171254873275757,
+      "rewards/accuracy_reward": 0.9036839008331299,
+      "rewards/format_reward": 1.0,
+      "step": 574
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.05048370361328,
+      "epoch": 8.455882352941176,
+      "grad_norm": 1.7713296358743136,
+      "kl": 0.061767578125,
+      "learning_rate": 5.772058823529411e-07,
+      "loss": 0.0025,
+      "reward": 1.9190058708190918,
+      "reward_std": 0.018266772851347923,
+      "rewards/accuracy_reward": 0.919005811214447,
+      "rewards/format_reward": 1.0,
+      "step": 575
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.00241088867188,
+      "epoch": 8.470588235294118,
+      "grad_norm": 1.9874750258288356,
+      "kl": 0.055908203125,
+      "learning_rate": 5.76470588235294e-07,
+      "loss": 0.0023,
+      "reward": 1.9230592250823975,
+      "reward_std": 0.013289669528603554,
+      "rewards/accuracy_reward": 0.9230592250823975,
+      "rewards/format_reward": 1.0,
+      "step": 576
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.89063262939453,
+      "epoch": 8.485294117647058,
+      "grad_norm": 2.286169294844814,
+      "kl": 0.0576171875,
+      "learning_rate": 5.75735294117647e-07,
+      "loss": 0.0023,
+      "reward": 1.8973294496536255,
+      "reward_std": 0.01872727833688259,
+      "rewards/accuracy_reward": 0.8973293900489807,
+      "rewards/format_reward": 1.0,
+      "step": 577
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.49279022216797,
+      "epoch": 8.5,
+      "grad_norm": 2.4071803200242416,
+      "kl": 0.0654296875,
+      "learning_rate": 5.749999999999999e-07,
+      "loss": 0.0026,
+      "reward": 1.9316534996032715,
+      "reward_std": 0.017379358410835266,
+      "rewards/accuracy_reward": 0.9316534399986267,
+      "rewards/format_reward": 1.0,
+      "step": 578
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.60216522216797,
+      "epoch": 8.514705882352942,
+      "grad_norm": 1.79762667011906,
+      "kl": 0.064453125,
+      "learning_rate": 5.742647058823529e-07,
+      "loss": 0.0026,
+      "reward": 1.9131649732589722,
+      "reward_std": 0.020887164399027824,
+      "rewards/accuracy_reward": 0.9143666625022888,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 579
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.56971740722656,
+      "epoch": 8.529411764705882,
+      "grad_norm": 2.3602888807544526,
+      "kl": 0.053955078125,
+      "learning_rate": 5.735294117647059e-07,
+      "loss": 0.0022,
+      "reward": 1.9001418352127075,
+      "reward_std": 0.01754673384130001,
+      "rewards/accuracy_reward": 0.9001417756080627,
+      "rewards/format_reward": 1.0,
+      "step": 580
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.32091522216797,
+      "epoch": 8.544117647058824,
+      "grad_norm": 1.8304863518291064,
+      "kl": 0.05615234375,
+      "learning_rate": 5.727941176470587e-07,
+      "loss": 0.0022,
+      "reward": 1.9000111818313599,
+      "reward_std": 0.017091548070311546,
+      "rewards/accuracy_reward": 0.9000111222267151,
+      "rewards/format_reward": 1.0,
+      "step": 581
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.98558044433594,
+      "epoch": 8.558823529411764,
+      "grad_norm": 2.667131501264164,
+      "kl": 0.0517578125,
+      "learning_rate": 5.720588235294117e-07,
+      "loss": 0.0021,
+      "reward": 1.9127933979034424,
+      "reward_std": 0.01638048328459263,
+      "rewards/accuracy_reward": 0.9127933979034424,
+      "rewards/format_reward": 1.0,
+      "step": 582
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.97116088867188,
+      "epoch": 8.573529411764707,
+      "grad_norm": 3.838255993972307,
+      "kl": 0.05322265625,
+      "learning_rate": 5.713235294117647e-07,
+      "loss": 0.0021,
+      "reward": 1.9052388668060303,
+      "reward_std": 0.015317871235311031,
+      "rewards/accuracy_reward": 0.9052388668060303,
+      "rewards/format_reward": 1.0,
+      "step": 583
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.7800521850586,
+      "epoch": 8.588235294117647,
+      "grad_norm": 1.8912826875619027,
+      "kl": 0.0517578125,
+      "learning_rate": 5.705882352941176e-07,
+      "loss": 0.0021,
+      "reward": 1.9118890762329102,
+      "reward_std": 0.016504064202308655,
+      "rewards/accuracy_reward": 0.9118890166282654,
+      "rewards/format_reward": 1.0,
+      "step": 584
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.36538696289062,
+      "epoch": 8.602941176470589,
+      "grad_norm": 1.9564532336203913,
+      "kl": 0.05517578125,
+      "learning_rate": 5.698529411764705e-07,
+      "loss": 0.0022,
+      "reward": 1.9083279371261597,
+      "reward_std": 0.01711435243487358,
+      "rewards/accuracy_reward": 0.9083278775215149,
+      "rewards/format_reward": 1.0,
+      "step": 585
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.89784240722656,
+      "epoch": 8.617647058823529,
+      "grad_norm": 1.411173293415255,
+      "kl": 0.049072265625,
+      "learning_rate": 5.691176470588235e-07,
+      "loss": 0.002,
+      "reward": 1.9171125888824463,
+      "reward_std": 0.016976075246930122,
+      "rewards/accuracy_reward": 0.9183142781257629,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 586
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.58053588867188,
+      "epoch": 8.632352941176471,
+      "grad_norm": 1.8153636392557528,
+      "kl": 0.050537109375,
+      "learning_rate": 5.683823529411764e-07,
+      "loss": 0.002,
+      "reward": 1.9182407855987549,
+      "reward_std": 0.013324742205440998,
+      "rewards/accuracy_reward": 0.9182407855987549,
+      "rewards/format_reward": 1.0,
+      "step": 587
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.02644348144531,
+      "epoch": 8.647058823529411,
+      "grad_norm": 2.991122791360467,
+      "kl": 0.049072265625,
+      "learning_rate": 5.676470588235294e-07,
+      "loss": 0.002,
+      "reward": 1.9190689325332642,
+      "reward_std": 0.01496278028935194,
+      "rewards/accuracy_reward": 0.9190687537193298,
+      "rewards/format_reward": 1.0,
+      "step": 588
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.1971206665039,
+      "epoch": 8.661764705882353,
+      "grad_norm": 1.5660894873733406,
+      "kl": 0.05078125,
+      "learning_rate": 5.669117647058823e-07,
+      "loss": 0.002,
+      "reward": 1.8968241214752197,
+      "reward_std": 0.015287560410797596,
+      "rewards/accuracy_reward": 0.896824061870575,
+      "rewards/format_reward": 1.0,
+      "step": 589
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.95072174072266,
+      "epoch": 8.676470588235293,
+      "grad_norm": 1.7772209705137463,
+      "kl": 0.054443359375,
+      "learning_rate": 5.661764705882352e-07,
+      "loss": 0.0022,
+      "reward": 1.9128403663635254,
+      "reward_std": 0.020346660166978836,
+      "rewards/accuracy_reward": 0.9140422940254211,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 590
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.78726196289062,
+      "epoch": 8.691176470588236,
+      "grad_norm": 1.6721267407160185,
+      "kl": 0.046630859375,
+      "learning_rate": 5.654411764705882e-07,
+      "loss": 0.0019,
+      "reward": 1.9050085544586182,
+      "reward_std": 0.018483325839042664,
+      "rewards/accuracy_reward": 0.9050085544586182,
+      "rewards/format_reward": 1.0,
+      "step": 591
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.97956848144531,
+      "epoch": 8.705882352941176,
+      "grad_norm": 1.6709044867050522,
+      "kl": 0.052490234375,
+      "learning_rate": 5.647058823529412e-07,
+      "loss": 0.0021,
+      "reward": 1.924027442932129,
+      "reward_std": 0.019660327583551407,
+      "rewards/accuracy_reward": 0.9252294301986694,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 592
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.9988021850586,
+      "epoch": 8.720588235294118,
+      "grad_norm": 4.487069114218076,
+      "kl": 0.05029296875,
+      "learning_rate": 5.639705882352941e-07,
+      "loss": 0.002,
+      "reward": 1.8979933261871338,
+      "reward_std": 0.01683800294995308,
+      "rewards/accuracy_reward": 0.897993266582489,
+      "rewards/format_reward": 1.0,
+      "step": 593
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.0769271850586,
+      "epoch": 8.735294117647058,
+      "grad_norm": 2.4615353009495817,
+      "kl": 0.07763671875,
+      "learning_rate": 5.63235294117647e-07,
+      "loss": 0.0031,
+      "reward": 1.8970861434936523,
+      "reward_std": 0.019765671342611313,
+      "rewards/accuracy_reward": 0.8982879519462585,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 594
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.85216522216797,
+      "epoch": 8.75,
+      "grad_norm": 1.735078503716415,
+      "kl": 0.051025390625,
+      "learning_rate": 5.625e-07,
+      "loss": 0.002,
+      "reward": 1.9034225940704346,
+      "reward_std": 0.018201464787125587,
+      "rewards/accuracy_reward": 0.9034226536750793,
+      "rewards/format_reward": 1.0,
+      "step": 595
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.47116088867188,
+      "epoch": 8.764705882352942,
+      "grad_norm": 2.193668601424281,
+      "kl": 0.048095703125,
+      "learning_rate": 5.617647058823529e-07,
+      "loss": 0.0019,
+      "reward": 1.8874914646148682,
+      "reward_std": 0.024996280670166016,
+      "rewards/accuracy_reward": 0.8898953795433044,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 596
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.53366088867188,
+      "epoch": 8.779411764705882,
+      "grad_norm": 1.5231562376577301,
+      "kl": 0.0498046875,
+      "learning_rate": 5.610294117647059e-07,
+      "loss": 0.002,
+      "reward": 1.910040259361267,
+      "reward_std": 0.016859347000718117,
+      "rewards/accuracy_reward": 0.9100401997566223,
+      "rewards/format_reward": 1.0,
+      "step": 597
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.86058044433594,
+      "epoch": 8.794117647058824,
+      "grad_norm": 6.4583154297757455,
+      "kl": 0.05029296875,
+      "learning_rate": 5.602941176470588e-07,
+      "loss": 0.002,
+      "reward": 1.9111155271530151,
+      "reward_std": 0.016216028481721878,
+      "rewards/accuracy_reward": 0.9111157059669495,
+      "rewards/format_reward": 1.0,
+      "step": 598
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.24279022216797,
+      "epoch": 8.808823529411764,
+      "grad_norm": 1.775836022535222,
+      "kl": 0.05224609375,
+      "learning_rate": 5.595588235294117e-07,
+      "loss": 0.0021,
+      "reward": 1.917399287223816,
+      "reward_std": 0.016161981970071793,
+      "rewards/accuracy_reward": 0.9173993468284607,
+      "rewards/format_reward": 1.0,
+      "step": 599
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.46514892578125,
+      "epoch": 8.823529411764707,
+      "grad_norm": 1.374775331428731,
+      "kl": 0.052001953125,
+      "learning_rate": 5.588235294117647e-07,
+      "loss": 0.0021,
+      "reward": 1.8994345664978027,
+      "reward_std": 0.017103036865592003,
+      "rewards/accuracy_reward": 0.8994343876838684,
+      "rewards/format_reward": 1.0,
+      "step": 600
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.3894271850586,
+      "epoch": 8.838235294117647,
+      "grad_norm": 2.9229408253654365,
+      "kl": 0.05029296875,
+      "learning_rate": 5.580882352941177e-07,
+      "loss": 0.002,
+      "reward": 1.9245597124099731,
+      "reward_std": 0.016064738854765892,
+      "rewards/accuracy_reward": 0.9245598316192627,
+      "rewards/format_reward": 1.0,
+      "step": 601
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.09134674072266,
+      "epoch": 8.852941176470589,
+      "grad_norm": 1.761051200827917,
+      "kl": 0.053955078125,
+      "learning_rate": 5.573529411764706e-07,
+      "loss": 0.0022,
+      "reward": 1.9057958126068115,
+      "reward_std": 0.018393708392977715,
+      "rewards/accuracy_reward": 0.9057958126068115,
+      "rewards/format_reward": 1.0,
+      "step": 602
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.97956848144531,
+      "epoch": 8.867647058823529,
+      "grad_norm": 1.9085242419393635,
+      "kl": 0.05419921875,
+      "learning_rate": 5.566176470588235e-07,
+      "loss": 0.0022,
+      "reward": 1.9057040214538574,
+      "reward_std": 0.02528255060315132,
+      "rewards/accuracy_reward": 0.9069057106971741,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 603
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.55409240722656,
+      "epoch": 8.882352941176471,
+      "grad_norm": 2.2056191207524742,
+      "kl": 0.050537109375,
+      "learning_rate": 5.558823529411765e-07,
+      "loss": 0.002,
+      "reward": 1.9001765251159668,
+      "reward_std": 0.01882914826273918,
+      "rewards/accuracy_reward": 0.9001763463020325,
+      "rewards/format_reward": 1.0,
+      "step": 604
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.1189956665039,
+      "epoch": 8.897058823529411,
+      "grad_norm": 1.6684830952892549,
+      "kl": 0.049072265625,
+      "learning_rate": 5.551470588235294e-07,
+      "loss": 0.002,
+      "reward": 1.9099537134170532,
+      "reward_std": 0.015918085351586342,
+      "rewards/accuracy_reward": 0.9099536538124084,
+      "rewards/format_reward": 1.0,
+      "step": 605
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.1081771850586,
+      "epoch": 8.911764705882353,
+      "grad_norm": 2.437963685533989,
+      "kl": 0.052001953125,
+      "learning_rate": 5.544117647058824e-07,
+      "loss": 0.0021,
+      "reward": 1.9115108251571655,
+      "reward_std": 0.01843283697962761,
+      "rewards/accuracy_reward": 0.911510705947876,
+      "rewards/format_reward": 1.0,
+      "step": 606
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.92188262939453,
+      "epoch": 8.926470588235293,
+      "grad_norm": 5.175318731979124,
+      "kl": 0.053466796875,
+      "learning_rate": 5.536764705882353e-07,
+      "loss": 0.0021,
+      "reward": 1.9014861583709717,
+      "reward_std": 0.016052136197686195,
+      "rewards/accuracy_reward": 0.9014859199523926,
+      "rewards/format_reward": 1.0,
+      "step": 607
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.3377456665039,
+      "epoch": 8.941176470588236,
+      "grad_norm": 1.466199052775766,
+      "kl": 0.055419921875,
+      "learning_rate": 5.529411764705882e-07,
+      "loss": 0.0022,
+      "reward": 1.8980929851531982,
+      "reward_std": 0.020213749259710312,
+      "rewards/accuracy_reward": 0.8992946743965149,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 608
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.5252456665039,
+      "epoch": 8.955882352941176,
+      "grad_norm": 2.4803683445732196,
+      "kl": 0.05224609375,
+      "learning_rate": 5.522058823529412e-07,
+      "loss": 0.0021,
+      "reward": 1.8946930170059204,
+      "reward_std": 0.020943893119692802,
+      "rewards/accuracy_reward": 0.8946928381919861,
+      "rewards/format_reward": 1.0,
+      "step": 609
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.81851196289062,
+      "epoch": 8.970588235294118,
+      "grad_norm": 2.199347709692285,
+      "kl": 0.056396484375,
+      "learning_rate": 5.514705882352942e-07,
+      "loss": 0.0023,
+      "reward": 1.9163153171539307,
+      "reward_std": 0.019964834675192833,
+      "rewards/accuracy_reward": 0.9175171852111816,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 610
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.43389892578125,
+      "epoch": 8.985294117647058,
+      "grad_norm": 1.3598803521986345,
+      "kl": 0.058349609375,
+      "learning_rate": 5.50735294117647e-07,
+      "loss": 0.0023,
+      "reward": 1.9314730167388916,
+      "reward_std": 0.01666920818388462,
+      "rewards/accuracy_reward": 0.9314729571342468,
+      "rewards/format_reward": 1.0,
+      "step": 611
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.38204193115234,
+      "epoch": 9.0,
+      "grad_norm": 2.5292498675314574,
+      "kl": 0.0546875,
+      "learning_rate": 5.5e-07,
+      "loss": 0.0022,
+      "reward": 1.9147766828536987,
+      "reward_std": 0.012550020590424538,
+      "rewards/accuracy_reward": 0.9147767424583435,
+      "rewards/format_reward": 1.0,
+      "step": 612
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.68269348144531,
+      "epoch": 9.014705882352942,
+      "grad_norm": 1.359576415697596,
+      "kl": 0.05419921875,
+      "learning_rate": 5.49264705882353e-07,
+      "loss": 0.0022,
+      "reward": 1.9160057306289673,
+      "reward_std": 0.018689963966608047,
+      "rewards/accuracy_reward": 0.9172075986862183,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 613
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.44831848144531,
+      "epoch": 9.029411764705882,
+      "grad_norm": 1.5142397828555845,
+      "kl": 0.0595703125,
+      "learning_rate": 5.485294117647059e-07,
+      "loss": 0.0024,
+      "reward": 1.8921860456466675,
+      "reward_std": 0.021658875048160553,
+      "rewards/accuracy_reward": 0.8933879137039185,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 614
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.91226196289062,
+      "epoch": 9.044117647058824,
+      "grad_norm": 5.469469688964059,
+      "kl": 0.05615234375,
+      "learning_rate": 5.477941176470589e-07,
+      "loss": 0.0023,
+      "reward": 1.9030665159225464,
+      "reward_std": 0.02024255506694317,
+      "rewards/accuracy_reward": 0.9042685627937317,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 615
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.23197174072266,
+      "epoch": 9.058823529411764,
+      "grad_norm": 2.869773876567515,
+      "kl": 0.07080078125,
+      "learning_rate": 5.470588235294118e-07,
+      "loss": 0.0028,
+      "reward": 1.912828803062439,
+      "reward_std": 0.018641209229826927,
+      "rewards/accuracy_reward": 0.9128286838531494,
+      "rewards/format_reward": 1.0,
+      "step": 616
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.40625762939453,
+      "epoch": 9.073529411764707,
+      "grad_norm": 1.880572098436719,
+      "kl": 0.05712890625,
+      "learning_rate": 5.463235294117646e-07,
+      "loss": 0.0023,
+      "reward": 1.9045672416687012,
+      "reward_std": 0.01760552078485489,
+      "rewards/accuracy_reward": 0.9045671820640564,
+      "rewards/format_reward": 1.0,
+      "step": 617
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.60216522216797,
+      "epoch": 9.088235294117647,
+      "grad_norm": 1.6132861487685206,
+      "kl": 0.056884765625,
+      "learning_rate": 5.455882352941176e-07,
+      "loss": 0.0023,
+      "reward": 1.9064825773239136,
+      "reward_std": 0.02047489397227764,
+      "rewards/accuracy_reward": 0.9064825773239136,
+      "rewards/format_reward": 1.0,
+      "step": 618
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.12139892578125,
+      "epoch": 9.102941176470589,
+      "grad_norm": 1.8352294433327585,
+      "kl": 0.061279296875,
+      "learning_rate": 5.448529411764706e-07,
+      "loss": 0.0024,
+      "reward": 1.9166762828826904,
+      "reward_std": 0.017683234065771103,
+      "rewards/accuracy_reward": 0.9166761636734009,
+      "rewards/format_reward": 1.0,
+      "step": 619
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.82813262939453,
+      "epoch": 9.117647058823529,
+      "grad_norm": 1.9584175345112422,
+      "kl": 0.05810546875,
+      "learning_rate": 5.441176470588234e-07,
+      "loss": 0.0023,
+      "reward": 1.9098379611968994,
+      "reward_std": 0.019358208402991295,
+      "rewards/accuracy_reward": 0.9098379015922546,
+      "rewards/format_reward": 1.0,
+      "step": 620
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.54928588867188,
+      "epoch": 9.132352941176471,
+      "grad_norm": 1.681570036282453,
+      "kl": 0.05908203125,
+      "learning_rate": 5.433823529411764e-07,
+      "loss": 0.0024,
+      "reward": 1.903365135192871,
+      "reward_std": 0.018463198095560074,
+      "rewards/accuracy_reward": 0.9045670032501221,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 621
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.81370544433594,
+      "epoch": 9.147058823529411,
+      "grad_norm": 2.175735273009701,
+      "kl": 0.06787109375,
+      "learning_rate": 5.426470588235294e-07,
+      "loss": 0.0027,
+      "reward": 1.9245383739471436,
+      "reward_std": 0.015885451808571815,
+      "rewards/accuracy_reward": 0.9245381355285645,
+      "rewards/format_reward": 1.0,
+      "step": 622
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.20313262939453,
+      "epoch": 9.161764705882353,
+      "grad_norm": 2.20406419419079,
+      "kl": 0.05810546875,
+      "learning_rate": 5.419117647058823e-07,
+      "loss": 0.0023,
+      "reward": 1.9137063026428223,
+      "reward_std": 0.018180424347519875,
+      "rewards/accuracy_reward": 0.9137064218521118,
+      "rewards/format_reward": 1.0,
+      "step": 623
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.13221740722656,
+      "epoch": 9.176470588235293,
+      "grad_norm": 1.7531001551644594,
+      "kl": 0.057373046875,
+      "learning_rate": 5.411764705882353e-07,
+      "loss": 0.0023,
+      "reward": 1.9103367328643799,
+      "reward_std": 0.01696370542049408,
+      "rewards/accuracy_reward": 0.9103366136550903,
+      "rewards/format_reward": 1.0,
+      "step": 624
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.02163696289062,
+      "epoch": 9.191176470588236,
+      "grad_norm": 1.9594882556521003,
+      "kl": 0.06201171875,
+      "learning_rate": 5.404411764705882e-07,
+      "loss": 0.0025,
+      "reward": 1.9130682945251465,
+      "reward_std": 0.014644093811511993,
+      "rewards/accuracy_reward": 0.9130681157112122,
+      "rewards/format_reward": 1.0,
+      "step": 625
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.65866088867188,
+      "epoch": 9.205882352941176,
+      "grad_norm": 2.6647032908164303,
+      "kl": 0.0595703125,
+      "learning_rate": 5.397058823529411e-07,
+      "loss": 0.0024,
+      "reward": 1.9141494035720825,
+      "reward_std": 0.013022142462432384,
+      "rewards/accuracy_reward": 0.9141494035720825,
+      "rewards/format_reward": 1.0,
+      "step": 626
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.36298370361328,
+      "epoch": 9.220588235294118,
+      "grad_norm": 2.5371748740188753,
+      "kl": 0.060791015625,
+      "learning_rate": 5.389705882352941e-07,
+      "loss": 0.0025,
+      "reward": 1.91664719581604,
+      "reward_std": 0.017335418611764908,
+      "rewards/accuracy_reward": 0.9166471362113953,
+      "rewards/format_reward": 1.0,
+      "step": 627
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.4627456665039,
+      "epoch": 9.235294117647058,
+      "grad_norm": 1.7615752856014142,
+      "kl": 0.05712890625,
+      "learning_rate": 5.382352941176471e-07,
+      "loss": 0.0023,
+      "reward": 1.8938990831375122,
+      "reward_std": 0.016765251755714417,
+      "rewards/accuracy_reward": 0.893899142742157,
+      "rewards/format_reward": 1.0,
+      "step": 628
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.77404022216797,
+      "epoch": 9.25,
+      "grad_norm": 2.0562724198467914,
+      "kl": 0.0634765625,
+      "learning_rate": 5.374999999999999e-07,
+      "loss": 0.0026,
+      "reward": 1.9214463233947754,
+      "reward_std": 0.014558774419128895,
+      "rewards/accuracy_reward": 0.9214461445808411,
+      "rewards/format_reward": 1.0,
+      "step": 629
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.44351196289062,
+      "epoch": 9.264705882352942,
+      "grad_norm": 2.1195056229903555,
+      "kl": 0.0576171875,
+      "learning_rate": 5.367647058823529e-07,
+      "loss": 0.0023,
+      "reward": 1.9065834283828735,
+      "reward_std": 0.016932621598243713,
+      "rewards/accuracy_reward": 0.906583309173584,
+      "rewards/format_reward": 1.0,
+      "step": 630
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.56610870361328,
+      "epoch": 9.279411764705882,
+      "grad_norm": 1.5239442402080228,
+      "kl": 0.0546875,
+      "learning_rate": 5.360294117647058e-07,
+      "loss": 0.0022,
+      "reward": 1.9177383184432983,
+      "reward_std": 0.014962482266128063,
+      "rewards/accuracy_reward": 0.9177382588386536,
+      "rewards/format_reward": 1.0,
+      "step": 631
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.86538696289062,
+      "epoch": 9.294117647058824,
+      "grad_norm": 2.027623877286765,
+      "kl": 0.05517578125,
+      "learning_rate": 5.352941176470588e-07,
+      "loss": 0.0022,
+      "reward": 1.9050103425979614,
+      "reward_std": 0.014742089435458183,
+      "rewards/accuracy_reward": 0.9050101637840271,
+      "rewards/format_reward": 1.0,
+      "step": 632
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.48558044433594,
+      "epoch": 9.308823529411764,
+      "grad_norm": 1.9258858099537597,
+      "kl": 0.060546875,
+      "learning_rate": 5.345588235294117e-07,
+      "loss": 0.0024,
+      "reward": 1.9041670560836792,
+      "reward_std": 0.017647940665483475,
+      "rewards/accuracy_reward": 0.9053689241409302,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 633
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.88221740722656,
+      "epoch": 9.323529411764707,
+      "grad_norm": 5.481626944028736,
+      "kl": 0.05419921875,
+      "learning_rate": 5.338235294117646e-07,
+      "loss": 0.0022,
+      "reward": 1.8989014625549316,
+      "reward_std": 0.017911560833454132,
+      "rewards/accuracy_reward": 0.8989012837409973,
+      "rewards/format_reward": 1.0,
+      "step": 634
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.16947174072266,
+      "epoch": 9.338235294117647,
+      "grad_norm": 1.5838252460154945,
+      "kl": 0.052734375,
+      "learning_rate": 5.330882352941176e-07,
+      "loss": 0.0021,
+      "reward": 1.9077017307281494,
+      "reward_std": 0.01683455891907215,
+      "rewards/accuracy_reward": 0.9089035987854004,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 635
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.02764892578125,
+      "epoch": 9.352941176470589,
+      "grad_norm": 20.072264259712355,
+      "kl": 0.05908203125,
+      "learning_rate": 5.323529411764706e-07,
+      "loss": 0.0024,
+      "reward": 1.9148465394973755,
+      "reward_std": 0.01663627289235592,
+      "rewards/accuracy_reward": 0.9160483479499817,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 636
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.4158706665039,
+      "epoch": 9.367647058823529,
+      "grad_norm": 1.5736397571374754,
+      "kl": 0.05712890625,
+      "learning_rate": 5.316176470588235e-07,
+      "loss": 0.0023,
+      "reward": 1.9175165891647339,
+      "reward_std": 0.017536688596010208,
+      "rewards/accuracy_reward": 0.9175165891647339,
+      "rewards/format_reward": 1.0,
+      "step": 637
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.81971740722656,
+      "epoch": 9.382352941176471,
+      "grad_norm": 6.15992463353084,
+      "kl": 0.054931640625,
+      "learning_rate": 5.308823529411764e-07,
+      "loss": 0.0022,
+      "reward": 1.9007567167282104,
+      "reward_std": 0.015532239340245724,
+      "rewards/accuracy_reward": 0.9007567167282104,
+      "rewards/format_reward": 1.0,
+      "step": 638
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.92668914794922,
+      "epoch": 9.397058823529411,
+      "grad_norm": 1.9267289588312566,
+      "kl": 0.05712890625,
+      "learning_rate": 5.301470588235294e-07,
+      "loss": 0.0023,
+      "reward": 1.910290241241455,
+      "reward_std": 0.017649637535214424,
+      "rewards/accuracy_reward": 0.9114922285079956,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 639
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.70553588867188,
+      "epoch": 9.411764705882353,
+      "grad_norm": 2.331055894294996,
+      "kl": 0.057861328125,
+      "learning_rate": 5.294117647058823e-07,
+      "loss": 0.0023,
+      "reward": 1.9117798805236816,
+      "reward_std": 0.014000487513840199,
+      "rewards/accuracy_reward": 0.9117798209190369,
+      "rewards/format_reward": 1.0,
+      "step": 640
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.7644271850586,
+      "epoch": 9.426470588235293,
+      "grad_norm": 2.9782765384820302,
+      "kl": 0.05908203125,
+      "learning_rate": 5.286764705882353e-07,
+      "loss": 0.0024,
+      "reward": 1.8985432386398315,
+      "reward_std": 0.016746515408158302,
+      "rewards/accuracy_reward": 0.8985430002212524,
+      "rewards/format_reward": 1.0,
+      "step": 641
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.16106414794922,
+      "epoch": 9.441176470588236,
+      "grad_norm": 3.0263333302618585,
+      "kl": 0.0576171875,
+      "learning_rate": 5.279411764705882e-07,
+      "loss": 0.0023,
+      "reward": 1.9077398777008057,
+      "reward_std": 0.016522472724318504,
+      "rewards/accuracy_reward": 0.9089417457580566,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 642
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.0252456665039,
+      "epoch": 9.455882352941176,
+      "grad_norm": 3.6088229354945858,
+      "kl": 0.05615234375,
+      "learning_rate": 5.272058823529411e-07,
+      "loss": 0.0023,
+      "reward": 1.912189245223999,
+      "reward_std": 0.01710669696331024,
+      "rewards/accuracy_reward": 0.9133911728858948,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 643
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.2908706665039,
+      "epoch": 9.470588235294118,
+      "grad_norm": 8.188981831182572,
+      "kl": 0.053466796875,
+      "learning_rate": 5.264705882352941e-07,
+      "loss": 0.0021,
+      "reward": 1.9111279249191284,
+      "reward_std": 0.0145622119307518,
+      "rewards/accuracy_reward": 0.9111279249191284,
+      "rewards/format_reward": 1.0,
+      "step": 644
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.99639892578125,
+      "epoch": 9.485294117647058,
+      "grad_norm": 3.3433584553752866,
+      "kl": 0.054931640625,
+      "learning_rate": 5.257352941176471e-07,
+      "loss": 0.0022,
+      "reward": 1.9080722332000732,
+      "reward_std": 0.01804247312247753,
+      "rewards/accuracy_reward": 0.9092740416526794,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 645
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.66346740722656,
+      "epoch": 9.5,
+      "grad_norm": 1.3581128929115824,
+      "kl": 0.05908203125,
+      "learning_rate": 5.25e-07,
+      "loss": 0.0024,
+      "reward": 1.9173364639282227,
+      "reward_std": 0.0154867609962821,
+      "rewards/accuracy_reward": 0.9173362851142883,
+      "rewards/format_reward": 1.0,
+      "step": 646
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.97956848144531,
+      "epoch": 9.514705882352942,
+      "grad_norm": 3.583185511646481,
+      "kl": 0.0517578125,
+      "learning_rate": 5.242647058823529e-07,
+      "loss": 0.0021,
+      "reward": 1.913078784942627,
+      "reward_std": 0.01793815754354,
+      "rewards/accuracy_reward": 0.9142807126045227,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 647
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.81971740722656,
+      "epoch": 9.529411764705882,
+      "grad_norm": 2.2219404022193494,
+      "kl": 0.058349609375,
+      "learning_rate": 5.235294117647059e-07,
+      "loss": 0.0023,
+      "reward": 1.9022469520568848,
+      "reward_std": 0.019146382808685303,
+      "rewards/accuracy_reward": 0.9046508073806763,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 648
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.07813262939453,
+      "epoch": 9.544117647058824,
+      "grad_norm": 2.825487501905413,
+      "kl": 0.0625,
+      "learning_rate": 5.227941176470588e-07,
+      "loss": 0.0025,
+      "reward": 1.916589617729187,
+      "reward_std": 0.016064953058958054,
+      "rewards/accuracy_reward": 0.9177915453910828,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 649
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.50601196289062,
+      "epoch": 9.558823529411764,
+      "grad_norm": 5.158018889820331,
+      "kl": 0.051513671875,
+      "learning_rate": 5.220588235294118e-07,
+      "loss": 0.0021,
+      "reward": 1.9119791984558105,
+      "reward_std": 0.015298433601856232,
+      "rewards/accuracy_reward": 0.9119791388511658,
+      "rewards/format_reward": 1.0,
+      "step": 650
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.58413696289062,
+      "epoch": 9.573529411764707,
+      "grad_norm": 2.329896463825327,
+      "kl": 0.047119140625,
+      "learning_rate": 5.213235294117647e-07,
+      "loss": 0.0019,
+      "reward": 1.9101742506027222,
+      "reward_std": 0.015595993027091026,
+      "rewards/accuracy_reward": 0.9101741909980774,
+      "rewards/format_reward": 1.0,
+      "step": 651
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.80769348144531,
+      "epoch": 9.588235294117647,
+      "grad_norm": 1.93503535510301,
+      "kl": 0.05322265625,
+      "learning_rate": 5.205882352941176e-07,
+      "loss": 0.0021,
+      "reward": 1.9048891067504883,
+      "reward_std": 0.014783341437578201,
+      "rewards/accuracy_reward": 0.904888927936554,
+      "rewards/format_reward": 1.0,
+      "step": 652
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.08534240722656,
+      "epoch": 9.602941176470589,
+      "grad_norm": 2.84495334769509,
+      "kl": 0.060791015625,
+      "learning_rate": 5.198529411764706e-07,
+      "loss": 0.0024,
+      "reward": 1.9240138530731201,
+      "reward_std": 0.014201764948666096,
+      "rewards/accuracy_reward": 0.9240137338638306,
+      "rewards/format_reward": 1.0,
+      "step": 653
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.84856414794922,
+      "epoch": 9.617647058823529,
+      "grad_norm": 3.012251558930328,
+      "kl": 0.05419921875,
+      "learning_rate": 5.191176470588236e-07,
+      "loss": 0.0022,
+      "reward": 1.905051350593567,
+      "reward_std": 0.013432993553578854,
+      "rewards/accuracy_reward": 0.9050513505935669,
+      "rewards/format_reward": 1.0,
+      "step": 654
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.21394348144531,
+      "epoch": 9.632352941176471,
+      "grad_norm": 2.1316420678211956,
+      "kl": 0.056640625,
+      "learning_rate": 5.183823529411764e-07,
+      "loss": 0.0023,
+      "reward": 1.9092304706573486,
+      "reward_std": 0.01678118109703064,
+      "rewards/accuracy_reward": 0.9092304706573486,
+      "rewards/format_reward": 1.0,
+      "step": 655
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.33654022216797,
+      "epoch": 9.647058823529411,
+      "grad_norm": 2.2956788687873826,
+      "kl": 0.06298828125,
+      "learning_rate": 5.176470588235294e-07,
+      "loss": 0.0025,
+      "reward": 1.92042076587677,
+      "reward_std": 0.016655659303069115,
+      "rewards/accuracy_reward": 0.92042076587677,
+      "rewards/format_reward": 1.0,
+      "step": 656
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.62259674072266,
+      "epoch": 9.661764705882353,
+      "grad_norm": 2.467082612046906,
+      "kl": 0.054931640625,
+      "learning_rate": 5.169117647058824e-07,
+      "loss": 0.0022,
+      "reward": 1.9160475730895996,
+      "reward_std": 0.01348184235394001,
+      "rewards/accuracy_reward": 0.9160475134849548,
+      "rewards/format_reward": 1.0,
+      "step": 657
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.80529022216797,
+      "epoch": 9.676470588235293,
+      "grad_norm": 7.66545161952161,
+      "kl": 0.06396484375,
+      "learning_rate": 5.161764705882353e-07,
+      "loss": 0.0026,
+      "reward": 1.9231846332550049,
+      "reward_std": 0.016615498811006546,
+      "rewards/accuracy_reward": 0.9231847524642944,
+      "rewards/format_reward": 1.0,
+      "step": 658
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.1189956665039,
+      "epoch": 9.691176470588236,
+      "grad_norm": 2.341042431156855,
+      "kl": 0.057373046875,
+      "learning_rate": 5.154411764705882e-07,
+      "loss": 0.0023,
+      "reward": 1.912707805633545,
+      "reward_std": 0.018813416361808777,
+      "rewards/accuracy_reward": 0.9127077460289001,
+      "rewards/format_reward": 1.0,
+      "step": 659
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.10938262939453,
+      "epoch": 9.705882352941176,
+      "grad_norm": 2.1614288925748966,
+      "kl": 0.0654296875,
+      "learning_rate": 5.147058823529411e-07,
+      "loss": 0.0026,
+      "reward": 1.9185718297958374,
+      "reward_std": 0.01388857513666153,
+      "rewards/accuracy_reward": 0.9185718894004822,
+      "rewards/format_reward": 1.0,
+      "step": 660
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.92788696289062,
+      "epoch": 9.720588235294118,
+      "grad_norm": 2.80121444579991,
+      "kl": 0.06396484375,
+      "learning_rate": 5.13970588235294e-07,
+      "loss": 0.0026,
+      "reward": 1.9083857536315918,
+      "reward_std": 0.01726207509636879,
+      "rewards/accuracy_reward": 0.9083855152130127,
+      "rewards/format_reward": 1.0,
+      "step": 661
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.50721740722656,
+      "epoch": 9.735294117647058,
+      "grad_norm": 2.7135290225206936,
+      "kl": 0.060546875,
+      "learning_rate": 5.13235294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.8957988023757935,
+      "reward_std": 0.017370108515024185,
+      "rewards/accuracy_reward": 0.8957986235618591,
+      "rewards/format_reward": 1.0,
+      "step": 662
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.86298370361328,
+      "epoch": 9.75,
+      "grad_norm": 1.6206042221923098,
+      "kl": 0.054931640625,
+      "learning_rate": 5.125e-07,
+      "loss": 0.0022,
+      "reward": 1.9014039039611816,
+      "reward_std": 0.0170334093272686,
+      "rewards/accuracy_reward": 0.9014038443565369,
+      "rewards/format_reward": 1.0,
+      "step": 663
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.10577392578125,
+      "epoch": 9.764705882352942,
+      "grad_norm": 2.1238141084640287,
+      "kl": 0.056396484375,
+      "learning_rate": 5.117647058823528e-07,
+      "loss": 0.0022,
+      "reward": 1.9057307243347168,
+      "reward_std": 0.017774611711502075,
+      "rewards/accuracy_reward": 0.9057307243347168,
+      "rewards/format_reward": 1.0,
+      "step": 664
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.60096740722656,
+      "epoch": 9.779411764705882,
+      "grad_norm": 1.6218363054606357,
+      "kl": 0.052490234375,
+      "learning_rate": 5.110294117647058e-07,
+      "loss": 0.0021,
+      "reward": 1.892624855041504,
+      "reward_std": 0.018729185685515404,
+      "rewards/accuracy_reward": 0.8926247954368591,
+      "rewards/format_reward": 1.0,
+      "step": 665
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.73798370361328,
+      "epoch": 9.794117647058824,
+      "grad_norm": 4.41894162926749,
+      "kl": 0.061279296875,
+      "learning_rate": 5.102941176470588e-07,
+      "loss": 0.0024,
+      "reward": 1.8940479755401611,
+      "reward_std": 0.019051702693104744,
+      "rewards/accuracy_reward": 0.8940479159355164,
+      "rewards/format_reward": 1.0,
+      "step": 666
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.1706771850586,
+      "epoch": 9.808823529411764,
+      "grad_norm": 2.5204373524492754,
+      "kl": 0.0625,
+      "learning_rate": 5.095588235294117e-07,
+      "loss": 0.0025,
+      "reward": 1.9303815364837646,
+      "reward_std": 0.016049683094024658,
+      "rewards/accuracy_reward": 0.9303815364837646,
+      "rewards/format_reward": 1.0,
+      "step": 667
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.2596206665039,
+      "epoch": 9.823529411764707,
+      "grad_norm": 2.247808117324336,
+      "kl": 0.062255859375,
+      "learning_rate": 5.088235294117646e-07,
+      "loss": 0.0025,
+      "reward": 1.905713677406311,
+      "reward_std": 0.0260589811950922,
+      "rewards/accuracy_reward": 0.9069154858589172,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 668
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.00360870361328,
+      "epoch": 9.838235294117647,
+      "grad_norm": 2.9378173829289542,
+      "kl": 0.05517578125,
+      "learning_rate": 5.080882352941176e-07,
+      "loss": 0.0022,
+      "reward": 1.9220634698867798,
+      "reward_std": 0.018444770947098732,
+      "rewards/accuracy_reward": 0.923265278339386,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 669
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.38341522216797,
+      "epoch": 9.852941176470589,
+      "grad_norm": 3.1381468730562396,
+      "kl": 0.055908203125,
+      "learning_rate": 5.073529411764705e-07,
+      "loss": 0.0022,
+      "reward": 1.9113494157791138,
+      "reward_std": 0.017814362421631813,
+      "rewards/accuracy_reward": 0.91255122423172,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 670
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.08654022216797,
+      "epoch": 9.867647058823529,
+      "grad_norm": 1.8503806680078807,
+      "kl": 0.053466796875,
+      "learning_rate": 5.066176470588235e-07,
+      "loss": 0.0021,
+      "reward": 1.9079095125198364,
+      "reward_std": 0.01949225552380085,
+      "rewards/accuracy_reward": 0.9091113209724426,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 671
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.08654022216797,
+      "epoch": 9.882352941176471,
+      "grad_norm": 1.6795640500896751,
+      "kl": 0.051025390625,
+      "learning_rate": 5.058823529411765e-07,
+      "loss": 0.002,
+      "reward": 1.9217894077301025,
+      "reward_std": 0.01603555493056774,
+      "rewards/accuracy_reward": 0.9217892289161682,
+      "rewards/format_reward": 1.0,
+      "step": 672
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.29327392578125,
+      "epoch": 9.897058823529411,
+      "grad_norm": 1.9285682708702305,
+      "kl": 0.05126953125,
+      "learning_rate": 5.051470588235293e-07,
+      "loss": 0.002,
+      "reward": 1.9053161144256592,
+      "reward_std": 0.01650179550051689,
+      "rewards/accuracy_reward": 0.9053158760070801,
+      "rewards/format_reward": 1.0,
+      "step": 673
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.87981414794922,
+      "epoch": 9.911764705882353,
+      "grad_norm": 2.038439014531085,
+      "kl": 0.05126953125,
+      "learning_rate": 5.044117647058823e-07,
+      "loss": 0.002,
+      "reward": 1.9242396354675293,
+      "reward_std": 0.016753748059272766,
+      "rewards/accuracy_reward": 0.9242396354675293,
+      "rewards/format_reward": 1.0,
+      "step": 674
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.26202392578125,
+      "epoch": 9.926470588235293,
+      "grad_norm": 1.6234604098074579,
+      "kl": 0.053955078125,
+      "learning_rate": 5.036764705882353e-07,
+      "loss": 0.0022,
+      "reward": 1.914339542388916,
+      "reward_std": 0.01580924354493618,
+      "rewards/accuracy_reward": 0.9143396019935608,
+      "rewards/format_reward": 1.0,
+      "step": 675
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.72116088867188,
+      "epoch": 9.941176470588236,
+      "grad_norm": 5.62812065845287,
+      "kl": 0.0537109375,
+      "learning_rate": 5.029411764705882e-07,
+      "loss": 0.0021,
+      "reward": 1.905260443687439,
+      "reward_std": 0.018505161628127098,
+      "rewards/accuracy_reward": 0.9052603244781494,
+      "rewards/format_reward": 1.0,
+      "step": 676
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.61538696289062,
+      "epoch": 9.955882352941176,
+      "grad_norm": 1.7952493866544998,
+      "kl": 0.048095703125,
+      "learning_rate": 5.022058823529411e-07,
+      "loss": 0.0019,
+      "reward": 1.9118279218673706,
+      "reward_std": 0.01776190474629402,
+      "rewards/accuracy_reward": 0.9118278622627258,
+      "rewards/format_reward": 1.0,
+      "step": 677
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.45433044433594,
+      "epoch": 9.970588235294118,
+      "grad_norm": 1.7192070770567416,
+      "kl": 0.059814453125,
+      "learning_rate": 5.014705882352941e-07,
+      "loss": 0.0024,
+      "reward": 1.9066780805587769,
+      "reward_std": 0.01788855344057083,
+      "rewards/accuracy_reward": 0.9066779613494873,
+      "rewards/format_reward": 1.0,
+      "step": 678
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.34856414794922,
+      "epoch": 9.985294117647058,
+      "grad_norm": 2.906612944219061,
+      "kl": 0.06005859375,
+      "learning_rate": 5.00735294117647e-07,
+      "loss": 0.0024,
+      "reward": 1.9021302461624146,
+      "reward_std": 0.022055475041270256,
+      "rewards/accuracy_reward": 0.9021300673484802,
+      "rewards/format_reward": 1.0,
+      "step": 679
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.98021697998047,
+      "epoch": 10.0,
+      "grad_norm": 1.6562395375859271,
+      "kl": 0.047607421875,
+      "learning_rate": 5e-07,
+      "loss": 0.0019,
+      "reward": 1.9186270236968994,
+      "reward_std": 0.019354015588760376,
+      "rewards/accuracy_reward": 0.9201489090919495,
+      "rewards/format_reward": 0.9984779357910156,
+      "step": 680
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.89423370361328,
+      "epoch": 10.014705882352942,
+      "grad_norm": 2.9068911534041764,
+      "kl": 0.05712890625,
+      "learning_rate": 4.99264705882353e-07,
+      "loss": 0.0023,
+      "reward": 1.907537817955017,
+      "reward_std": 0.01922653429210186,
+      "rewards/accuracy_reward": 0.9075377583503723,
+      "rewards/format_reward": 1.0,
+      "step": 681
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.22356414794922,
+      "epoch": 10.029411764705882,
+      "grad_norm": 3.5541747826461805,
+      "kl": 0.055419921875,
+      "learning_rate": 4.985294117647058e-07,
+      "loss": 0.0022,
+      "reward": 1.917733073234558,
+      "reward_std": 0.01717350259423256,
+      "rewards/accuracy_reward": 0.9177329540252686,
+      "rewards/format_reward": 1.0,
+      "step": 682
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.38341522216797,
+      "epoch": 10.044117647058824,
+      "grad_norm": 1.903386359782145,
+      "kl": 0.057373046875,
+      "learning_rate": 4.977941176470588e-07,
+      "loss": 0.0023,
+      "reward": 1.9117200374603271,
+      "reward_std": 0.01899932697415352,
+      "rewards/accuracy_reward": 0.9129219651222229,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 683
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.29688262939453,
+      "epoch": 10.058823529411764,
+      "grad_norm": 1.8861273273154469,
+      "kl": 0.05810546875,
+      "learning_rate": 4.970588235294118e-07,
+      "loss": 0.0023,
+      "reward": 1.9124337434768677,
+      "reward_std": 0.02022276073694229,
+      "rewards/accuracy_reward": 0.9136357307434082,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 684
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.86058044433594,
+      "epoch": 10.073529411764707,
+      "grad_norm": 2.8430111341562823,
+      "kl": 0.05078125,
+      "learning_rate": 4.963235294117647e-07,
+      "loss": 0.002,
+      "reward": 1.9203444719314575,
+      "reward_std": 0.016688454896211624,
+      "rewards/accuracy_reward": 0.9203445315361023,
+      "rewards/format_reward": 1.0,
+      "step": 685
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.33053588867188,
+      "epoch": 10.088235294117647,
+      "grad_norm": 2.803964635402021,
+      "kl": 0.05859375,
+      "learning_rate": 4.955882352941176e-07,
+      "loss": 0.0023,
+      "reward": 1.9130260944366455,
+      "reward_std": 0.019971825182437897,
+      "rewards/accuracy_reward": 0.9142280220985413,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 686
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.25481414794922,
+      "epoch": 10.102941176470589,
+      "grad_norm": 2.1290481897859066,
+      "kl": 0.05517578125,
+      "learning_rate": 4.948529411764706e-07,
+      "loss": 0.0022,
+      "reward": 1.9154521226882935,
+      "reward_std": 0.019178370013833046,
+      "rewards/accuracy_reward": 0.9166539311408997,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 687
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.15264892578125,
+      "epoch": 10.117647058823529,
+      "grad_norm": 2.2315042906400944,
+      "kl": 0.0556640625,
+      "learning_rate": 4.941176470588235e-07,
+      "loss": 0.0022,
+      "reward": 1.9171024560928345,
+      "reward_std": 0.015508743934333324,
+      "rewards/accuracy_reward": 0.9171023368835449,
+      "rewards/format_reward": 1.0,
+      "step": 688
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.18991088867188,
+      "epoch": 10.132352941176471,
+      "grad_norm": 8.899062168348006,
+      "kl": 0.05859375,
+      "learning_rate": 4.933823529411765e-07,
+      "loss": 0.0023,
+      "reward": 1.918565034866333,
+      "reward_std": 0.016680248081684113,
+      "rewards/accuracy_reward": 0.9185649156570435,
+      "rewards/format_reward": 1.0,
+      "step": 689
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.71154022216797,
+      "epoch": 10.147058823529411,
+      "grad_norm": 1.684126014415895,
+      "kl": 0.05712890625,
+      "learning_rate": 4.926470588235295e-07,
+      "loss": 0.0023,
+      "reward": 1.8880369663238525,
+      "reward_std": 0.01872415281832218,
+      "rewards/accuracy_reward": 0.8880367875099182,
+      "rewards/format_reward": 1.0,
+      "step": 690
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.60697174072266,
+      "epoch": 10.161764705882353,
+      "grad_norm": 2.3449846328839596,
+      "kl": 0.05712890625,
+      "learning_rate": 4.919117647058823e-07,
+      "loss": 0.0023,
+      "reward": 1.910075306892395,
+      "reward_std": 0.024183494970202446,
+      "rewards/accuracy_reward": 0.9124791622161865,
+      "rewards/format_reward": 0.9975962042808533,
+      "step": 691
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.82572174072266,
+      "epoch": 10.176470588235293,
+      "grad_norm": 3.23331569595653,
+      "kl": 0.06298828125,
+      "learning_rate": 4.911764705882352e-07,
+      "loss": 0.0025,
+      "reward": 1.9134478569030762,
+      "reward_std": 0.01637120544910431,
+      "rewards/accuracy_reward": 0.9134476184844971,
+      "rewards/format_reward": 1.0,
+      "step": 692
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.63702392578125,
+      "epoch": 10.191176470588236,
+      "grad_norm": 2.3313697919069027,
+      "kl": 0.054931640625,
+      "learning_rate": 4.904411764705882e-07,
+      "loss": 0.0022,
+      "reward": 1.905639410018921,
+      "reward_std": 0.016094274818897247,
+      "rewards/accuracy_reward": 0.9056394100189209,
+      "rewards/format_reward": 1.0,
+      "step": 693
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.92188262939453,
+      "epoch": 10.205882352941176,
+      "grad_norm": 2.6050118805934894,
+      "kl": 0.061767578125,
+      "learning_rate": 4.897058823529412e-07,
+      "loss": 0.0025,
+      "reward": 1.9092425107955933,
+      "reward_std": 0.017230795696377754,
+      "rewards/accuracy_reward": 0.9092425107955933,
+      "rewards/format_reward": 1.0,
+      "step": 694
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.80409240722656,
+      "epoch": 10.220588235294118,
+      "grad_norm": 2.25480395310822,
+      "kl": 0.05615234375,
+      "learning_rate": 4.88970588235294e-07,
+      "loss": 0.0023,
+      "reward": 1.9153361320495605,
+      "reward_std": 0.01601433753967285,
+      "rewards/accuracy_reward": 0.9153360724449158,
+      "rewards/format_reward": 1.0,
+      "step": 695
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.41226196289062,
+      "epoch": 10.235294117647058,
+      "grad_norm": 1.8257275879557004,
+      "kl": 0.05712890625,
+      "learning_rate": 4.88235294117647e-07,
+      "loss": 0.0023,
+      "reward": 1.9105335474014282,
+      "reward_std": 0.014649370685219765,
+      "rewards/accuracy_reward": 0.9105334877967834,
+      "rewards/format_reward": 1.0,
+      "step": 696
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.64063262939453,
+      "epoch": 10.25,
+      "grad_norm": 1.904503451542197,
+      "kl": 0.05712890625,
+      "learning_rate": 4.875e-07,
+      "loss": 0.0023,
+      "reward": 1.9262598752975464,
+      "reward_std": 0.01650756038725376,
+      "rewards/accuracy_reward": 0.9262598752975464,
+      "rewards/format_reward": 1.0,
+      "step": 697
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.5144271850586,
+      "epoch": 10.264705882352942,
+      "grad_norm": 2.0347260430337584,
+      "kl": 0.05712890625,
+      "learning_rate": 4.867647058823529e-07,
+      "loss": 0.0023,
+      "reward": 1.9039766788482666,
+      "reward_std": 0.01813569664955139,
+      "rewards/accuracy_reward": 0.9051785469055176,
+      "rewards/format_reward": 0.998798131942749,
+      "step": 698
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.98798370361328,
+      "epoch": 10.279411764705882,
+      "grad_norm": 3.903638499409672,
+      "kl": 0.055908203125,
+      "learning_rate": 4.860294117647058e-07,
+      "loss": 0.0022,
+      "reward": 1.9150954484939575,
+      "reward_std": 0.014970256946980953,
+      "rewards/accuracy_reward": 0.9150952696800232,
+      "rewards/format_reward": 1.0,
+      "step": 699
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.92308044433594,
+      "epoch": 10.294117647058824,
+      "grad_norm": 2.18266269445753,
+      "kl": 0.0595703125,
+      "learning_rate": 4.852941176470588e-07,
+      "loss": 0.0024,
+      "reward": 1.9265812635421753,
+      "reward_std": 0.013946098275482655,
+      "rewards/accuracy_reward": 0.9265811443328857,
+      "rewards/format_reward": 1.0,
+      "step": 700
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 1360,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 52,
+  "trial_name": null,
+  "trial_params": null
+}