diff --git "a/checkpoint-2811/trainer_state.json" "b/checkpoint-2811/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-2811/trainer_state.json"
@@ -0,0 +1,3686 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.997333333333333,
+  "eval_steps": 10,
+  "global_step": 2811,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 121.971875,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 0.156667098402977,
+      "kl": 2.0313262939453126e-05,
+      "learning_rate": 1.0638297872340426e-05,
+      "loss": 0.001,
+      "reward": 0.0125,
+      "reward_std": 0.025,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.003125,
+      "step": 10
+    },
+    {
+      "completion_length": 122.521875,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 0.0012713409960269928,
+      "kl": 0.00021836161613464355,
+      "learning_rate": 2.1276595744680852e-05,
+      "loss": 0.0051,
+      "reward": 0.015625,
+      "reward_std": 0.025966878235340118,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.0,
+      "step": 20
+    },
+    {
+      "completion_length": 117.5125,
+      "epoch": 0.032,
+      "grad_norm": 0.002654253738000989,
+      "kl": 0.0003068089485168457,
+      "learning_rate": 3.1914893617021275e-05,
+      "loss": -0.0002,
+      "reward": 0.00625,
+      "reward_std": 0.007216878235340118,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.0,
+      "step": 30
+    },
+    {
+      "completion_length": 118.871875,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 0.00353299081325531,
+      "kl": 0.000412750244140625,
+      "learning_rate": 4.2553191489361704e-05,
+      "loss": 0.0055,
+      "reward": 0.009375,
+      "reward_std": 0.01875,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.003125,
+      "step": 40
+    },
+    {
+      "completion_length": 121.046875,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.003619612194597721,
+      "kl": 0.0004070043563842773,
+      "learning_rate": 5.319148936170213e-05,
+      "loss": 0.0,
+      "reward": 0.0,
+      "reward_std": 0.0,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/format_reward": 0.0,
+      "step": 50
+    },
+    {
+      "completion_length": 119.4375,
+      "epoch": 0.064,
+      "grad_norm": 0.11174867302179337,
+      "kl": 0.00045168399810791016,
+      "learning_rate": 6.382978723404255e-05,
+      "loss": 0.0064,
+      "reward": 0.01875,
+      "reward_std": 0.0375,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.003125,
+      "step": 60
+    },
+    {
+      "completion_length": 119.05625,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 0.006828859448432922,
+      "kl": 0.0011888980865478516,
+      "learning_rate": 7.446808510638297e-05,
+      "loss": 0.0012,
+      "reward": 0.0125,
+      "reward_std": 0.025,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "completion_length": 120.28125,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.0064537739381194115,
+      "kl": 0.0019659996032714844,
+      "learning_rate": 8.510638297872341e-05,
+      "loss": 0.0028,
+      "reward": 0.0125,
+      "reward_std": 0.025,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "completion_length": 117.559375,
+      "epoch": 0.096,
+      "grad_norm": 0.09068689495325089,
+      "kl": 0.0025023460388183595,
+      "learning_rate": 9.574468085106382e-05,
+      "loss": 0.003,
+      "reward": 0.021875,
+      "reward_std": 0.04375,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.003125,
+      "step": 90
+    },
+    {
+      "completion_length": 117.68125,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.16541939973831177,
+      "kl": 0.00291900634765625,
+      "learning_rate": 0.00010638297872340425,
+      "loss": 0.0008,
+      "reward": 0.021875,
+      "reward_std": 0.03125,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.0,
+      "step": 100
+    },
+    {
+      "completion_length": 116.971875,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 0.07206544280052185,
+      "kl": 0.0038990020751953126,
+      "learning_rate": 0.00011702127659574467,
+      "loss": 0.0026,
+      "reward": 0.015625,
+      "reward_std": 0.03125,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.003125,
+      "step": 110
+    },
+    {
+      "completion_length": 114.996875,
+      "epoch": 0.128,
+      "grad_norm": 0.02286006510257721,
+      "kl": 0.007346725463867188,
+      "learning_rate": 0.0001276595744680851,
+      "loss": 0.0076,
+      "reward": 0.025,
+      "reward_std": 0.05,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.009375,
+      "step": 120
+    },
+    {
+      "completion_length": 119.315625,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 0.015629781410098076,
+      "kl": 0.008090972900390625,
+      "learning_rate": 0.00013829787234042552,
+      "loss": 0.0011,
+      "reward": 0.009375,
+      "reward_std": 0.01875,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.0,
+      "step": 130
+    },
+    {
+      "completion_length": 121.821875,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 0.15498439967632294,
+      "kl": 0.006272506713867187,
+      "learning_rate": 0.00014893617021276593,
+      "loss": -0.0012,
+      "reward": 0.021875,
+      "reward_std": 0.03846687823534012,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.003125,
+      "step": 140
+    },
+    {
+      "completion_length": 121.409375,
+      "epoch": 0.16,
+      "grad_norm": 0.18756870925426483,
+      "kl": 0.00465240478515625,
+      "learning_rate": 0.00015957446808510637,
+      "loss": 0.0012,
+      "reward": 0.021875,
+      "reward_std": 0.03846687823534012,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.003125,
+      "step": 150
+    },
+    {
+      "completion_length": 118.26875,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.011626984924077988,
+      "kl": 0.01092681884765625,
+      "learning_rate": 0.00017021276595744682,
+      "loss": -0.0011,
+      "reward": 0.021875,
+      "reward_std": 0.03318375647068024,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.003125,
+      "step": 160
+    },
+    {
+      "completion_length": 119.25,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.00764912273734808,
+      "kl": 0.00976104736328125,
+      "learning_rate": 0.0001808510638297872,
+      "loss": 0.0045,
+      "reward": 0.021875,
+      "reward_std": 0.04375,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.0,
+      "step": 170
+    },
+    {
+      "completion_length": 115.35,
+      "epoch": 0.192,
+      "grad_norm": 0.0785018652677536,
+      "kl": 0.014077377319335938,
+      "learning_rate": 0.00019148936170212765,
+      "loss": 0.0037,
+      "reward": 0.025,
+      "reward_std": 0.04471687823534012,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.015625,
+      "step": 180
+    },
+    {
+      "completion_length": 108.6125,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 0.13107918202877045,
+      "kl": 0.039361572265625,
+      "learning_rate": 0.00020212765957446807,
+      "loss": 0.0412,
+      "reward": 0.11875,
+      "reward_std": 0.18907372057437896,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.096875,
+      "step": 190
+    },
+    {
+      "completion_length": 89.915625,
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.19012346863746643,
+      "kl": 0.08895263671875,
+      "learning_rate": 0.0002127659574468085,
+      "loss": 0.1321,
+      "reward": 0.46875,
+      "reward_std": 0.41404569447040557,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.453125,
+      "step": 200
+    },
+    {
+      "completion_length": 47.74375,
+      "epoch": 0.224,
+      "grad_norm": 0.4668453335762024,
+      "kl": 0.26416015625,
+      "learning_rate": 0.0002234042553191489,
+      "loss": 0.0712,
+      "reward": 0.871875,
+      "reward_std": 0.19805223047733306,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.853125,
+      "step": 210
+    },
+    {
+      "completion_length": 45.15625,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 0.21052278578281403,
+      "kl": 0.3112213134765625,
+      "learning_rate": 0.00023404255319148934,
+      "loss": 0.0464,
+      "reward": 0.890625,
+      "reward_std": 0.11346687823534012,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.878125,
+      "step": 220
+    },
+    {
+      "completion_length": 57.2875,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 0.16618619859218597,
+      "kl": 0.254522705078125,
+      "learning_rate": 0.00024468085106382976,
+      "loss": 0.0589,
+      "reward": 0.834375,
+      "reward_std": 0.12261751294136047,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.821875,
+      "step": 230
+    },
+    {
+      "completion_length": 68.153125,
+      "epoch": 0.256,
+      "grad_norm": 0.17739807069301605,
+      "kl": 0.214471435546875,
+      "learning_rate": 0.0002553191489361702,
+      "loss": 0.1375,
+      "reward": 0.659375,
+      "reward_std": 0.28527562469244006,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.65,
+      "step": 240
+    },
+    {
+      "completion_length": 52.709375,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.09843996912240982,
+      "kl": 0.2847900390625,
+      "learning_rate": 0.0002659574468085106,
+      "loss": 0.1085,
+      "reward": 0.834375,
+      "reward_std": 0.290549997985363,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.8,
+      "step": 250
+    },
+    {
+      "completion_length": 51.55625,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 0.1133696436882019,
+      "kl": 0.276953125,
+      "learning_rate": 0.00027659574468085103,
+      "loss": 0.0437,
+      "reward": 0.903125,
+      "reward_std": 0.1361730858683586,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.878125,
+      "step": 260
+    },
+    {
+      "completion_length": 55.046875,
+      "epoch": 0.288,
+      "grad_norm": 0.14536090195178986,
+      "kl": 0.2501953125,
+      "learning_rate": 0.0002872340425531915,
+      "loss": 0.0588,
+      "reward": 0.878125,
+      "reward_std": 0.13846687823534012,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.871875,
+      "step": 270
+    },
+    {
+      "completion_length": 55.853125,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 0.1799221634864807,
+      "kl": 0.3143310546875,
+      "learning_rate": 0.00029787234042553186,
+      "loss": 0.0609,
+      "reward": 0.90625,
+      "reward_std": 0.18080126941204072,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.884375,
+      "step": 280
+    },
+    {
+      "completion_length": 59.85,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 0.10688479989767075,
+      "kl": 0.20706787109375,
+      "learning_rate": 0.0002999925930442553,
+      "loss": 0.0522,
+      "reward": 0.815625,
+      "reward_std": 0.2959165498614311,
+      "rewards/accuracy_reward": 0.065625,
+      "rewards/format_reward": 0.75,
+      "step": 290
+    },
+    {
+      "completion_length": 64.625,
+      "epoch": 0.32,
+      "grad_norm": 0.03851361572742462,
+      "kl": 0.201220703125,
+      "learning_rate": 0.00029996250354024344,
+      "loss": 0.0815,
+      "reward": 0.8625,
+      "reward_std": 0.21301814764738083,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.85,
+      "step": 300
+    },
+    {
+      "completion_length": 57.95,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 0.23480646312236786,
+      "kl": 0.221240234375,
+      "learning_rate": 0.0002999092731927958,
+      "loss": 0.0292,
+      "reward": 0.921875,
+      "reward_std": 0.15895397514104842,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.884375,
+      "step": 310
+    },
+    {
+      "completion_length": 64.196875,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 0.1151675432920456,
+      "kl": 0.20123291015625,
+      "learning_rate": 0.0002998329102159332,
+      "loss": 0.0491,
+      "reward": 0.83125,
+      "reward_std": 0.19258119761943818,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.8125,
+      "step": 320
+    },
+    {
+      "completion_length": 70.628125,
+      "epoch": 0.352,
+      "grad_norm": 0.1377689391374588,
+      "kl": 0.1906005859375,
+      "learning_rate": 0.0002997334263932927,
+      "loss": 0.0841,
+      "reward": 0.846875,
+      "reward_std": 0.21890811175107955,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.83125,
+      "step": 330
+    },
+    {
+      "completion_length": 61.54375,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 0.0947548896074295,
+      "kl": 0.21240234375,
+      "learning_rate": 0.0002996108370763087,
+      "loss": 0.062,
+      "reward": 0.88125,
+      "reward_std": 0.13713996410369872,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.865625,
+      "step": 340
+    },
+    {
+      "completion_length": 60.109375,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.14599719643592834,
+      "kl": 0.2236083984375,
+      "learning_rate": 0.0002994651611818448,
+      "loss": 0.0408,
+      "reward": 0.928125,
+      "reward_std": 0.18282372057437896,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.9,
+      "step": 350
+    },
+    {
+      "completion_length": 62.90625,
+      "epoch": 0.384,
+      "grad_norm": 0.3738599121570587,
+      "kl": 0.2464111328125,
+      "learning_rate": 0.00029929642118927394,
+      "loss": 0.0753,
+      "reward": 0.834375,
+      "reward_std": 0.20676814764738083,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.809375,
+      "step": 360
+    },
+    {
+      "completion_length": 70.1625,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 2.8762810230255127,
+      "kl": 0.88681640625,
+      "learning_rate": 0.00029910464313701013,
+      "loss": 0.2053,
+      "reward": 0.640625,
+      "reward_std": 0.38192625939846037,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.63125,
+      "step": 370
+    },
+    {
+      "completion_length": 43.578125,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 0.9355350136756897,
+      "kl": 2.06865234375,
+      "learning_rate": 0.0002988898566184902,
+      "loss": 0.2631,
+      "reward": 0.725,
+      "reward_std": 0.32462068647146225,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/format_reward": 0.725,
+      "step": 380
+    },
+    {
+      "completion_length": 44.19375,
+      "epoch": 0.416,
+      "grad_norm": 0.709173858165741,
+      "kl": 3.47939453125,
+      "learning_rate": 0.0002986520947776074,
+      "loss": 0.3225,
+      "reward": 0.6125,
+      "reward_std": 0.3950331017374992,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.603125,
+      "step": 390
+    },
+    {
+      "completion_length": 54.775,
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.6549698114395142,
+      "kl": 4.3202392578125,
+      "learning_rate": 0.0002983913943035968,
+      "loss": 0.3808,
+      "reward": 0.66875,
+      "reward_std": 0.3901100158691406,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.65,
+      "step": 400
+    },
+    {
+      "completion_length": 53.871875,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 0.01826515607535839,
+      "kl": 2.477734375,
+      "learning_rate": 0.00029810779542537355,
+      "loss": 0.2661,
+      "reward": 0.79375,
+      "reward_std": 0.22999776750802994,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.76875,
+      "step": 410
+    },
+    {
+      "completion_length": 49.434375,
+      "epoch": 0.448,
+      "grad_norm": 0.5134692192077637,
+      "kl": 2.07587890625,
+      "learning_rate": 0.0002978013419053255,
+      "loss": 0.2091,
+      "reward": 0.771875,
+      "reward_std": 0.26785253882408144,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/format_reward": 0.771875,
+      "step": 420
+    },
+    {
+      "completion_length": 59.475,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 0.7835673689842224,
+      "kl": 2.516943359375,
+      "learning_rate": 0.00029747208103256,
+      "loss": 0.2312,
+      "reward": 0.740625,
+      "reward_std": 0.31220938116312025,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.728125,
+      "step": 430
+    },
+    {
+      "completion_length": 58.815625,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 0.021143430843949318,
+      "kl": 2.1959716796875,
+      "learning_rate": 0.0002971200636156068,
+      "loss": 0.2386,
+      "reward": 0.796875,
+      "reward_std": 0.2231356605887413,
+      "rewards/accuracy_reward": 0.0,
+      "rewards/format_reward": 0.796875,
+      "step": 440
+    },
+    {
+      "completion_length": 55.696875,
+      "epoch": 0.48,
+      "grad_norm": 1.8231980800628662,
+      "kl": 2.9314697265625,
+      "learning_rate": 0.00029674534397457745,
+      "loss": 0.3506,
+      "reward": 0.796875,
+      "reward_std": 0.2616912335157394,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.775,
+      "step": 450
+    },
+    {
+      "completion_length": 49.778125,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 0.5252532362937927,
+      "kl": 1.47607421875,
+      "learning_rate": 0.00029634797993278333,
+      "loss": 0.2026,
+      "reward": 0.89375,
+      "reward_std": 0.11293471753597259,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.88125,
+      "step": 460
+    },
+    {
+      "completion_length": 57.903125,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 0.10718824714422226,
+      "kl": 1.683837890625,
+      "learning_rate": 0.000295928032807813,
+      "loss": 0.1887,
+      "reward": 0.859375,
+      "reward_std": 0.1423343911767006,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.853125,
+      "step": 470
+    },
+    {
+      "completion_length": 63.734375,
+      "epoch": 0.512,
+      "grad_norm": 0.32101932168006897,
+      "kl": 2.9671875,
+      "learning_rate": 0.00029548556740206994,
+      "loss": 0.3254,
+      "reward": 0.79375,
+      "reward_std": 0.29874250292778015,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.784375,
+      "step": 480
+    },
+    {
+      "completion_length": 66.253125,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 0.7132259011268616,
+      "kl": 2.6101806640625,
+      "learning_rate": 0.0002950206519927731,
+      "loss": 0.2574,
+      "reward": 0.728125,
+      "reward_std": 0.3086773693561554,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.709375,
+      "step": 490
+    },
+    {
+      "completion_length": 64.765625,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.7293491959571838,
+      "kl": 3.3251220703125,
+      "learning_rate": 0.00029453335832142075,
+      "loss": 0.3315,
+      "reward": 0.75625,
+      "reward_std": 0.27030970752239225,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.746875,
+      "step": 500
+    },
+    {
+      "completion_length": 61.140625,
+      "epoch": 0.544,
+      "grad_norm": 0.4900813400745392,
+      "kl": 1.6069091796875,
+      "learning_rate": 0.0002940237615827202,
+      "loss": 0.162,
+      "reward": 0.86875,
+      "reward_std": 0.21899680644273758,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.815625,
+      "step": 510
+    },
+    {
+      "completion_length": 59.621875,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 0.4984245002269745,
+      "kl": 1.695751953125,
+      "learning_rate": 0.00029349194041298435,
+      "loss": 0.2075,
+      "reward": 0.903125,
+      "reward_std": 0.16081304997205734,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.86875,
+      "step": 520
+    },
+    {
+      "completion_length": 62.209375,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 0.25215986371040344,
+      "kl": 1.5575439453125,
+      "learning_rate": 0.0002929379768779971,
+      "loss": 0.1648,
+      "reward": 0.890625,
+      "reward_std": 0.17983439117670058,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.85,
+      "step": 530
+    },
+    {
+      "completion_length": 65.865625,
+      "epoch": 0.576,
+      "grad_norm": 0.1489488184452057,
+      "kl": 2.0063720703125,
+      "learning_rate": 0.0002923619564603501,
+      "loss": 0.187,
+      "reward": 0.78125,
+      "reward_std": 0.2043856605887413,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.75,
+      "step": 540
+    },
+    {
+      "completion_length": 68.68125,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 0.24991311132907867,
+      "kl": 1.0421142578125,
+      "learning_rate": 0.00029176396804625135,
+      "loss": 0.0977,
+      "reward": 0.909375,
+      "reward_std": 0.1441847175359726,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.8875,
+      "step": 550
+    },
+    {
+      "completion_length": 64.2,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 0.7193971872329712,
+      "kl": 2.2302490234375,
+      "learning_rate": 0.00029114410391180946,
+      "loss": 0.2166,
+      "reward": 0.834375,
+      "reward_std": 0.2048343911767006,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.81875,
+      "step": 560
+    },
+    {
+      "completion_length": 66.01875,
+      "epoch": 0.608,
+      "grad_norm": 0.59996098279953,
+      "kl": 2.652294921875,
+      "learning_rate": 0.0002905024597087945,
+      "loss": 0.2907,
+      "reward": 0.815625,
+      "reward_std": 0.20596464574337006,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.7875,
+      "step": 570
+    },
+    {
+      "completion_length": 69.009375,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.32363754510879517,
+      "kl": 0.75888671875,
+      "learning_rate": 0.0002898391344498775,
+      "loss": 0.112,
+      "reward": 0.896875,
+      "reward_std": 0.14761751294136047,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.8625,
+      "step": 580
+    },
+    {
+      "completion_length": 58.490625,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 0.5817243456840515,
+      "kl": 3.2273193359375,
+      "learning_rate": 0.0002891542304933521,
+      "loss": 0.3775,
+      "reward": 0.796875,
+      "reward_std": 0.24620190411806106,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.790625,
+      "step": 590
+    },
+    {
+      "completion_length": 56.890625,
+      "epoch": 0.64,
+      "grad_norm": 0.561817467212677,
+      "kl": 1.345703125,
+      "learning_rate": 0.00028844785352733924,
+      "loss": 0.1409,
+      "reward": 0.884375,
+      "reward_std": 0.1315855011343956,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.8625,
+      "step": 600
+    },
+    {
+      "completion_length": 54.61875,
+      "epoch": 0.6506666666666666,
+      "grad_norm": 0.41451311111450195,
+      "kl": 1.3966552734375,
+      "learning_rate": 0.00028772011255347873,
+      "loss": 0.1476,
+      "reward": 0.890625,
+      "reward_std": 0.16838996410369872,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.871875,
+      "step": 610
+    },
+    {
+      "completion_length": 58.15,
+      "epoch": 0.6613333333333333,
+      "grad_norm": 0.38927924633026123,
+      "kl": 2.2388427734375,
+      "learning_rate": 0.00028697111987010865,
+      "loss": 0.2576,
+      "reward": 0.871875,
+      "reward_std": 0.1995512694120407,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.84375,
+      "step": 620
+    },
+    {
+      "completion_length": 57.20625,
+      "epoch": 0.672,
+      "grad_norm": 0.09751415997743607,
+      "kl": 0.793994140625,
+      "learning_rate": 0.0002862009910549369,
+      "loss": 0.0629,
+      "reward": 0.9375,
+      "reward_std": 0.10386751294136047,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.91875,
+      "step": 630
+    },
+    {
+      "completion_length": 65.846875,
+      "epoch": 0.6826666666666666,
+      "grad_norm": 0.2675510048866272,
+      "kl": 2.466015625,
+      "learning_rate": 0.0002854098449472061,
+      "loss": 0.2627,
+      "reward": 0.79375,
+      "reward_std": 0.2520918682217598,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.778125,
+      "step": 640
+    },
+    {
+      "completion_length": 62.98125,
+      "epoch": 0.6933333333333334,
+      "grad_norm": 0.15855202078819275,
+      "kl": 1.8398193359375,
+      "learning_rate": 0.00028459780362935527,
+      "loss": 0.177,
+      "reward": 0.91875,
+      "reward_std": 0.15879059880971907,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.90625,
+      "step": 650
+    },
+    {
+      "completion_length": 62.6,
+      "epoch": 0.704,
+      "grad_norm": 0.12087615579366684,
+      "kl": 2.0813720703125,
+      "learning_rate": 0.0002837649924081816,
+      "loss": 0.1866,
+      "reward": 0.90625,
+      "reward_std": 0.19479155987501146,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.875,
+      "step": 660
+    },
+    {
+      "completion_length": 65.925,
+      "epoch": 0.7146666666666667,
+      "grad_norm": 0.39411771297454834,
+      "kl": 1.392919921875,
+      "learning_rate": 0.00028291153979550387,
+      "loss": 0.2015,
+      "reward": 0.915625,
+      "reward_std": 0.1775405988097191,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.8875,
+      "step": 670
+    },
+    {
+      "completion_length": 64.378125,
+      "epoch": 0.7253333333333334,
+      "grad_norm": 1.1659783124923706,
+      "kl": 2.8165283203125,
+      "learning_rate": 0.00028203757748833174,
+      "loss": 0.3109,
+      "reward": 0.778125,
+      "reward_std": 0.21169123351573943,
+      "rewards/accuracy_reward": 0.003125,
+      "rewards/format_reward": 0.775,
+      "step": 680
+    },
+    {
+      "completion_length": 57.334375,
+      "epoch": 0.736,
+      "grad_norm": 0.27627384662628174,
+      "kl": 1.0085693359375,
+      "learning_rate": 0.0002811432403485437,
+      "loss": 0.1226,
+      "reward": 0.859375,
+      "reward_std": 0.11540063470602036,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.85,
+      "step": 690
+    },
+    {
+      "completion_length": 54.6,
+      "epoch": 0.7466666666666667,
+      "grad_norm": 0.4506663382053375,
+      "kl": 2.3274169921875,
+      "learning_rate": 0.00028022866638207624,
+      "loss": 0.2726,
+      "reward": 0.853125,
+      "reward_std": 0.2143363133072853,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.828125,
+      "step": 700
+    },
+    {
+      "completion_length": 64.646875,
+      "epoch": 0.7573333333333333,
+      "grad_norm": 0.24161870777606964,
+      "kl": 1.10751953125,
+      "learning_rate": 0.00027929399671762793,
+      "loss": 0.1497,
+      "reward": 0.878125,
+      "reward_std": 0.18096464574337007,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.834375,
+      "step": 710
+    },
+    {
+      "completion_length": 65.128125,
+      "epoch": 0.768,
+      "grad_norm": 0.22652657330036163,
+      "kl": 1.7567138671875,
+      "learning_rate": 0.00027833937558488183,
+      "loss": 0.1692,
+      "reward": 0.865625,
+      "reward_std": 0.19575843811035157,
+      "rewards/accuracy_reward": 0.059375,
+      "rewards/format_reward": 0.80625,
+      "step": 720
+    },
+    {
+      "completion_length": 76.259375,
+      "epoch": 0.7786666666666666,
+      "grad_norm": 0.46417316794395447,
+      "kl": 3.4798828125,
+      "learning_rate": 0.0002773649502922495,
+      "loss": 0.3618,
+      "reward": 0.7125,
+      "reward_std": 0.31879488229751585,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.7,
+      "step": 730
+    },
+    {
+      "completion_length": 64.928125,
+      "epoch": 0.7893333333333333,
+      "grad_norm": 0.8743041753768921,
+      "kl": 2.166650390625,
+      "learning_rate": 0.00027637087120413933,
+      "loss": 0.2562,
+      "reward": 0.840625,
+      "reward_std": 0.2851921945810318,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.803125,
+      "step": 740
+    },
+    {
+      "completion_length": 58.028125,
+      "epoch": 0.8,
+      "grad_norm": 0.18655003607273102,
+      "kl": 1.73994140625,
+      "learning_rate": 0.000275357291717754,
+      "loss": 0.191,
+      "reward": 0.909375,
+      "reward_std": 0.19460364878177644,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.85625,
+      "step": 750
+    },
+    {
+      "completion_length": 60.803125,
+      "epoch": 0.8106666666666666,
+      "grad_norm": 0.04459076747298241,
+      "kl": 1.7782470703125,
+      "learning_rate": 0.0002743243682394195,
+      "loss": 0.2117,
+      "reward": 0.83125,
+      "reward_std": 0.16213996410369874,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.80625,
+      "step": 760
+    },
+    {
+      "completion_length": 56.203125,
+      "epoch": 0.8213333333333334,
+      "grad_norm": 0.04220689460635185,
+      "kl": 1.7406494140625,
+      "learning_rate": 0.00027327226016044963,
+      "loss": 0.1999,
+      "reward": 0.878125,
+      "reward_std": 0.1423343911767006,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.84375,
+      "step": 770
+    },
+    {
+      "completion_length": 58.375,
+      "epoch": 0.832,
+      "grad_norm": 0.3807085156440735,
+      "kl": 1.8222412109375,
+      "learning_rate": 0.00027220112983255087,
+      "loss": 0.2296,
+      "reward": 0.903125,
+      "reward_std": 0.20482564270496367,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.86875,
+      "step": 780
+    },
+    {
+      "completion_length": 63.378125,
+      "epoch": 0.8426666666666667,
+      "grad_norm": 0.01206011138856411,
+      "kl": 2.458740234375,
+      "learning_rate": 0.00027111114254276913,
+      "loss": 0.3096,
+      "reward": 0.84375,
+      "reward_std": 0.2114198923110962,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.821875,
+      "step": 790
+    },
+    {
+      "completion_length": 59.7625,
+      "epoch": 0.8533333333333334,
+      "grad_norm": 0.40591439604759216,
+      "kl": 1.378076171875,
+      "learning_rate": 0.00027000246648798456,
+      "loss": 0.1403,
+      "reward": 0.934375,
+      "reward_std": 0.14083535224199295,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.903125,
+      "step": 800
+    },
+    {
+      "completion_length": 62.284375,
+      "epoch": 0.864,
+      "grad_norm": 0.27511999011039734,
+      "kl": 2.2107177734375,
+      "learning_rate": 0.0002688752727489565,
+      "loss": 0.2636,
+      "reward": 0.8875,
+      "reward_std": 0.21739855110645295,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.859375,
+      "step": 810
+    },
+    {
+      "completion_length": 65.265625,
+      "epoch": 0.8746666666666667,
+      "grad_norm": 0.2582601010799408,
+      "kl": 2.3897705078125,
+      "learning_rate": 0.00026772973526392453,
+      "loss": 0.2965,
+      "reward": 0.83125,
+      "reward_std": 0.2494538262486458,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.803125,
+      "step": 820
+    },
+    {
+      "completion_length": 54.865625,
+      "epoch": 0.8853333333333333,
+      "grad_norm": 0.23494267463684082,
+      "kl": 2.6015625,
+      "learning_rate": 0.0002665660308017671,
+      "loss": 0.252,
+      "reward": 0.9,
+      "reward_std": 0.23950843811035155,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.85625,
+      "step": 830
+    },
+    {
+      "completion_length": 55.503125,
+      "epoch": 0.896,
+      "grad_norm": 0.20798054337501526,
+      "kl": 1.5889892578125,
+      "learning_rate": 0.000265384338934725,
+      "loss": 0.1996,
+      "reward": 0.9375,
+      "reward_std": 0.20120493620634078,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.875,
+      "step": 840
+    },
+    {
+      "completion_length": 59.909375,
+      "epoch": 0.9066666666666666,
+      "grad_norm": 0.23807695508003235,
+      "kl": 1.656982421875,
+      "learning_rate": 0.00026418484201069055,
+      "loss": 0.194,
+      "reward": 0.840625,
+      "reward_std": 0.17524680644273757,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.80625,
+      "step": 850
+    },
+    {
+      "completion_length": 56.74375,
+      "epoch": 0.9173333333333333,
+      "grad_norm": 0.21559438109397888,
+      "kl": 0.813427734375,
+      "learning_rate": 0.00026296772512507025,
+      "loss": 0.1054,
+      "reward": 0.884375,
+      "reward_std": 0.13916241526603698,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.8625,
+      "step": 860
+    },
+    {
+      "completion_length": 62.390625,
+      "epoch": 0.928,
+      "grad_norm": 0.1291944831609726,
+      "kl": 1.9663330078125,
+      "learning_rate": 0.0002617331760922218,
+      "loss": 0.2316,
+      "reward": 0.85625,
+      "reward_std": 0.15685684233903885,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.84375,
+      "step": 870
+    },
+    {
+      "completion_length": 56.009375,
+      "epoch": 0.9386666666666666,
+      "grad_norm": 1.045857548713684,
+      "kl": 1.652001953125,
+      "learning_rate": 0.0002604813854164726,
+      "loss": 0.1616,
+      "reward": 0.9375,
+      "reward_std": 0.16336943507194518,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.909375,
+      "step": 880
+    },
+    {
+      "completion_length": 64.446875,
+      "epoch": 0.9493333333333334,
+      "grad_norm": 0.33091413974761963,
+      "kl": 3.235400390625,
+      "learning_rate": 0.0002592125462627231,
+      "loss": 0.3973,
+      "reward": 0.796875,
+      "reward_std": 0.2716366216540337,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.78125,
+      "step": 890
+    },
+    {
+      "completion_length": 59.0,
+      "epoch": 0.96,
+      "grad_norm": 0.25974419713020325,
+      "kl": 1.752197265625,
+      "learning_rate": 0.00025792685442663877,
+      "loss": 0.1938,
+      "reward": 0.89375,
+      "reward_std": 0.1826515957713127,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.865625,
+      "step": 900
+    },
+    {
+      "completion_length": 57.4125,
+      "epoch": 0.9706666666666667,
+      "grad_norm": 0.2569887936115265,
+      "kl": 2.5720703125,
+      "learning_rate": 0.00025662450830443733,
+      "loss": 0.3213,
+      "reward": 0.846875,
+      "reward_std": 0.22065922170877456,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.815625,
+      "step": 910
+    },
+    {
+      "completion_length": 51.140625,
+      "epoch": 0.9813333333333333,
+      "grad_norm": 0.18798935413360596,
+      "kl": 1.6124267578125,
+      "learning_rate": 0.0002553057088622736,
+      "loss": 0.2214,
+      "reward": 0.925,
+      "reward_std": 0.1477062076330185,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.909375,
+      "step": 920
+    },
+    {
+      "completion_length": 53.921875,
+      "epoch": 0.992,
+      "grad_norm": 0.8309330940246582,
+      "kl": 1.651806640625,
+      "learning_rate": 0.0002539706596052286,
+      "loss": 0.1893,
+      "reward": 0.909375,
+      "reward_std": 0.16504059880971908,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.878125,
+      "step": 930
+    },
+    {
+      "completion_length": 59.96052631578947,
+      "epoch": 1.0021333333333333,
+      "grad_norm": 0.3670661151409149,
+      "kl": 3.8713250411184212,
+      "learning_rate": 0.000252619566545906,
+      "loss": 0.4192,
+      "reward": 0.7796052631578947,
+      "reward_std": 0.2917690135930714,
+      "rewards/accuracy_reward": 0.01644736842105263,
+      "rewards/format_reward": 0.7631578947368421,
+      "step": 940
+    },
+    {
+      "completion_length": 59.1125,
+      "epoch": 1.0128,
+      "grad_norm": 0.2787770926952362,
+      "kl": 2.920068359375,
+      "learning_rate": 0.0002512526381726427,
+      "loss": 0.4194,
+      "reward": 0.734375,
+      "reward_std": 0.3439827933907509,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.715625,
+      "step": 950
+    },
+    {
+      "completion_length": 57.63125,
+      "epoch": 1.0234666666666667,
+      "grad_norm": 0.15397749841213226,
+      "kl": 2.7442626953125,
+      "learning_rate": 0.00024987008541733663,
+      "loss": 0.3308,
+      "reward": 0.81875,
+      "reward_std": 0.24064744114875794,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.80625,
+      "step": 960
+    },
+    {
+      "completion_length": 51.134375,
+      "epoch": 1.0341333333333333,
+      "grad_norm": 0.41957736015319824,
+      "kl": 1.035986328125,
+      "learning_rate": 0.0002484721216228974,
+      "loss": 0.1489,
+      "reward": 0.9625,
+      "reward_std": 0.10561862289905548,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.940625,
+      "step": 970
+    },
+    {
+      "completion_length": 56.68125,
+      "epoch": 1.0448,
+      "grad_norm": 0.11578945815563202,
+      "kl": 1.0271240234375,
+      "learning_rate": 0.0002470589625103255,
+      "loss": 0.1162,
+      "reward": 0.9,
+      "reward_std": 0.13415063470602034,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.865625,
+      "step": 980
+    },
+    {
+      "completion_length": 55.071875,
+      "epoch": 1.0554666666666668,
+      "grad_norm": 0.006299301981925964,
+      "kl": 1.3626953125,
+      "learning_rate": 0.0002456308261454241,
+      "loss": 0.1452,
+      "reward": 0.909375,
+      "reward_std": 0.1264015957713127,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.8875,
+      "step": 990
+    },
+    {
+      "completion_length": 58.0875,
+      "epoch": 1.0661333333333334,
+      "grad_norm": 0.007178621832281351,
+      "kl": 1.4033203125,
+      "learning_rate": 0.00024418793290514906,
+      "loss": 0.1534,
+      "reward": 0.86875,
+      "reward_std": 0.13291241526603698,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.85,
+      "step": 1000
+    },
+    {
+      "completion_length": 58.828125,
+      "epoch": 1.0768,
+      "grad_norm": 0.1767469048500061,
+      "kl": 1.26591796875,
+      "learning_rate": 0.0002427305054436024,
+      "loss": 0.1309,
+      "reward": 0.915625,
+      "reward_std": 0.15447435528039932,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.88125,
+      "step": 1010
+    },
+    {
+      "completion_length": 60.309375,
+      "epoch": 1.0874666666666666,
+      "grad_norm": 0.20854564011096954,
+      "kl": 2.313330078125,
+      "learning_rate": 0.00024125876865767438,
+      "loss": 0.2191,
+      "reward": 0.90625,
+      "reward_std": 0.17595286518335343,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.853125,
+      "step": 1020
+    },
+    {
+      "completion_length": 60.540625,
+      "epoch": 1.0981333333333334,
+      "grad_norm": 0.13962095975875854,
+      "kl": 2.0788818359375,
+      "learning_rate": 0.0002397729496523396,
+      "loss": 0.226,
+      "reward": 0.890625,
+      "reward_std": 0.18282372057437896,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.865625,
+      "step": 1030
+    },
+    {
+      "completion_length": 58.89375,
+      "epoch": 1.1088,
+      "grad_norm": 0.25990164279937744,
+      "kl": 1.064794921875,
+      "learning_rate": 0.0002382732777056119,
+      "loss": 0.1602,
+      "reward": 0.915625,
+      "reward_std": 0.16433631330728532,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.890625,
+      "step": 1040
+    },
+    {
+      "completion_length": 58.478125,
+      "epoch": 1.1194666666666666,
+      "grad_norm": 0.21004174649715424,
+      "kl": 1.47158203125,
+      "learning_rate": 0.00023675998423316457,
+      "loss": 0.1682,
+      "reward": 0.953125,
+      "reward_std": 0.18810684233903885,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.903125,
+      "step": 1050
+    },
+    {
+      "completion_length": 59.39375,
+      "epoch": 1.1301333333333332,
+      "grad_norm": 0.08877279609441757,
+      "kl": 1.388037109375,
+      "learning_rate": 0.00023523330275262037,
+      "loss": 0.1636,
+      "reward": 0.9125,
+      "reward_std": 0.1637136846780777,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.875,
+      "step": 1060
+    },
+    {
+      "completion_length": 63.709375,
+      "epoch": 1.1408,
+      "grad_norm": 0.218344584107399,
+      "kl": 1.704052734375,
+      "learning_rate": 0.00023369346884751706,
+      "loss": 0.2163,
+      "reward": 0.884375,
+      "reward_std": 0.23439744114875793,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.8375,
+      "step": 1070
+    },
+    {
+      "completion_length": 60.93125,
+      "epoch": 1.1514666666666666,
+      "grad_norm": 0.24491117894649506,
+      "kl": 1.15849609375,
+      "learning_rate": 0.00023214072013095434,
+      "loss": 0.1445,
+      "reward": 0.925,
+      "reward_std": 0.14206304997205735,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.890625,
+      "step": 1080
+    },
+    {
+      "completion_length": 55.40625,
+      "epoch": 1.1621333333333332,
+      "grad_norm": 1.4203561544418335,
+      "kl": 1.4958740234375,
+      "learning_rate": 0.00023057529620892773,
+      "loss": 0.2111,
+      "reward": 0.946875,
+      "reward_std": 0.18555223047733307,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.90625,
+      "step": 1090
+    },
+    {
+      "completion_length": 61.6375,
+      "epoch": 1.1728,
+      "grad_norm": 0.07859649509191513,
+      "kl": 2.87578125,
+      "learning_rate": 0.00022899743864335462,
+      "loss": 0.3232,
+      "reward": 0.825,
+      "reward_std": 0.22410253882408143,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.8125,
+      "step": 1100
+    },
+    {
+      "completion_length": 56.74375,
+      "epoch": 1.1834666666666667,
+      "grad_norm": 0.6798639893531799,
+      "kl": 2.165380859375,
+      "learning_rate": 0.0002274073909147986,
+      "loss": 0.29,
+      "reward": 0.86875,
+      "reward_std": 0.17693375647068024,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.85625,
+      "step": 1110
+    },
+    {
+      "completion_length": 56.08125,
+      "epoch": 1.1941333333333333,
+      "grad_norm": 0.46122825145721436,
+      "kl": 2.6029052734375,
+      "learning_rate": 0.000225805398384898,
+      "loss": 0.2877,
+      "reward": 0.85625,
+      "reward_std": 0.15879059880971907,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.8375,
+      "step": 1120
+    },
+    {
+      "completion_length": 53.4875,
+      "epoch": 1.2048,
+      "grad_norm": 0.0872046947479248,
+      "kl": 1.120751953125,
+      "learning_rate": 0.0002241917082585036,
+      "loss": 0.1583,
+      "reward": 0.959375,
+      "reward_std": 0.16838996410369872,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.9125,
+      "step": 1130
+    },
+    {
+      "completion_length": 52.09375,
+      "epoch": 1.2154666666666667,
+      "grad_norm": 0.2135591208934784,
+      "kl": 1.0230712890625,
+      "learning_rate": 0.00022256656954553245,
+      "loss": 0.1191,
+      "reward": 0.9625,
+      "reward_std": 0.14858439117670058,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.90625,
+      "step": 1140
+    },
+    {
+      "completion_length": 50.771875,
+      "epoch": 1.2261333333333333,
+      "grad_norm": 0.2628862261772156,
+      "kl": 2.2327880859375,
+      "learning_rate": 0.00022093023302254295,
+      "loss": 0.2802,
+      "reward": 0.953125,
+      "reward_std": 0.19831304997205734,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.9,
+      "step": 1150
+    },
+    {
+      "completion_length": 55.653125,
+      "epoch": 1.2368000000000001,
+      "grad_norm": 0.01043323241174221,
+      "kl": 1.43515625,
+      "learning_rate": 0.0002192829511940371,
+      "loss": 0.216,
+      "reward": 0.871875,
+      "reward_std": 0.17604155987501144,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.840625,
+      "step": 1160
+    },
+    {
+      "completion_length": 56.01875,
+      "epoch": 1.2474666666666667,
+      "grad_norm": 0.30781543254852295,
+      "kl": 1.352490234375,
+      "learning_rate": 0.00021762497825349663,
+      "loss": 0.1604,
+      "reward": 0.875,
+      "reward_std": 0.18801814764738084,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.834375,
+      "step": 1170
+    },
+    {
+      "completion_length": 57.009375,
+      "epoch": 1.2581333333333333,
+      "grad_norm": 0.268877774477005,
+      "kl": 1.6961669921875,
+      "learning_rate": 0.00021595657004415777,
+      "loss": 0.2207,
+      "reward": 0.896875,
+      "reward_std": 0.18351925760507584,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.865625,
+      "step": 1180
+    },
+    {
+      "completion_length": 53.909375,
+      "epoch": 1.2688,
+      "grad_norm": 0.10004394501447678,
+      "kl": 1.055419921875,
+      "learning_rate": 0.00021427798401953233,
+      "loss": 0.1206,
+      "reward": 0.915625,
+      "reward_std": 0.10359617173671723,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.878125,
+      "step": 1190
+    },
+    {
+      "completion_length": 55.93125,
+      "epoch": 1.2794666666666665,
+      "grad_norm": 0.16535454988479614,
+      "kl": 2.3431884765625,
+      "learning_rate": 0.0002125894792036794,
+      "loss": 0.3288,
+      "reward": 0.903125,
+      "reward_std": 0.22568152397871016,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.865625,
+      "step": 1200
+    },
+    {
+      "completion_length": 55.09375,
+      "epoch": 1.2901333333333334,
+      "grad_norm": 0.19274021685123444,
+      "kl": 1.0988037109375,
+      "learning_rate": 0.0002108913161512354,
+      "loss": 0.1432,
+      "reward": 0.9375,
+      "reward_std": 0.12358439117670059,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.903125,
+      "step": 1210
+    },
+    {
+      "completion_length": 55.675,
+      "epoch": 1.3008,
+      "grad_norm": 0.15594810247421265,
+      "kl": 1.42451171875,
+      "learning_rate": 0.0002091837569072076,
+      "loss": 0.1693,
+      "reward": 0.94375,
+      "reward_std": 0.16636751294136048,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.896875,
+      "step": 1220
+    },
+    {
+      "completion_length": 56.88125,
+      "epoch": 1.3114666666666666,
+      "grad_norm": 0.3196319341659546,
+      "kl": 1.6291015625,
+      "learning_rate": 0.00020746706496653765,
+      "loss": 0.2144,
+      "reward": 0.915625,
+      "reward_std": 0.1927691087126732,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.86875,
+      "step": 1230
+    },
+    {
+      "completion_length": 58.7875,
+      "epoch": 1.3221333333333334,
+      "grad_norm": 0.13602705299854279,
+      "kl": 1.5197509765625,
+      "learning_rate": 0.00020574150523344152,
+      "loss": 0.1651,
+      "reward": 0.94375,
+      "reward_std": 0.16670301407575608,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.90625,
+      "step": 1240
+    },
+    {
+      "completion_length": 62.996875,
+      "epoch": 1.3328,
+      "grad_norm": 0.05853046849370003,
+      "kl": 1.815576171875,
+      "learning_rate": 0.00020400734398053186,
+      "loss": 0.1795,
+      "reward": 0.86875,
+      "reward_std": 0.19752006977796555,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.83125,
+      "step": 1250
+    },
+    {
+      "completion_length": 62.54375,
+      "epoch": 1.3434666666666666,
+      "grad_norm": 0.007495929021388292,
+      "kl": 1.3890869140625,
+      "learning_rate": 0.0002022648488077294,
+      "loss": 0.1695,
+      "reward": 0.884375,
+      "reward_std": 0.1775405988097191,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.85625,
+      "step": 1260
+    },
+    {
+      "completion_length": 61.278125,
+      "epoch": 1.3541333333333334,
+      "grad_norm": 0.1842016726732254,
+      "kl": 1.7712646484375,
+      "learning_rate": 0.0002005142886009691,
+      "loss": 0.2379,
+      "reward": 0.875,
+      "reward_std": 0.16706304997205734,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.853125,
+      "step": 1270
+    },
+    {
+      "completion_length": 62.025,
+      "epoch": 1.3648,
+      "grad_norm": 0.16039888560771942,
+      "kl": 1.9799560546875,
+      "learning_rate": 0.00019875593349070832,
+      "loss": 0.2323,
+      "reward": 0.91875,
+      "reward_std": 0.20685684233903884,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.871875,
+      "step": 1280
+    },
+    {
+      "completion_length": 61.0375,
+      "epoch": 1.3754666666666666,
+      "grad_norm": 0.15333615243434906,
+      "kl": 2.3615478515625,
+      "learning_rate": 0.0001969900548102427,
+      "loss": 0.2778,
+      "reward": 0.84375,
+      "reward_std": 0.20719234347343446,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.825,
+      "step": 1290
+    },
+    {
+      "completion_length": 58.328125,
+      "epoch": 1.3861333333333334,
+      "grad_norm": 0.07369455695152283,
+      "kl": 2.516748046875,
+      "learning_rate": 0.00019521692505383657,
+      "loss": 0.3136,
+      "reward": 0.85,
+      "reward_std": 0.19249776750802994,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.825,
+      "step": 1300
+    },
+    {
+      "completion_length": 55.35625,
+      "epoch": 1.3968,
+      "grad_norm": 0.2832612693309784,
+      "kl": 1.73642578125,
+      "learning_rate": 0.000193436817834674,
+      "loss": 0.2319,
+      "reward": 0.925,
+      "reward_std": 0.21794123351573944,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.890625,
+      "step": 1310
+    },
+    {
+      "completion_length": 59.0875,
+      "epoch": 1.4074666666666666,
+      "grad_norm": 0.24120619893074036,
+      "kl": 2.8723876953125,
+      "learning_rate": 0.0001916500078426373,
+      "loss": 0.3392,
+      "reward": 0.8375,
+      "reward_std": 0.22023502588272095,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.8125,
+      "step": 1320
+    },
+    {
+      "completion_length": 55.025,
+      "epoch": 1.4181333333333335,
+      "grad_norm": 0.18106360733509064,
+      "kl": 1.674365234375,
+      "learning_rate": 0.0001898567708019196,
+      "loss": 0.2313,
+      "reward": 0.88125,
+      "reward_std": 0.17693375647068024,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.8625,
+      "step": 1330
+    },
+    {
+      "completion_length": 55.15625,
+      "epoch": 1.4288,
+      "grad_norm": 0.1365566849708557,
+      "kl": 2.2543212890625,
+      "learning_rate": 0.00018805738342847727,
+      "loss": 0.3175,
+      "reward": 0.878125,
+      "reward_std": 0.2048343911767006,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.85,
+      "step": 1340
+    },
+    {
+      "completion_length": 55.5625,
+      "epoch": 1.4394666666666667,
+      "grad_norm": 0.10590548813343048,
+      "kl": 1.425244140625,
+      "learning_rate": 0.00018625212338733,
+      "loss": 0.1371,
+      "reward": 0.8875,
+      "reward_std": 0.10915063470602035,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.859375,
+      "step": 1350
+    },
+    {
+      "completion_length": 53.225,
+      "epoch": 1.4501333333333333,
+      "grad_norm": 0.15103192627429962,
+      "kl": 1.1132080078125,
+      "learning_rate": 0.00018444126924971387,
+      "loss": 0.1228,
+      "reward": 0.953125,
+      "reward_std": 0.11838996410369873,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.915625,
+      "step": 1360
+    },
+    {
+      "completion_length": 57.265625,
+      "epoch": 1.4607999999999999,
+      "grad_norm": 0.05684982240200043,
+      "kl": 1.13671875,
+      "learning_rate": 0.0001826251004500947,
+      "loss": 0.1437,
+      "reward": 0.9125,
+      "reward_std": 0.19223694801330565,
+      "rewards/accuracy_reward": 0.065625,
+      "rewards/format_reward": 0.846875,
+      "step": 1370
+    },
+    {
+      "completion_length": 59.325,
+      "epoch": 1.4714666666666667,
+      "grad_norm": 0.17307031154632568,
+      "kl": 1.8568115234375,
+      "learning_rate": 0.0001808038972430486,
+      "loss": 0.2279,
+      "reward": 0.871875,
+      "reward_std": 0.16398502588272096,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.853125,
+      "step": 1380
+    },
+    {
+      "completion_length": 59.646875,
+      "epoch": 1.4821333333333333,
+      "grad_norm": 0.007796150632202625,
+      "kl": 2.4506591796875,
+      "learning_rate": 0.00017897794066001524,
+      "loss": 0.2992,
+      "reward": 0.84375,
+      "reward_std": 0.20420301407575608,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.809375,
+      "step": 1390
+    },
+    {
+      "completion_length": 58.3,
+      "epoch": 1.4928,
+      "grad_norm": 0.1803148239850998,
+      "kl": 1.1746826171875,
+      "learning_rate": 0.00017714751246593197,
+      "loss": 0.1374,
+      "reward": 0.85625,
+      "reward_std": 0.09665063470602035,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.84375,
+      "step": 1400
+    },
+    {
+      "completion_length": 51.30625,
+      "epoch": 1.5034666666666667,
+      "grad_norm": 0.12448029220104218,
+      "kl": 1.7854248046875,
+      "learning_rate": 0.00017531289511575425,
+      "loss": 0.2174,
+      "reward": 0.940625,
+      "reward_std": 0.17032372057437897,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.896875,
+      "step": 1410
+    },
+    {
+      "completion_length": 50.446875,
+      "epoch": 1.5141333333333333,
+      "grad_norm": 0.1189781054854393,
+      "kl": 1.4447998046875,
+      "learning_rate": 0.0001734743717108699,
+      "loss": 0.1672,
+      "reward": 0.959375,
+      "reward_std": 0.13282372057437897,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.921875,
+      "step": 1420
+    },
+    {
+      "completion_length": 55.278125,
+      "epoch": 1.5248,
+      "grad_norm": 0.10093328356742859,
+      "kl": 1.5136474609375,
+      "learning_rate": 0.0001716322259554132,
+      "loss": 0.1768,
+      "reward": 0.96875,
+      "reward_std": 0.19523502588272096,
+      "rewards/accuracy_reward": 0.065625,
+      "rewards/format_reward": 0.903125,
+      "step": 1430
+    },
+    {
+      "completion_length": 59.934375,
+      "epoch": 1.5354666666666668,
+      "grad_norm": 0.2610551714897156,
+      "kl": 1.778369140625,
+      "learning_rate": 0.00016978674211248673,
+      "loss": 0.2314,
+      "reward": 0.84375,
+      "reward_std": 0.20420301407575608,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.8125,
+      "step": 1440
+    },
+    {
+      "completion_length": 54.96875,
+      "epoch": 1.5461333333333334,
+      "grad_norm": 0.19814546406269073,
+      "kl": 2.6193603515625,
+      "learning_rate": 0.00016793820496029623,
+      "loss": 0.3738,
+      "reward": 0.88125,
+      "reward_std": 0.2356409251689911,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.85625,
+      "step": 1450
+    },
+    {
+      "completion_length": 51.703125,
+      "epoch": 1.5568,
+      "grad_norm": 0.1247173473238945,
+      "kl": 2.2877197265625,
+      "learning_rate": 0.000166086899748206,
+      "loss": 0.236,
+      "reward": 0.940625,
+      "reward_std": 0.2197028651833534,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.8875,
+      "step": 1460
+    },
+    {
+      "completion_length": 51.096875,
+      "epoch": 1.5674666666666668,
+      "grad_norm": 0.16843904554843903,
+      "kl": 1.2693115234375,
+      "learning_rate": 0.0001642331121527223,
+      "loss": 0.1801,
+      "reward": 0.959375,
+      "reward_std": 0.14867308586835862,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.915625,
+      "step": 1470
+    },
+    {
+      "completion_length": 55.465625,
+      "epoch": 1.5781333333333334,
+      "grad_norm": 0.12530925869941711,
+      "kl": 1.403857421875,
+      "learning_rate": 0.0001623771282334099,
+      "loss": 0.1621,
+      "reward": 0.921875,
+      "reward_std": 0.17568152397871017,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.875,
+      "step": 1480
+    },
+    {
+      "completion_length": 58.23125,
+      "epoch": 1.5888,
+      "grad_norm": 0.016702894121408463,
+      "kl": 0.8815673828125,
+      "learning_rate": 0.00016051923438875035,
+      "loss": 0.0918,
+      "reward": 0.8875,
+      "reward_std": 0.0879347175359726,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.853125,
+      "step": 1490
+    },
+    {
+      "completion_length": 53.565625,
+      "epoch": 1.5994666666666668,
+      "grad_norm": 0.09671846032142639,
+      "kl": 0.7630126953125,
+      "learning_rate": 0.00015865971731194738,
+      "loss": 0.0861,
+      "reward": 0.98125,
+      "reward_std": 0.11971687823534012,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.940625,
+      "step": 1500
+    },
+    {
+      "completion_length": 57.234375,
+      "epoch": 1.6101333333333332,
+      "grad_norm": 0.16410210728645325,
+      "kl": 0.922265625,
+      "learning_rate": 0.00015679886394668707,
+      "loss": 0.1231,
+      "reward": 0.925,
+      "reward_std": 0.10915063470602035,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.896875,
+      "step": 1510
+    },
+    {
+      "completion_length": 95.821875,
+      "epoch": 1.6208,
+      "grad_norm": 0.09543804824352264,
+      "kl": 6.0548583984375,
+      "learning_rate": 0.00015493696144285935,
+      "loss": 0.4094,
+      "reward": 0.284375,
+      "reward_std": 0.2681046098470688,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.259375,
+      "step": 1520
+    },
+    {
+      "completion_length": 113.446875,
+      "epoch": 1.6314666666666666,
+      "grad_norm": 0.006384687032550573,
+      "kl": 0.34674072265625,
+      "learning_rate": 0.00015307429711224754,
+      "loss": 0.0323,
+      "reward": 0.05,
+      "reward_std": 0.08221687823534012,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.01875,
+      "step": 1530
+    },
+    {
+      "completion_length": 116.7,
+      "epoch": 1.6421333333333332,
+      "grad_norm": 0.03033365309238434,
+      "kl": 0.18465576171875,
+      "learning_rate": 0.0001512111583841933,
+      "loss": 0.0367,
+      "reward": 0.053125,
+      "reward_std": 0.10625,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.025,
+      "step": 1540
+    },
+    {
+      "completion_length": 117.84375,
+      "epoch": 1.6528,
+      "grad_norm": 0.007193129975348711,
+      "kl": 0.1654541015625,
+      "learning_rate": 0.00014934783276124278,
+      "loss": 0.0284,
+      "reward": 0.059375,
+      "reward_std": 0.08318375647068024,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.034375,
+      "step": 1550
+    },
+    {
+      "completion_length": 111.459375,
+      "epoch": 1.6634666666666666,
+      "grad_norm": 0.024860132485628128,
+      "kl": 0.17579345703125,
+      "learning_rate": 0.00014748460777478208,
+      "loss": 0.0751,
+      "reward": 0.13125,
+      "reward_std": 0.20580126941204072,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.121875,
+      "step": 1560
+    },
+    {
+      "completion_length": 91.90625,
+      "epoch": 1.6741333333333333,
+      "grad_norm": 0.07996781170368195,
+      "kl": 0.19302978515625,
+      "learning_rate": 0.00014562177094066812,
+      "loss": 0.1666,
+      "reward": 0.478125,
+      "reward_std": 0.42759600281715393,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.45625,
+      "step": 1570
+    },
+    {
+      "completion_length": 73.571875,
+      "epoch": 1.6848,
+      "grad_norm": 0.07270823419094086,
+      "kl": 0.2430419921875,
+      "learning_rate": 0.0001437596097148615,
+      "loss": 0.1744,
+      "reward": 0.76875,
+      "reward_std": 0.332449671626091,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.740625,
+      "step": 1580
+    },
+    {
+      "completion_length": 64.4875,
+      "epoch": 1.6954666666666667,
+      "grad_norm": 0.1185784786939621,
+      "kl": 0.27333984375,
+      "learning_rate": 0.00014189841144906926,
+      "loss": 0.1684,
+      "reward": 0.80625,
+      "reward_std": 0.28343056291341784,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.778125,
+      "step": 1590
+    },
+    {
+      "completion_length": 59.8375,
+      "epoch": 1.7061333333333333,
+      "grad_norm": 0.30180656909942627,
+      "kl": 0.5548828125,
+      "learning_rate": 0.00014003846334640323,
+      "loss": 0.2054,
+      "reward": 0.740625,
+      "reward_std": 0.29256718456745145,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.728125,
+      "step": 1600
+    },
+    {
+      "completion_length": 59.284375,
+      "epoch": 1.7168,
+      "grad_norm": 0.26433998346328735,
+      "kl": 2.827392578125,
+      "learning_rate": 0.00013818005241706145,
+      "loss": 0.469,
+      "reward": 0.70625,
+      "reward_std": 0.33853629529476165,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.7,
+      "step": 1610
+    },
+    {
+      "completion_length": 53.78125,
+      "epoch": 1.7274666666666667,
+      "grad_norm": 0.2971569299697876,
+      "kl": 3.8951171875,
+      "learning_rate": 0.00013632346543403947,
+      "loss": 0.451,
+      "reward": 0.81875,
+      "reward_std": 0.23273502588272094,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.80625,
+      "step": 1620
+    },
+    {
+      "completion_length": 53.903125,
+      "epoch": 1.7381333333333333,
+      "grad_norm": 0.0691404640674591,
+      "kl": 1.3744140625,
+      "learning_rate": 0.00013446898888887804,
+      "loss": 0.1657,
+      "reward": 0.93125,
+      "reward_std": 0.13943375647068024,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.915625,
+      "step": 1630
+    },
+    {
+      "completion_length": 57.153125,
+      "epoch": 1.7488000000000001,
+      "grad_norm": 0.03758076950907707,
+      "kl": 0.640966796875,
+      "learning_rate": 0.00013261690894745442,
+      "loss": 0.0775,
+      "reward": 0.884375,
+      "reward_std": 0.07596687823534012,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.871875,
+      "step": 1640
+    },
+    {
+      "completion_length": 58.0125,
+      "epoch": 1.7594666666666665,
+      "grad_norm": 0.009222053922712803,
+      "kl": 1.1447265625,
+      "learning_rate": 0.00013076751140582394,
+      "loss": 0.1472,
+      "reward": 0.88125,
+      "reward_std": 0.14963996410369873,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.83125,
+      "step": 1650
+    },
+    {
+      "completion_length": 55.746875,
+      "epoch": 1.7701333333333333,
+      "grad_norm": 0.029693789780139923,
+      "kl": 1.351123046875,
+      "learning_rate": 0.00012892108164611857,
+      "loss": 0.161,
+      "reward": 0.890625,
+      "reward_std": 0.14867308586835862,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.86875,
+      "step": 1660
+    },
+    {
+      "completion_length": 53.184375,
+      "epoch": 1.7808000000000002,
+      "grad_norm": 0.23443636298179626,
+      "kl": 1.46533203125,
+      "learning_rate": 0.00012707790459250904,
+      "loss": 0.1583,
+      "reward": 0.9625,
+      "reward_std": 0.14665063470602036,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.928125,
+      "step": 1670
+    },
+    {
+      "completion_length": 52.959375,
+      "epoch": 1.7914666666666665,
+      "grad_norm": 0.010363437235355377,
+      "kl": 1.73623046875,
+      "learning_rate": 0.0001252382646672384,
+      "loss": 0.1757,
+      "reward": 0.921875,
+      "reward_std": 0.16504059880971908,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.9,
+      "step": 1680
+    },
+    {
+      "completion_length": 55.971875,
+      "epoch": 1.8021333333333334,
+      "grad_norm": 0.0745161771774292,
+      "kl": 1.418701171875,
+      "learning_rate": 0.00012340244574673238,
+      "loss": 0.1882,
+      "reward": 0.91875,
+      "reward_std": 0.1851816728711128,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.871875,
+      "step": 1690
+    },
+    {
+      "completion_length": 57.903125,
+      "epoch": 1.8128,
+      "grad_norm": 0.09005508571863174,
+      "kl": 1.6597900390625,
+      "learning_rate": 0.000121570731117794,
+      "loss": 0.1998,
+      "reward": 0.8875,
+      "reward_std": 0.1477808892726898,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.86875,
+      "step": 1700
+    },
+    {
+      "completion_length": 58.240625,
+      "epoch": 1.8234666666666666,
+      "grad_norm": 0.09877178072929382,
+      "kl": 1.344873046875,
+      "learning_rate": 0.00011974340343388972,
+      "loss": 0.163,
+      "reward": 0.921875,
+      "reward_std": 0.18511751294136047,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.8875,
+      "step": 1710
+    },
+    {
+      "completion_length": 66.65,
+      "epoch": 1.8341333333333334,
+      "grad_norm": 0.09796544909477234,
+      "kl": 2.161865234375,
+      "learning_rate": 0.00011792074467153248,
+      "loss": 0.2649,
+      "reward": 0.81875,
+      "reward_std": 0.20843056291341783,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.784375,
+      "step": 1720
+    },
+    {
+      "completion_length": 65.9,
+      "epoch": 1.8448,
+      "grad_norm": 0.12498176097869873,
+      "kl": 2.451318359375,
+      "learning_rate": 0.00011610303608677008,
+      "loss": 0.3047,
+      "reward": 0.809375,
+      "reward_std": 0.268188039958477,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.76875,
+      "step": 1730
+    },
+    {
+      "completion_length": 61.346875,
+      "epoch": 1.8554666666666666,
+      "grad_norm": 0.40386486053466797,
+      "kl": 3.258935546875,
+      "learning_rate": 0.00011429055817178411,
+      "loss": 0.3857,
+      "reward": 0.846875,
+      "reward_std": 0.28369315564632414,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.81875,
+      "step": 1740
+    },
+    {
+      "completion_length": 58.859375,
+      "epoch": 1.8661333333333334,
+      "grad_norm": 0.12184485048055649,
+      "kl": 3.230029296875,
+      "learning_rate": 0.00011248359061160698,
+      "loss": 0.3751,
+      "reward": 0.86875,
+      "reward_std": 0.2617799282073975,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.8375,
+      "step": 1750
+    },
+    {
+      "completion_length": 57.69375,
+      "epoch": 1.8768,
+      "grad_norm": 0.1783817708492279,
+      "kl": 2.4540283203125,
+      "learning_rate": 0.00011068241224096347,
+      "loss": 0.2785,
+      "reward": 0.875,
+      "reward_std": 0.20430223047733306,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.85625,
+      "step": 1760
+    },
+    {
+      "completion_length": 64.459375,
+      "epoch": 1.8874666666666666,
+      "grad_norm": 0.038270145654678345,
+      "kl": 2.579345703125,
+      "learning_rate": 0.00010888730100124353,
+      "loss": 0.2952,
+      "reward": 0.815625,
+      "reward_std": 0.24689744114875795,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.790625,
+      "step": 1770
+    },
+    {
+      "completion_length": 61.403125,
+      "epoch": 1.8981333333333335,
+      "grad_norm": 0.07670488953590393,
+      "kl": 2.2768310546875,
+      "learning_rate": 0.00010709853389761286,
+      "loss": 0.3084,
+      "reward": 0.884375,
+      "reward_std": 0.23545301407575608,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.846875,
+      "step": 1780
+    },
+    {
+      "completion_length": 62.98125,
+      "epoch": 1.9088,
+      "grad_norm": 0.21110066771507263,
+      "kl": 3.0247314453125,
+      "learning_rate": 0.00010531638695626811,
+      "loss": 0.3866,
+      "reward": 0.8,
+      "reward_std": 0.2520918682217598,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.76875,
+      "step": 1790
+    },
+    {
+      "completion_length": 64.06875,
+      "epoch": 1.9194666666666667,
+      "grad_norm": 0.19934044778347015,
+      "kl": 4.1655029296875,
+      "learning_rate": 0.00010354113518184303,
+      "loss": 0.4661,
+      "reward": 0.784375,
+      "reward_std": 0.2908942475914955,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.75625,
+      "step": 1800
+    },
+    {
+      "completion_length": 57.765625,
+      "epoch": 1.9301333333333335,
+      "grad_norm": 0.07659115642309189,
+      "kl": 1.541015625,
+      "learning_rate": 0.000101773052514972,
+      "loss": 0.1994,
+      "reward": 0.865625,
+      "reward_std": 0.17630237936973572,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.840625,
+      "step": 1810
+    },
+    {
+      "completion_length": 58.328125,
+      "epoch": 1.9407999999999999,
+      "grad_norm": 0.053584493696689606,
+      "kl": 1.616796875,
+      "learning_rate": 0.00010001241179001836,
+      "loss": 0.2072,
+      "reward": 0.88125,
+      "reward_std": 0.15386751294136047,
+      "rewards/accuracy_reward": 0.0125,
+      "rewards/format_reward": 0.86875,
+      "step": 1820
+    },
+    {
+      "completion_length": 59.36875,
+      "epoch": 1.9514666666666667,
+      "grad_norm": 0.06573835760354996,
+      "kl": 1.567236328125,
+      "learning_rate": 9.825948469297301e-05,
+      "loss": 0.2048,
+      "reward": 0.921875,
+      "reward_std": 0.19004059880971907,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.86875,
+      "step": 1830
+    },
+    {
+      "completion_length": 55.890625,
+      "epoch": 1.9621333333333333,
+      "grad_norm": 0.18969739973545074,
+      "kl": 1.7246337890625,
+      "learning_rate": 9.651454171953012e-05,
+      "loss": 0.2224,
+      "reward": 0.946875,
+      "reward_std": 0.1664562076330185,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.90625,
+      "step": 1840
+    },
+    {
+      "completion_length": 60.59375,
+      "epoch": 1.9727999999999999,
+      "grad_norm": 0.04010459780693054,
+      "kl": 2.70771484375,
+      "learning_rate": 9.477785213334706e-05,
+      "loss": 0.3228,
+      "reward": 0.834375,
+      "reward_std": 0.22910557091236114,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.809375,
+      "step": 1850
+    },
+    {
+      "completion_length": 63.846875,
+      "epoch": 1.9834666666666667,
+      "grad_norm": 0.2044885903596878,
+      "kl": 3.81484375,
+      "learning_rate": 9.30496839244936e-05,
+      "loss": 0.4808,
+      "reward": 0.753125,
+      "reward_std": 0.307637582719326,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.725,
+      "step": 1860
+    },
+    {
+      "completion_length": 63.05,
+      "epoch": 1.9941333333333333,
+      "grad_norm": 0.32608747482299805,
+      "kl": 2.995166015625,
+      "learning_rate": 9.133030376809867e-05,
+      "loss": 0.4066,
+      "reward": 0.734375,
+      "reward_std": 0.31027562469244,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.725,
+      "step": 1870
+    },
+    {
+      "completion_length": 62.14473684210526,
+      "epoch": 2.0042666666666666,
+      "grad_norm": 0.19676542282104492,
+      "kl": 2.9421258223684212,
+      "learning_rate": 8.961997698319912e-05,
+      "loss": 0.3671,
+      "reward": 0.7861842105263158,
+      "reward_std": 0.24553519801089638,
+      "rewards/accuracy_reward": 0.013157894736842105,
+      "rewards/format_reward": 0.7730263157894737,
+      "step": 1880
+    },
+    {
+      "completion_length": 58.6125,
+      "epoch": 2.0149333333333335,
+      "grad_norm": 0.3623450994491577,
+      "kl": 1.9839111328125,
+      "learning_rate": 8.79189674917983e-05,
+      "loss": 0.2562,
+      "reward": 0.840625,
+      "reward_std": 0.20684282928705217,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.809375,
+      "step": 1890
+    },
+    {
+      "completion_length": 58.20625,
+      "epoch": 2.0256,
+      "grad_norm": 0.14830775558948517,
+      "kl": 1.731103515625,
+      "learning_rate": 8.622753777813978e-05,
+      "loss": 0.2133,
+      "reward": 0.909375,
+      "reward_std": 0.16573613584041597,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.846875,
+      "step": 1900
+    },
+    {
+      "completion_length": 55.796875,
+      "epoch": 2.0362666666666667,
+      "grad_norm": 0.1913866549730301,
+      "kl": 1.334716796875,
+      "learning_rate": 8.454594884820358e-05,
+      "loss": 0.1778,
+      "reward": 0.921875,
+      "reward_std": 0.14040063470602035,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.903125,
+      "step": 1910
+    },
+    {
+      "completion_length": 55.759375,
+      "epoch": 2.0469333333333335,
+      "grad_norm": 0.21011610329151154,
+      "kl": 1.82216796875,
+      "learning_rate": 8.287446018942971e-05,
+      "loss": 0.204,
+      "reward": 0.94375,
+      "reward_std": 0.16476925760507583,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.9125,
+      "step": 1920
+    },
+    {
+      "completion_length": 60.0,
+      "epoch": 2.0576,
+      "grad_norm": 0.28949812054634094,
+      "kl": 2.074462890625,
+      "learning_rate": 8.121332973067665e-05,
+      "loss": 0.2961,
+      "reward": 0.871875,
+      "reward_std": 0.19040063470602037,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.846875,
+      "step": 1930
+    },
+    {
+      "completion_length": 61.309375,
+      "epoch": 2.0682666666666667,
+      "grad_norm": 0.24213965237140656,
+      "kl": 2.480908203125,
+      "learning_rate": 7.956281380242029e-05,
+      "loss": 0.29,
+      "reward": 0.81875,
+      "reward_std": 0.21265811175107957,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.7875,
+      "step": 1940
+    },
+    {
+      "completion_length": 56.003125,
+      "epoch": 2.0789333333333335,
+      "grad_norm": 0.20885713398456573,
+      "kl": 2.0755615234375,
+      "learning_rate": 7.792316709719874e-05,
+      "loss": 0.287,
+      "reward": 0.925,
+      "reward_std": 0.22208535224199294,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.878125,
+      "step": 1950
+    },
+    {
+      "completion_length": 59.065625,
+      "epoch": 2.0896,
+      "grad_norm": 0.18546347320079803,
+      "kl": 1.8335693359375,
+      "learning_rate": 7.629464263031132e-05,
+      "loss": 0.2215,
+      "reward": 0.91875,
+      "reward_std": 0.1766484022140503,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.890625,
+      "step": 1960
+    },
+    {
+      "completion_length": 56.5,
+      "epoch": 2.1002666666666667,
+      "grad_norm": 0.25101250410079956,
+      "kl": 1.4716064453125,
+      "learning_rate": 7.46774917007749e-05,
+      "loss": 0.1931,
+      "reward": 0.9125,
+      "reward_std": 0.14136751294136046,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.890625,
+      "step": 1970
+    },
+    {
+      "completion_length": 59.453125,
+      "epoch": 2.1109333333333336,
+      "grad_norm": 0.2480248659849167,
+      "kl": 1.67666015625,
+      "learning_rate": 7.30719638525462e-05,
+      "loss": 0.1832,
+      "reward": 0.903125,
+      "reward_std": 0.1623033404350281,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.85625,
+      "step": 1980
+    },
+    {
+      "completion_length": 58.46875,
+      "epoch": 2.1216,
+      "grad_norm": 0.19903503358364105,
+      "kl": 1.7174560546875,
+      "learning_rate": 7.147830683601388e-05,
+      "loss": 0.1958,
+      "reward": 0.921875,
+      "reward_std": 0.20825843811035155,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.871875,
+      "step": 1990
+    },
+    {
+      "completion_length": 57.909375,
+      "epoch": 2.1322666666666668,
+      "grad_norm": 0.16613556444644928,
+      "kl": 2.1371826171875,
+      "learning_rate": 6.989676656976868e-05,
+      "loss": 0.3016,
+      "reward": 0.896875,
+      "reward_std": 0.20526910871267318,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.8625,
+      "step": 2000
+    },
+    {
+      "completion_length": 54.175,
+      "epoch": 2.142933333333333,
+      "grad_norm": 0.37461480498313904,
+      "kl": 1.8574951171875,
+      "learning_rate": 6.832758710265492e-05,
+      "loss": 0.2408,
+      "reward": 0.93125,
+      "reward_std": 0.17427992820739746,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.890625,
+      "step": 2010
+    },
+    {
+      "completion_length": 60.190625,
+      "epoch": 2.1536,
+      "grad_norm": 0.15081331133842468,
+      "kl": 2.0186279296875,
+      "learning_rate": 6.677101057611133e-05,
+      "loss": 0.229,
+      "reward": 0.8875,
+      "reward_std": 0.2087818503379822,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.83125,
+      "step": 2020
+    },
+    {
+      "completion_length": 56.49375,
+      "epoch": 2.164266666666667,
+      "grad_norm": 0.28891250491142273,
+      "kl": 1.998779296875,
+      "learning_rate": 6.522727718680623e-05,
+      "loss": 0.2448,
+      "reward": 0.875,
+      "reward_std": 0.18149680644273758,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.85,
+      "step": 2030
+    },
+    {
+      "completion_length": 55.95,
+      "epoch": 2.174933333333333,
+      "grad_norm": 0.2829490900039673,
+      "kl": 2.0568115234375,
+      "learning_rate": 6.36966251495719e-05,
+      "loss": 0.2872,
+      "reward": 0.884375,
+      "reward_std": 0.1923343911767006,
+      "rewards/accuracy_reward": 0.015625,
+      "rewards/format_reward": 0.86875,
+      "step": 2040
+    },
+    {
+      "completion_length": 56.1,
+      "epoch": 2.1856,
+      "grad_norm": 0.46314913034439087,
+      "kl": 1.8526611328125,
+      "learning_rate": 6.217929066064636e-05,
+      "loss": 0.2034,
+      "reward": 0.890625,
+      "reward_std": 0.15861847400665283,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.85,
+      "step": 2050
+    },
+    {
+      "completion_length": 52.740625,
+      "epoch": 2.196266666666667,
+      "grad_norm": 0.07802554219961166,
+      "kl": 1.4162841796875,
+      "learning_rate": 6.067550786122496e-05,
+      "loss": 0.174,
+      "reward": 0.953125,
+      "reward_std": 0.16124776750802994,
+      "rewards/accuracy_reward": 0.059375,
+      "rewards/format_reward": 0.89375,
+      "step": 2060
+    },
+    {
+      "completion_length": 53.54375,
+      "epoch": 2.206933333333333,
+      "grad_norm": 0.4088180959224701,
+      "kl": 1.8029052734375,
+      "learning_rate": 5.918550880133017e-05,
+      "loss": 0.2458,
+      "reward": 0.9125,
+      "reward_std": 0.20420301407575608,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.875,
+      "step": 2070
+    },
+    {
+      "completion_length": 54.8,
+      "epoch": 2.2176,
+      "grad_norm": 0.3543941378593445,
+      "kl": 1.8623046875,
+      "learning_rate": 5.77095234040039e-05,
+      "loss": 0.2642,
+      "reward": 0.884375,
+      "reward_std": 0.18361847400665282,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.853125,
+      "step": 2080
+    },
+    {
+      "completion_length": 54.496875,
+      "epoch": 2.228266666666667,
+      "grad_norm": 0.20835542678833008,
+      "kl": 1.9629638671875,
+      "learning_rate": 5.624777942982734e-05,
+      "loss": 0.2696,
+      "reward": 0.8625,
+      "reward_std": 0.17620493620634078,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.840625,
+      "step": 2090
+    },
+    {
+      "completion_length": 58.75625,
+      "epoch": 2.238933333333333,
+      "grad_norm": 0.39805445075035095,
+      "kl": 3.082763671875,
+      "learning_rate": 5.480050244177572e-05,
+      "loss": 0.4061,
+      "reward": 0.8375,
+      "reward_std": 0.28123598694801333,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.8,
+      "step": 2100
+    },
+    {
+      "completion_length": 54.8125,
+      "epoch": 2.2496,
+      "grad_norm": 0.24060776829719543,
+      "kl": 2.3712890625,
+      "learning_rate": 5.33679157704109e-05,
+      "loss": 0.3109,
+      "reward": 0.89375,
+      "reward_std": 0.1928578034043312,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.85,
+      "step": 2110
+    },
+    {
+      "completion_length": 58.071875,
+      "epoch": 2.2602666666666664,
+      "grad_norm": 0.22757568955421448,
+      "kl": 1.7650634765625,
+      "learning_rate": 5.195024047941955e-05,
+      "loss": 0.2339,
+      "reward": 0.859375,
+      "reward_std": 0.19197435528039933,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.821875,
+      "step": 2120
+    },
+    {
+      "completion_length": 56.8125,
+      "epoch": 2.2709333333333332,
+      "grad_norm": 0.1922132819890976,
+      "kl": 2.193359375,
+      "learning_rate": 5.054769533149998e-05,
+      "loss": 0.282,
+      "reward": 0.86875,
+      "reward_std": 0.20499776750802995,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.8375,
+      "step": 2130
+    },
+    {
+      "completion_length": 53.096875,
+      "epoch": 2.2816,
+      "grad_norm": 0.1918802708387375,
+      "kl": 1.8790283203125,
+      "learning_rate": 4.916049675460543e-05,
+      "loss": 0.2532,
+      "reward": 0.9125,
+      "reward_std": 0.19136751294136048,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.875,
+      "step": 2140
+    },
+    {
+      "completion_length": 56.621875,
+      "epoch": 2.2922666666666665,
+      "grad_norm": 0.1691223531961441,
+      "kl": 2.2410888671875,
+      "learning_rate": 4.7788858808546275e-05,
+      "loss": 0.3084,
+      "reward": 0.871875,
+      "reward_std": 0.2093140110373497,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.821875,
+      "step": 2150
+    },
+    {
+      "completion_length": 53.76875,
+      "epoch": 2.3029333333333333,
+      "grad_norm": 0.1904859095811844,
+      "kl": 2.6338623046875,
+      "learning_rate": 4.6432993151958546e-05,
+      "loss": 0.3819,
+      "reward": 0.884375,
+      "reward_std": 0.20359617173671724,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.85625,
+      "step": 2160
+    },
+    {
+      "completion_length": 52.234375,
+      "epoch": 2.3136,
+      "grad_norm": 0.20980538427829742,
+      "kl": 1.95830078125,
+      "learning_rate": 4.5093109009642855e-05,
+      "loss": 0.2513,
+      "reward": 0.915625,
+      "reward_std": 0.1976921945810318,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.875,
+      "step": 2170
+    },
+    {
+      "completion_length": 56.2375,
+      "epoch": 2.3242666666666665,
+      "grad_norm": 1.081332802772522,
+      "kl": 1.9156494140625,
+      "learning_rate": 4.376941314027819e-05,
+      "loss": 0.2811,
+      "reward": 0.86875,
+      "reward_std": 0.1936367705464363,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.8375,
+      "step": 2180
+    },
+    {
+      "completion_length": 50.08125,
+      "epoch": 2.3349333333333333,
+      "grad_norm": 0.263333261013031,
+      "kl": 2.2359375,
+      "learning_rate": 4.246210980451749e-05,
+      "loss": 0.3355,
+      "reward": 0.903125,
+      "reward_std": 0.20254059880971909,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.884375,
+      "step": 2190
+    },
+    {
+      "completion_length": 59.2125,
+      "epoch": 2.3456,
+      "grad_norm": 0.4202430248260498,
+      "kl": 3.7255859375,
+      "learning_rate": 4.117140073346749e-05,
+      "loss": 0.515,
+      "reward": 0.78125,
+      "reward_std": 0.2895918682217598,
+      "rewards/accuracy_reward": 0.009375,
+      "rewards/format_reward": 0.771875,
+      "step": 2200
+    },
+    {
+      "completion_length": 55.38125,
+      "epoch": 2.3562666666666665,
+      "grad_norm": 0.2832404673099518,
+      "kl": 2.9238525390625,
+      "learning_rate": 3.9897485097559684e-05,
+      "loss": 0.4053,
+      "reward": 0.875,
+      "reward_std": 0.2679324850440025,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.8375,
+      "step": 2210
+    },
+    {
+      "completion_length": 59.365625,
+      "epoch": 2.3669333333333333,
+      "grad_norm": 0.2592329978942871,
+      "kl": 2.598388671875,
+      "learning_rate": 3.864055947581605e-05,
+      "loss": 0.3084,
+      "reward": 0.815625,
+      "reward_std": 0.22225747704505922,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.790625,
+      "step": 2220
+    },
+    {
+      "completion_length": 53.565625,
+      "epoch": 2.3776,
+      "grad_norm": 0.12750284373760223,
+      "kl": 2.2623291015625,
+      "learning_rate": 3.740081782551477e-05,
+      "loss": 0.3018,
+      "reward": 0.9125,
+      "reward_std": 0.19629059880971908,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.884375,
+      "step": 2230
+    },
+    {
+      "completion_length": 55.828125,
+      "epoch": 2.3882666666666665,
+      "grad_norm": 0.12283790111541748,
+      "kl": 1.74697265625,
+      "learning_rate": 3.617845145226067e-05,
+      "loss": 0.2485,
+      "reward": 0.921875,
+      "reward_std": 0.18282372057437896,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.884375,
+      "step": 2240
+    },
+    {
+      "completion_length": 50.853125,
+      "epoch": 2.3989333333333334,
+      "grad_norm": 0.1259896457195282,
+      "kl": 1.4160400390625,
+      "learning_rate": 3.4973648980464446e-05,
+      "loss": 0.1784,
+      "reward": 0.928125,
+      "reward_std": 0.13282372057437897,
+      "rewards/accuracy_reward": 0.028125,
+      "rewards/format_reward": 0.9,
+      "step": 2250
+    },
+    {
+      "completion_length": 51.35,
+      "epoch": 2.4096,
+      "grad_norm": 0.1746770590543747,
+      "kl": 1.1815673828125,
+      "learning_rate": 3.378659632423616e-05,
+      "loss": 0.1853,
+      "reward": 0.9625,
+      "reward_std": 0.14136751294136046,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.915625,
+      "step": 2260
+    },
+    {
+      "completion_length": 50.85625,
+      "epoch": 2.4202666666666666,
+      "grad_norm": 0.2644580006599426,
+      "kl": 1.69404296875,
+      "learning_rate": 3.261747665869621e-05,
+      "loss": 0.2072,
+      "reward": 0.965625,
+      "reward_std": 0.16124776750802994,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.9125,
+      "step": 2270
+    },
+    {
+      "completion_length": 56.90625,
+      "epoch": 2.4309333333333334,
+      "grad_norm": 0.1671292930841446,
+      "kl": 1.673974609375,
+      "learning_rate": 3.146647039171002e-05,
+      "loss": 0.1965,
+      "reward": 0.85,
+      "reward_std": 0.1655640110373497,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.828125,
+      "step": 2280
+    },
+    {
+      "completion_length": 57.678125,
+      "epoch": 2.4416,
+      "grad_norm": 0.25073471665382385,
+      "kl": 1.8162353515625,
+      "learning_rate": 3.0333755136048664e-05,
+      "loss": 0.2599,
+      "reward": 0.859375,
+      "reward_std": 0.20060684233903886,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.809375,
+      "step": 2290
+    },
+    {
+      "completion_length": 54.340625,
+      "epoch": 2.4522666666666666,
+      "grad_norm": 0.2447560727596283,
+      "kl": 1.910791015625,
+      "learning_rate": 2.9219505681981524e-05,
+      "loss": 0.2393,
+      "reward": 0.91875,
+      "reward_std": 0.18528088927268982,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.86875,
+      "step": 2300
+    },
+    {
+      "completion_length": 54.85625,
+      "epoch": 2.4629333333333334,
+      "grad_norm": 0.5278003215789795,
+      "kl": 1.694287109375,
+      "learning_rate": 2.812389397030415e-05,
+      "loss": 0.2223,
+      "reward": 0.88125,
+      "reward_std": 0.1822168782353401,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.84375,
+      "step": 2310
+    },
+    {
+      "completion_length": 57.6375,
+      "epoch": 2.4736000000000002,
+      "grad_norm": 0.1791255623102188,
+      "kl": 2.213330078125,
+      "learning_rate": 2.7047089065805977e-05,
+      "loss": 0.2678,
+      "reward": 0.84375,
+      "reward_std": 0.1837905988097191,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.803125,
+      "step": 2320
+    },
+    {
+      "completion_length": 54.99375,
+      "epoch": 2.4842666666666666,
+      "grad_norm": 0.06923685222864151,
+      "kl": 2.011474609375,
+      "learning_rate": 2.5989257131181874e-05,
+      "loss": 0.2702,
+      "reward": 0.890625,
+      "reward_std": 0.17790063470602036,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.853125,
+      "step": 2330
+    },
+    {
+      "completion_length": 55.078125,
+      "epoch": 2.4949333333333334,
+      "grad_norm": 0.2437313050031662,
+      "kl": 1.8513427734375,
+      "learning_rate": 2.4950561401391188e-05,
+      "loss": 0.2786,
+      "reward": 0.878125,
+      "reward_std": 0.1824636846780777,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.846875,
+      "step": 2340
+    },
+    {
+      "completion_length": 59.078125,
+      "epoch": 2.5056000000000003,
+      "grad_norm": 0.07926956564188004,
+      "kl": 1.988330078125,
+      "learning_rate": 2.3931162158469085e-05,
+      "loss": 0.2581,
+      "reward": 0.846875,
+      "reward_std": 0.16882468163967132,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.809375,
+      "step": 2350
+    },
+    {
+      "completion_length": 50.0875,
+      "epoch": 2.5162666666666667,
+      "grad_norm": 0.059707965701818466,
+      "kl": 1.9446533203125,
+      "learning_rate": 2.2931216706793136e-05,
+      "loss": 0.2305,
+      "reward": 0.940625,
+      "reward_std": 0.1668909251689911,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.884375,
+      "step": 2360
+    },
+    {
+      "completion_length": 50.753125,
+      "epoch": 2.5269333333333335,
+      "grad_norm": 0.12831467390060425,
+      "kl": 1.78369140625,
+      "learning_rate": 2.1950879348809546e-05,
+      "loss": 0.2416,
+      "reward": 0.94375,
+      "reward_std": 0.2137136846780777,
+      "rewards/accuracy_reward": 0.059375,
+      "rewards/format_reward": 0.884375,
+      "step": 2370
+    },
+    {
+      "completion_length": 60.515625,
+      "epoch": 2.5376,
+      "grad_norm": 0.11030708253383636,
+      "kl": 1.88526611328125,
+      "learning_rate": 2.099030136122276e-05,
+      "loss": 0.2618,
+      "reward": 0.8375,
+      "reward_std": 0.17886751294136047,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.803125,
+      "step": 2380
+    },
+    {
+      "completion_length": 51.6125,
+      "epoch": 2.5482666666666667,
+      "grad_norm": 1.4983174800872803,
+      "kl": 1.6604248046875,
+      "learning_rate": 2.0049630971651576e-05,
+      "loss": 0.2423,
+      "reward": 0.96875,
+      "reward_std": 0.1933012694120407,
+      "rewards/accuracy_reward": 0.059375,
+      "rewards/format_reward": 0.909375,
+      "step": 2390
+    },
+    {
+      "completion_length": 54.178125,
+      "epoch": 2.558933333333333,
+      "grad_norm": 0.2519737482070923,
+      "kl": 2.282080078125,
+      "learning_rate": 1.9129013335756316e-05,
+      "loss": 0.3116,
+      "reward": 0.890625,
+      "reward_std": 0.23096464574337006,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.85625,
+      "step": 2400
+    },
+    {
+      "completion_length": 56.959375,
+      "epoch": 2.5696,
+      "grad_norm": 0.16114293038845062,
+      "kl": 1.4489013671875,
+      "learning_rate": 1.822859051483932e-05,
+      "loss": 0.1778,
+      "reward": 0.925,
+      "reward_std": 0.1691707044839859,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.86875,
+      "step": 2410
+    },
+    {
+      "completion_length": 53.259375,
+      "epoch": 2.5802666666666667,
+      "grad_norm": 0.11866087466478348,
+      "kl": 1.4989501953125,
+      "learning_rate": 1.7348501453923826e-05,
+      "loss": 0.1715,
+      "reward": 0.90625,
+      "reward_std": 0.1558012694120407,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.875,
+      "step": 2420
+    },
+    {
+      "completion_length": 54.046875,
+      "epoch": 2.590933333333333,
+      "grad_norm": 0.24277645349502563,
+      "kl": 1.7145751953125,
+      "learning_rate": 1.64888819603129e-05,
+      "loss": 0.2216,
+      "reward": 0.925,
+      "reward_std": 0.16935684233903886,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.890625,
+      "step": 2430
+    },
+    {
+      "completion_length": 57.071875,
+      "epoch": 2.6016,
+      "grad_norm": 0.04312370344996452,
+      "kl": 2.06767578125,
+      "learning_rate": 1.564986468263298e-05,
+      "loss": 0.2706,
+      "reward": 0.8875,
+      "reward_std": 0.19136751294136048,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.85625,
+      "step": 2440
+    },
+    {
+      "completion_length": 54.8625,
+      "epoch": 2.6122666666666667,
+      "grad_norm": 0.2207166850566864,
+      "kl": 2.34619140625,
+      "learning_rate": 1.4831579090364832e-05,
+      "loss": 0.2831,
+      "reward": 0.871875,
+      "reward_std": 0.20552992820739746,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.846875,
+      "step": 2450
+    },
+    {
+      "completion_length": 51.11875,
+      "epoch": 2.622933333333333,
+      "grad_norm": 0.2076699137687683,
+      "kl": 1.8322509765625,
+      "learning_rate": 1.4034151453864845e-05,
+      "loss": 0.2557,
+      "reward": 0.940625,
+      "reward_std": 0.20333535224199295,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.884375,
+      "step": 2460
+    },
+    {
+      "completion_length": 55.934375,
+      "epoch": 2.6336,
+      "grad_norm": 0.27051660418510437,
+      "kl": 2.1045166015625,
+      "learning_rate": 1.3257704824880377e-05,
+      "loss": 0.2683,
+      "reward": 0.915625,
+      "reward_std": 0.23369315564632415,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.853125,
+      "step": 2470
+    },
+    {
+      "completion_length": 54.821875,
+      "epoch": 2.6442666666666668,
+      "grad_norm": 0.32035326957702637,
+      "kl": 2.1305419921875,
+      "learning_rate": 1.2502359017561297e-05,
+      "loss": 0.2833,
+      "reward": 0.928125,
+      "reward_std": 0.22261751294136048,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.871875,
+      "step": 2480
+    },
+    {
+      "completion_length": 52.3625,
+      "epoch": 2.654933333333333,
+      "grad_norm": 0.0916447713971138,
+      "kl": 1.389013671875,
+      "learning_rate": 1.1768230589971455e-05,
+      "loss": 0.1857,
+      "reward": 0.95625,
+      "reward_std": 0.17930223047733307,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.8875,
+      "step": 2490
+    },
+    {
+      "completion_length": 55.6625,
+      "epoch": 2.6656,
+      "grad_norm": 0.23957239091396332,
+      "kl": 1.5188232421875,
+      "learning_rate": 1.1055432826102589e-05,
+      "loss": 0.223,
+      "reward": 0.946875,
+      "reward_std": 0.1785961717367172,
+      "rewards/accuracy_reward": 0.075,
+      "rewards/format_reward": 0.871875,
+      "step": 2500
+    },
+    {
+      "completion_length": 52.58125,
+      "epoch": 2.676266666666667,
+      "grad_norm": 0.14451664686203003,
+      "kl": 1.84560546875,
+      "learning_rate": 1.0364075718393172e-05,
+      "loss": 0.224,
+      "reward": 0.9625,
+      "reward_std": 0.22999776750802994,
+      "rewards/accuracy_reward": 0.084375,
+      "rewards/format_reward": 0.878125,
+      "step": 2510
+    },
+    {
+      "completion_length": 58.35625,
+      "epoch": 2.686933333333333,
+      "grad_norm": 0.18883152306079865,
+      "kl": 2.30576171875,
+      "learning_rate": 9.69426595075566e-06,
+      "loss": 0.3123,
+      "reward": 0.865625,
+      "reward_std": 0.20895397514104844,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.828125,
+      "step": 2520
+    },
+    {
+      "completion_length": 58.309375,
+      "epoch": 2.6976,
+      "grad_norm": 0.3094783127307892,
+      "kl": 1.9437255859375,
+      "learning_rate": 9.046106882113751e-06,
+      "loss": 0.2174,
+      "reward": 0.875,
+      "reward_std": 0.15157372057437896,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.8375,
+      "step": 2530
+    },
+    {
+      "completion_length": 51.671875,
+      "epoch": 2.708266666666667,
+      "grad_norm": 0.3006289601325989,
+      "kl": 2.19658203125,
+      "learning_rate": 8.419698530453095e-06,
+      "loss": 0.3126,
+      "reward": 0.91875,
+      "reward_std": 0.1887136846780777,
+      "rewards/accuracy_reward": 0.021875,
+      "rewards/format_reward": 0.896875,
+      "step": 2540
+    },
+    {
+      "completion_length": 51.45,
+      "epoch": 2.718933333333333,
+      "grad_norm": 0.3948608338832855,
+      "kl": 2.006982421875,
+      "learning_rate": 7.815137557387419e-06,
+      "loss": 0.2757,
+      "reward": 0.95,
+      "reward_std": 0.2034987285733223,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.909375,
+      "step": 2550
+    },
+    {
+      "completion_length": 51.890625,
+      "epoch": 2.7296,
+      "grad_norm": 0.13767310976982117,
+      "kl": 1.519189453125,
+      "learning_rate": 7.232517253242636e-06,
+      "loss": 0.1851,
+      "reward": 0.925,
+      "reward_std": 0.16177992820739745,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.89375,
+      "step": 2560
+    },
+    {
+      "completion_length": 53.684375,
+      "epoch": 2.740266666666667,
+      "grad_norm": 0.15098528563976288,
+      "kl": 2.033837890625,
+      "learning_rate": 6.671927522661047e-06,
+      "loss": 0.2932,
+      "reward": 0.975,
+      "reward_std": 0.2466260999441147,
+      "rewards/accuracy_reward": 0.0875,
+      "rewards/format_reward": 0.8875,
+      "step": 2570
+    },
+    {
+      "completion_length": 53.775,
+      "epoch": 2.7509333333333332,
+      "grad_norm": 0.1634114533662796,
+      "kl": 1.951416015625,
+      "learning_rate": 6.133454870728111e-06,
+      "loss": 0.2477,
+      "reward": 0.91875,
+      "reward_std": 0.198650324344635,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.875,
+      "step": 2580
+    },
+    {
+      "completion_length": 56.625,
+      "epoch": 2.7616,
+      "grad_norm": 0.1946033537387848,
+      "kl": 1.8022705078125,
+      "learning_rate": 5.617182389623814e-06,
+      "loss": 0.2218,
+      "reward": 0.93125,
+      "reward_std": 0.20193375647068024,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.8625,
+      "step": 2590
+    },
+    {
+      "completion_length": 55.571875,
+      "epoch": 2.772266666666667,
+      "grad_norm": 0.0797191932797432,
+      "kl": 2.0580810546875,
+      "learning_rate": 5.1231897458005765e-06,
+      "loss": 0.223,
+      "reward": 0.88125,
+      "reward_std": 0.2035074770450592,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.85,
+      "step": 2600
+    },
+    {
+      "completion_length": 58.39375,
+      "epoch": 2.7829333333333333,
+      "grad_norm": 0.2864690124988556,
+      "kl": 2.5972412109375,
+      "learning_rate": 4.651553167689931e-06,
+      "loss": 0.3517,
+      "reward": 0.88125,
+      "reward_std": 0.24293248504400253,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.825,
+      "step": 2610
+    },
+    {
+      "completion_length": 54.334375,
+      "epoch": 2.7936,
+      "grad_norm": 0.21106931567192078,
+      "kl": 1.7190673828125,
+      "learning_rate": 4.202345433939552e-06,
+      "loss": 0.2163,
+      "reward": 0.878125,
+      "reward_std": 0.19532372057437897,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.8375,
+      "step": 2620
+    },
+    {
+      "completion_length": 53.89375,
+      "epoch": 2.804266666666667,
+      "grad_norm": 0.21579229831695557,
+      "kl": 1.726171875,
+      "learning_rate": 3.7756358621827907e-06,
+      "loss": 0.2158,
+      "reward": 0.90625,
+      "reward_std": 0.1833471328020096,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.871875,
+      "step": 2630
+    },
+    {
+      "completion_length": 54.48125,
+      "epoch": 2.8149333333333333,
+      "grad_norm": 0.11399048566818237,
+      "kl": 1.6014404296875,
+      "learning_rate": 3.371490298342194e-06,
+      "loss": 0.1927,
+      "reward": 0.93125,
+      "reward_std": 0.16407372057437897,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.88125,
+      "step": 2640
+    },
+    {
+      "completion_length": 55.609375,
+      "epoch": 2.8256,
+      "grad_norm": 0.31620094180107117,
+      "kl": 2.4648193359375,
+      "learning_rate": 2.989971106468736e-06,
+      "loss": 0.3298,
+      "reward": 0.909375,
+      "reward_std": 0.24601925760507584,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.846875,
+      "step": 2650
+    },
+    {
+      "completion_length": 54.99375,
+      "epoch": 2.836266666666667,
+      "grad_norm": 0.3356408178806305,
+      "kl": 2.2318359375,
+      "learning_rate": 2.631137159118446e-06,
+      "loss": 0.2667,
+      "reward": 0.9,
+      "reward_std": 0.2262136846780777,
+      "rewards/accuracy_reward": 0.040625,
+      "rewards/format_reward": 0.859375,
+      "step": 2660
+    },
+    {
+      "completion_length": 55.634375,
+      "epoch": 2.8469333333333333,
+      "grad_norm": 0.6469197273254395,
+      "kl": 2.4311767578125,
+      "learning_rate": 2.2950438282676453e-06,
+      "loss": 0.3333,
+      "reward": 0.90625,
+      "reward_std": 0.25358215868473055,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.853125,
+      "step": 2670
+    },
+    {
+      "completion_length": 59.8375,
+      "epoch": 2.8576,
+      "grad_norm": 0.17483676970005035,
+      "kl": 2.5937255859375,
+      "learning_rate": 1.9817429767684466e-06,
+      "loss": 0.3489,
+      "reward": 0.8625,
+      "reward_std": 0.2525265857577324,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.8125,
+      "step": 2680
+    },
+    {
+      "completion_length": 55.5,
+      "epoch": 2.8682666666666665,
+      "grad_norm": 0.2512027621269226,
+      "kl": 2.55244140625,
+      "learning_rate": 1.6912829503458569e-06,
+      "loss": 0.3347,
+      "reward": 0.896875,
+      "reward_std": 0.24197435528039932,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.8625,
+      "step": 2690
+    },
+    {
+      "completion_length": 55.325,
+      "epoch": 2.8789333333333333,
+      "grad_norm": 0.16474415361881256,
+      "kl": 1.9730712890625,
+      "learning_rate": 1.4237085701374107e-06,
+      "loss": 0.2434,
+      "reward": 0.8875,
+      "reward_std": 0.22551814764738082,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.85,
+      "step": 2700
+    },
+    {
+      "completion_length": 55.084375,
+      "epoch": 2.8895999999999997,
+      "grad_norm": 0.06464667618274689,
+      "kl": 2.05751953125,
+      "learning_rate": 1.1790611257767868e-06,
+      "loss": 0.2714,
+      "reward": 0.925,
+      "reward_std": 0.2470608174800873,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.8625,
+      "step": 2710
+    },
+    {
+      "completion_length": 57.228125,
+      "epoch": 2.9002666666666665,
+      "grad_norm": 0.20830662548542023,
+      "kl": 2.081640625,
+      "learning_rate": 9.573783690224213e-07,
+      "loss": 0.2514,
+      "reward": 0.903125,
+      "reward_std": 0.22295301407575607,
+      "rewards/accuracy_reward": 0.059375,
+      "rewards/format_reward": 0.84375,
+      "step": 2720
+    },
+    {
+      "completion_length": 56.6375,
+      "epoch": 2.9109333333333334,
+      "grad_norm": 0.20990578830242157,
+      "kl": 1.5892578125,
+      "learning_rate": 7.586945079319673e-07,
+      "loss": 0.2132,
+      "reward": 0.9,
+      "reward_std": 0.1721687823534012,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.85,
+      "step": 2730
+    },
+    {
+      "completion_length": 58.346875,
+      "epoch": 2.9215999999999998,
+      "grad_norm": 0.010507356375455856,
+      "kl": 1.9132568359375,
+      "learning_rate": 5.830402015836178e-07,
+      "loss": 0.2316,
+      "reward": 0.853125,
+      "reward_std": 0.18854155987501145,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.821875,
+      "step": 2740
+    },
+    {
+      "completion_length": 57.3125,
+      "epoch": 2.9322666666666666,
+      "grad_norm": 0.4105237126350403,
+      "kl": 2.6902587890625,
+      "learning_rate": 4.304425553450896e-07,
+      "loss": 0.3641,
+      "reward": 0.865625,
+      "reward_std": 0.2506815239787102,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.834375,
+      "step": 2750
+    },
+    {
+      "completion_length": 53.96875,
+      "epoch": 2.9429333333333334,
+      "grad_norm": 0.11192090809345245,
+      "kl": 2.0924072265625,
+      "learning_rate": 3.009251166909699e-07,
+      "loss": 0.2969,
+      "reward": 0.928125,
+      "reward_std": 0.2324636846780777,
+      "rewards/accuracy_reward": 0.046875,
+      "rewards/format_reward": 0.88125,
+      "step": 2760
+    },
+    {
+      "completion_length": 57.19375,
+      "epoch": 2.9536,
+      "grad_norm": 0.16923871636390686,
+      "kl": 1.6264404296875,
+      "learning_rate": 1.9450787156907177e-07,
+      "loss": 0.209,
+      "reward": 0.865625,
+      "reward_std": 0.1672264263033867,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.834375,
+      "step": 2770
+    },
+    {
+      "completion_length": 55.79375,
+      "epoch": 2.9642666666666666,
+      "grad_norm": 0.19571039080619812,
+      "kl": 1.5086181640625,
+      "learning_rate": 1.1120724131638558e-07,
+      "loss": 0.186,
+      "reward": 0.90625,
+      "reward_std": 0.16512929350137712,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.86875,
+      "step": 2780
+    },
+    {
+      "completion_length": 54.446875,
+      "epoch": 2.9749333333333334,
+      "grad_norm": 0.2531106173992157,
+      "kl": 2.208544921875,
+      "learning_rate": 5.103608012512195e-08,
+      "loss": 0.2793,
+      "reward": 0.95625,
+      "reward_std": 0.2638788342475891,
+      "rewards/accuracy_reward": 0.0875,
+      "rewards/format_reward": 0.86875,
+      "step": 2790
+    },
+    {
+      "completion_length": 54.8875,
+      "epoch": 2.9856,
+      "grad_norm": 0.5443268418312073,
+      "kl": 2.21259765625,
+      "learning_rate": 1.400367305913197e-08,
+      "loss": 0.2949,
+      "reward": 0.89375,
+      "reward_std": 0.20729155987501144,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.859375,
+      "step": 2800
+    },
+    {
+      "completion_length": 56.165625,
+      "epoch": 2.9962666666666666,
+      "grad_norm": 0.1890791356563568,
+      "kl": 1.9867919921875,
+      "learning_rate": 1.1573462112002274e-10,
+      "loss": 0.2607,
+      "reward": 0.85625,
+      "reward_std": 0.19206304997205734,
+      "rewards/accuracy_reward": 0.034375,
+      "rewards/format_reward": 0.821875,
+      "step": 2810
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2811,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}