diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6542 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5714285714285714,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 2571.2083587646484,
+      "epoch": 0.001142857142857143,
+      "grad_norm": 0.19757072627544403,
+      "kl": 0.0,
+      "learning_rate": 2e-08,
+      "loss": -0.0,
+      "reward": 0.17825700528919697,
+      "reward_std": 0.804851658642292,
+      "rewards/cosine_scaled_reward": -0.015534311532974243,
+      "rewards/format_reward": 0.5208333488553762,
+      "step": 1
+    },
+    {
+      "completion_length": 2804.395881652832,
+      "epoch": 0.002285714285714286,
+      "grad_norm": 0.18212556838989258,
+      "kl": 0.0,
+      "learning_rate": 4e-08,
+      "loss": -0.0,
+      "reward": -0.1223274078220129,
+      "reward_std": 0.46937728114426136,
+      "rewards/cosine_scaled_reward": -0.04980122856795788,
+      "rewards/format_reward": 0.37500000558793545,
+      "step": 2
+    },
+    {
+      "completion_length": 3303.9583435058594,
+      "epoch": 0.0034285714285714284,
+      "grad_norm": 0.16496071219444275,
+      "kl": 4.756450653076172e-05,
+      "learning_rate": 6e-08,
+      "loss": 0.0,
+      "reward": -0.5349157964810729,
+      "reward_std": 0.4061080813407898,
+      "rewards/cosine_scaled_reward": -0.2544318288564682,
+      "rewards/format_reward": 0.1458333395421505,
+      "step": 3
+    },
+    {
+      "completion_length": 2260.6875228881836,
+      "epoch": 0.004571428571428572,
+      "grad_norm": 0.26921820640563965,
+      "kl": 3.6716461181640625e-05,
+      "learning_rate": 8e-08,
+      "loss": 0.0,
+      "reward": 0.1224252681422513,
+      "reward_std": 0.812014251947403,
+      "rewards/cosine_scaled_reward": -0.09193882904946804,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 4
+    },
+    {
+      "completion_length": 3346.6041870117188,
+      "epoch": 0.005714285714285714,
+      "grad_norm": 0.1722181737422943,
+      "kl": 4.376843571662903e-05,
+      "learning_rate": 1e-07,
+      "loss": 0.0,
+      "reward": -0.38172444701194763,
+      "reward_std": 0.5492917411029339,
+      "rewards/cosine_scaled_reward": -0.22456051781773567,
+      "rewards/format_reward": 0.25000000558793545,
+      "step": 5
+    },
+    {
+      "completion_length": 3113.7500610351562,
+      "epoch": 0.006857142857142857,
+      "grad_norm": 0.21919280290603638,
+      "kl": 4.5668333768844604e-05,
+      "learning_rate": 1.2e-07,
+      "loss": 0.0,
+      "reward": -0.2863283231854439,
+      "reward_std": 0.5716646872460842,
+      "rewards/cosine_scaled_reward": -0.19011332368245348,
+      "rewards/format_reward": 0.29166667349636555,
+      "step": 6
+    },
+    {
+      "completion_length": 3158.8333740234375,
+      "epoch": 0.008,
+      "grad_norm": 0.1657346487045288,
+      "kl": 2.4143606424331665e-05,
+      "learning_rate": 1.4e-07,
+      "loss": 0.0,
+      "reward": -0.0321456715464592,
+      "reward_std": 0.6449971524998546,
+      "rewards/cosine_scaled_reward": -0.1226729229092598,
+      "rewards/format_reward": 0.4791666828095913,
+      "step": 7
+    },
+    {
+      "completion_length": 2815.1250610351562,
+      "epoch": 0.009142857142857144,
+      "grad_norm": 0.15989142656326294,
+      "kl": 2.526119351387024e-05,
+      "learning_rate": 1.6e-07,
+      "loss": 0.0,
+      "reward": 0.3240444455295801,
+      "reward_std": 0.883681982755661,
+      "rewards/cosine_scaled_reward": 0.1182668274268508,
+      "rewards/format_reward": 0.5000000111758709,
+      "step": 8
+    },
+    {
+      "completion_length": 3149.0625915527344,
+      "epoch": 0.010285714285714285,
+      "grad_norm": 0.21666041016578674,
+      "kl": 4.5686960220336914e-05,
+      "learning_rate": 1.8e-07,
+      "loss": 0.0,
+      "reward": -0.18796737492084503,
+      "reward_std": 0.7943232320249081,
+      "rewards/cosine_scaled_reward": -0.13237779098562896,
+      "rewards/format_reward": 0.27083334140479565,
+      "step": 9
+    },
+    {
+      "completion_length": 2782.3750228881836,
+      "epoch": 0.011428571428571429,
+      "grad_norm": 0.18924005329608917,
+      "kl": 3.37064266204834e-05,
+      "learning_rate": 2e-07,
+      "loss": 0.0,
+      "reward": 0.11666052648797631,
+      "reward_std": 0.879204161465168,
+      "rewards/cosine_scaled_reward": -0.012820702977478504,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 10
+    },
+    {
+      "completion_length": 3473.062530517578,
+      "epoch": 0.012571428571428572,
+      "grad_norm": 0.20001095533370972,
+      "kl": 4.3779611587524414e-05,
+      "learning_rate": 2.1999999999999998e-07,
+      "loss": 0.0,
+      "reward": -0.6018680967390537,
+      "reward_std": 0.4545674379914999,
+      "rewards/cosine_scaled_reward": -0.28993429616093636,
+      "rewards/format_reward": 0.1041666679084301,
+      "step": 11
+    },
+    {
+      "completion_length": 2469.6667098999023,
+      "epoch": 0.013714285714285714,
+      "grad_norm": 0.24237319827079773,
+      "kl": 4.6290457248687744e-05,
+      "learning_rate": 2.4e-07,
+      "loss": 0.0,
+      "reward": 0.08407351560890675,
+      "reward_std": 0.751841738820076,
+      "rewards/cosine_scaled_reward": -0.1303967982530594,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 12
+    },
+    {
+      "completion_length": 2778.1458587646484,
+      "epoch": 0.014857142857142857,
+      "grad_norm": 0.18375596404075623,
+      "kl": 4.48375940322876e-05,
+      "learning_rate": 2.6e-07,
+      "loss": 0.0,
+      "reward": 0.09226825274527073,
+      "reward_std": 0.6979338899254799,
+      "rewards/cosine_scaled_reward": -0.015000073239207268,
+      "rewards/format_reward": 0.4166666865348816,
+      "step": 13
+    },
+    {
+      "completion_length": 2874.750045776367,
+      "epoch": 0.016,
+      "grad_norm": 0.1823539286851883,
+      "kl": 2.8234906494617462e-05,
+      "learning_rate": 2.8e-07,
+      "loss": 0.0,
+      "reward": -0.11271460726857185,
+      "reward_std": 0.7010148204863071,
+      "rewards/cosine_scaled_reward": -0.14169861702248454,
+      "rewards/format_reward": 0.39583333395421505,
+      "step": 14
+    },
+    {
+      "completion_length": 2797.395854949951,
+      "epoch": 0.017142857142857144,
+      "grad_norm": 0.20054183900356293,
+      "kl": 2.563674934208393e-05,
+      "learning_rate": 3e-07,
+      "loss": 0.0,
+      "reward": 0.018786390544846654,
+      "reward_std": 0.6112043932080269,
+      "rewards/cosine_scaled_reward": 0.03721225541085005,
+      "rewards/format_reward": 0.39583333767950535,
+      "step": 15
+    },
+    {
+      "completion_length": 3453.1458435058594,
+      "epoch": 0.018285714285714287,
+      "grad_norm": 0.18542714416980743,
+      "kl": 4.1812658309936523e-05,
+      "learning_rate": 3.2e-07,
+      "loss": 0.0,
+      "reward": -0.5245386594906449,
+      "reward_std": 0.5483577623963356,
+      "rewards/cosine_scaled_reward": -0.2334668217226863,
+      "rewards/format_reward": 0.06250000186264515,
+      "step": 16
+    },
+    {
+      "completion_length": 2326.8750610351562,
+      "epoch": 0.019428571428571427,
+      "grad_norm": 0.2586069405078888,
+      "kl": 3.917887806892395e-05,
+      "learning_rate": 3.4000000000000003e-07,
+      "loss": 0.0,
+      "reward": 0.29090012004598975,
+      "reward_std": 0.7025135271251202,
+      "rewards/cosine_scaled_reward": 0.04808543558465317,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 17
+    },
+    {
+      "completion_length": 2884.708366394043,
+      "epoch": 0.02057142857142857,
+      "grad_norm": 0.17255671322345734,
+      "kl": 2.2798776626586914e-05,
+      "learning_rate": 3.6e-07,
+      "loss": 0.0,
+      "reward": -0.06478883884847164,
+      "reward_std": 0.6950070075690746,
+      "rewards/cosine_scaled_reward": -0.09088864922523499,
+      "rewards/format_reward": 0.43750000558793545,
+      "step": 18
+    },
+    {
+      "completion_length": 2841.604202270508,
+      "epoch": 0.021714285714285714,
+      "grad_norm": 0.19047367572784424,
+      "kl": 3.0055642127990723e-05,
+      "learning_rate": 3.7999999999999996e-07,
+      "loss": 0.0,
+      "reward": 0.2590886615216732,
+      "reward_std": 0.9022833462804556,
+      "rewards/cosine_scaled_reward": 0.051947877276688814,
+      "rewards/format_reward": 0.4375000111758709,
+      "step": 19
+    },
+    {
+      "completion_length": 2353.1042289733887,
+      "epoch": 0.022857142857142857,
+      "grad_norm": 0.194220632314682,
+      "kl": 1.3179145753383636e-05,
+      "learning_rate": 4e-07,
+      "loss": 0.0,
+      "reward": 0.2806552555412054,
+      "reward_std": 0.8725750483572483,
+      "rewards/cosine_scaled_reward": 0.004965861327946186,
+      "rewards/format_reward": 0.6666666809469461,
+      "step": 20
+    },
+    {
+      "completion_length": 2684.687545776367,
+      "epoch": 0.024,
+      "grad_norm": 0.19551798701286316,
+      "kl": 4.254281520843506e-05,
+      "learning_rate": 4.1999999999999995e-07,
+      "loss": 0.0,
+      "reward": 0.09347914904356003,
+      "reward_std": 0.8260357603430748,
+      "rewards/cosine_scaled_reward": -0.07159630116075277,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 21
+    },
+    {
+      "completion_length": 1796.4375381469727,
+      "epoch": 0.025142857142857144,
+      "grad_norm": 0.3671523332595825,
+      "kl": 4.005804657936096e-05,
+      "learning_rate": 4.3999999999999997e-07,
+      "loss": 0.0,
+      "reward": 0.19362880755215883,
+      "reward_std": 0.8532019667327404,
+      "rewards/cosine_scaled_reward": -0.06471679511014372,
+      "rewards/format_reward": 0.7291666697710752,
+      "step": 22
+    },
+    {
+      "completion_length": 2500.062530517578,
+      "epoch": 0.026285714285714287,
+      "grad_norm": 0.22526901960372925,
+      "kl": 3.0137598514556885e-05,
+      "learning_rate": 4.6e-07,
+      "loss": 0.0,
+      "reward": 0.041429003700613976,
+      "reward_std": 0.7314254455268383,
+      "rewards/cosine_scaled_reward": -0.07520224852487445,
+      "rewards/format_reward": 0.5208333469927311,
+      "step": 23
+    },
+    {
+      "completion_length": 2648.291717529297,
+      "epoch": 0.027428571428571427,
+      "grad_norm": 0.21498121321201324,
+      "kl": 1.6693025827407837e-05,
+      "learning_rate": 4.8e-07,
+      "loss": 0.0,
+      "reward": 0.21239036042243242,
+      "reward_std": 0.6812088377773762,
+      "rewards/cosine_scaled_reward": 0.027016831561923027,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 24
+    },
+    {
+      "completion_length": 2769.2291946411133,
+      "epoch": 0.02857142857142857,
+      "grad_norm": 0.23084591329097748,
+      "kl": 3.578886389732361e-05,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "reward": 0.04653824120759964,
+      "reward_std": 0.7504720520228148,
+      "rewards/cosine_scaled_reward": -0.0376815393101424,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 25
+    },
+    {
+      "completion_length": 2962.562530517578,
+      "epoch": 0.029714285714285714,
+      "grad_norm": 0.16293752193450928,
+      "kl": 3.400072455406189e-05,
+      "learning_rate": 5.2e-07,
+      "loss": 0.0,
+      "reward": 0.010799127630889416,
+      "reward_std": 0.5875276923179626,
+      "rewards/cosine_scaled_reward": -0.043792182579636574,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 26
+    },
+    {
+      "completion_length": 2952.166717529297,
+      "epoch": 0.030857142857142857,
+      "grad_norm": 0.2215096801519394,
+      "kl": 3.3194024581462145e-05,
+      "learning_rate": 5.4e-07,
+      "loss": 0.0,
+      "reward": 0.11158058885484934,
+      "reward_std": 0.8504065573215485,
+      "rewards/cosine_scaled_reward": -0.05920940637588501,
+      "rewards/format_reward": 0.5208333469927311,
+      "step": 27
+    },
+    {
+      "completion_length": 2872.0625,
+      "epoch": 0.032,
+      "grad_norm": 0.18577344715595245,
+      "kl": 3.726780414581299e-05,
+      "learning_rate": 5.6e-07,
+      "loss": 0.0,
+      "reward": 0.13817322719842196,
+      "reward_std": 0.7248105835169554,
+      "rewards/cosine_scaled_reward": 0.027827581390738487,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 28
+    },
+    {
+      "completion_length": 3446.5833435058594,
+      "epoch": 0.03314285714285714,
+      "grad_norm": 0.21212224662303925,
+      "kl": 2.4769455194473267e-05,
+      "learning_rate": 5.8e-07,
+      "loss": 0.0,
+      "reward": -0.3688106779009104,
+      "reward_std": 0.6171325668692589,
+      "rewards/cosine_scaled_reward": -0.17023007571697235,
+      "rewards/format_reward": 0.14583333767950535,
+      "step": 29
+    },
+    {
+      "completion_length": 2783.3333587646484,
+      "epoch": 0.03428571428571429,
+      "grad_norm": 0.17819644510746002,
+      "kl": 2.261658664792776e-05,
+      "learning_rate": 6e-07,
+      "loss": 0.0,
+      "reward": 0.2445693917106837,
+      "reward_std": 0.9377000778913498,
+      "rewards/cosine_scaled_reward": 0.03347432427108288,
+      "rewards/format_reward": 0.4791666753590107,
+      "step": 30
+    },
+    {
+      "completion_length": 3050.2291870117188,
+      "epoch": 0.03542857142857143,
+      "grad_norm": 0.16932828724384308,
+      "kl": 2.38027423620224e-05,
+      "learning_rate": 6.2e-07,
+      "loss": 0.0,
+      "reward": -0.16898724623024464,
+      "reward_std": 0.648616686463356,
+      "rewards/cosine_scaled_reward": -0.11072872229851782,
+      "rewards/format_reward": 0.2708333395421505,
+      "step": 31
+    },
+    {
+      "completion_length": 3094.1250610351562,
+      "epoch": 0.036571428571428574,
+      "grad_norm": 0.18312151730060577,
+      "kl": 2.9257498681545258e-05,
+      "learning_rate": 6.4e-07,
+      "loss": 0.0,
+      "reward": 0.004837207496166229,
+      "reward_std": 0.6736676879227161,
+      "rewards/cosine_scaled_reward": -0.017333179406705312,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 32
+    },
+    {
+      "completion_length": 3368.562530517578,
+      "epoch": 0.037714285714285714,
+      "grad_norm": 0.15040728449821472,
+      "kl": 2.4586915969848633e-05,
+      "learning_rate": 6.6e-07,
+      "loss": 0.0,
+      "reward": -0.28359657526016235,
+      "reward_std": 0.6239799037575722,
+      "rewards/cosine_scaled_reward": -0.11568338703364134,
+      "rewards/format_reward": 0.1458333395421505,
+      "step": 33
+    },
+    {
+      "completion_length": 2505.1875610351562,
+      "epoch": 0.038857142857142854,
+      "grad_norm": 0.30060505867004395,
+      "kl": 2.2239633835852146e-05,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0,
+      "reward": 0.27598637342453003,
+      "reward_std": 0.8686297200620174,
+      "rewards/cosine_scaled_reward": 0.011391445528715849,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 34
+    },
+    {
+      "completion_length": 3063.2708740234375,
+      "epoch": 0.04,
+      "grad_norm": 0.24084553122520447,
+      "kl": 4.2811036109924316e-05,
+      "learning_rate": 7e-07,
+      "loss": 0.0,
+      "reward": -0.03728431276977062,
+      "reward_std": 0.9247638881206512,
+      "rewards/cosine_scaled_reward": -0.1015674127265811,
+      "rewards/format_reward": 0.3541666716337204,
+      "step": 35
+    },
+    {
+      "completion_length": 3358.7291870117188,
+      "epoch": 0.04114285714285714,
+      "grad_norm": 0.17074225842952728,
+      "kl": 2.5674700736999512e-05,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0,
+      "reward": -0.4219683278352022,
+      "reward_std": 0.516293577849865,
+      "rewards/cosine_scaled_reward": -0.20600515604019165,
+      "rewards/format_reward": 0.1875000074505806,
+      "step": 36
+    },
+    {
+      "completion_length": 3300.791717529297,
+      "epoch": 0.04228571428571429,
+      "grad_norm": 0.1499692052602768,
+      "kl": 1.6324222087860107e-05,
+      "learning_rate": 7.4e-07,
+      "loss": 0.0,
+      "reward": -0.42433968995464966,
+      "reward_std": 0.5183847993612289,
+      "rewards/cosine_scaled_reward": -0.21130944415926933,
+      "rewards/format_reward": 0.1875000074505806,
+      "step": 37
+    },
+    {
+      "completion_length": 3274.9166870117188,
+      "epoch": 0.04342857142857143,
+      "grad_norm": 0.18995541334152222,
+      "kl": 2.5459565222263336e-05,
+      "learning_rate": 7.599999999999999e-07,
+      "loss": 0.0,
+      "reward": -0.42816436290740967,
+      "reward_std": 0.3850083723664284,
+      "rewards/cosine_scaled_reward": -0.16101331263780594,
+      "rewards/format_reward": 0.14583333395421505,
+      "step": 38
+    },
+    {
+      "completion_length": 2845.81254196167,
+      "epoch": 0.044571428571428574,
+      "grad_norm": 0.16534960269927979,
+      "kl": 1.8164515495300293e-05,
+      "learning_rate": 7.799999999999999e-07,
+      "loss": 0.0,
+      "reward": -0.08653704356402159,
+      "reward_std": 0.5173388682305813,
+      "rewards/cosine_scaled_reward": -0.039690399542450905,
+      "rewards/format_reward": 0.3750000111758709,
+      "step": 39
+    },
+    {
+      "completion_length": 2536.583381652832,
+      "epoch": 0.045714285714285714,
+      "grad_norm": 0.25925570726394653,
+      "kl": 2.086721360683441e-05,
+      "learning_rate": 8e-07,
+      "loss": 0.0,
+      "reward": -0.0936742543708533,
+      "reward_std": 0.5705901933833957,
+      "rewards/cosine_scaled_reward": -0.09443798521533608,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 40
+    },
+    {
+      "completion_length": 2954.979217529297,
+      "epoch": 0.046857142857142854,
+      "grad_norm": 0.17119692265987396,
+      "kl": 5.511566996574402e-06,
+      "learning_rate": 8.199999999999999e-07,
+      "loss": 0.0,
+      "reward": -0.2362358495593071,
+      "reward_std": 0.6107039824128151,
+      "rewards/cosine_scaled_reward": -0.21764612046536058,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 41
+    },
+    {
+      "completion_length": 2852.250020980835,
+      "epoch": 0.048,
+      "grad_norm": 0.2615634500980377,
+      "kl": 3.533065319061279e-05,
+      "learning_rate": 8.399999999999999e-07,
+      "loss": 0.0,
+      "reward": -0.5097283767536283,
+      "reward_std": 0.4514521397650242,
+      "rewards/cosine_scaled_reward": -0.2993845697492361,
+      "rewards/format_reward": 0.27083333395421505,
+      "step": 42
+    },
+    {
+      "completion_length": 3095.4583435058594,
+      "epoch": 0.04914285714285714,
+      "grad_norm": 0.16259542107582092,
+      "kl": 2.100318670272827e-05,
+      "learning_rate": 8.599999999999999e-07,
+      "loss": 0.0,
+      "reward": -0.09295706450939178,
+      "reward_std": 0.6536407507956028,
+      "rewards/cosine_scaled_reward": -0.05758994724601507,
+      "rewards/format_reward": 0.25,
+      "step": 43
+    },
+    {
+      "completion_length": 2561.1875381469727,
+      "epoch": 0.05028571428571429,
+      "grad_norm": 0.2526821494102478,
+      "kl": 8.388608694076538e-05,
+      "learning_rate": 8.799999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.15549907088279724,
+      "reward_std": 0.8106728717684746,
+      "rewards/cosine_scaled_reward": -0.022572665475308895,
+      "rewards/format_reward": 0.5000000111758709,
+      "step": 44
+    },
+    {
+      "completion_length": 3464.729217529297,
+      "epoch": 0.05142857142857143,
+      "grad_norm": 0.14498470723628998,
+      "kl": 2.6337802410125732e-05,
+      "learning_rate": 9e-07,
+      "loss": 0.0,
+      "reward": -0.25844255089759827,
+      "reward_std": 0.6470912098884583,
+      "rewards/cosine_scaled_reward": -0.11992522329092026,
+      "rewards/format_reward": 0.1666666716337204,
+      "step": 45
+    },
+    {
+      "completion_length": 3175.437530517578,
+      "epoch": 0.052571428571428575,
+      "grad_norm": 0.18001262843608856,
+      "kl": 4.996359348297119e-05,
+      "learning_rate": 9.2e-07,
+      "loss": 0.0,
+      "reward": -0.4702068418264389,
+      "reward_std": 0.43945666775107384,
+      "rewards/cosine_scaled_reward": -0.22439202293753624,
+      "rewards/format_reward": 0.18750000186264515,
+      "step": 46
+    },
+    {
+      "completion_length": 2691.541763305664,
+      "epoch": 0.053714285714285714,
+      "grad_norm": 0.24902203679084778,
+      "kl": 2.5863759219646454e-05,
+      "learning_rate": 9.399999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.3736280268058181,
+      "reward_std": 0.9081602022051811,
+      "rewards/cosine_scaled_reward": 0.09569812752306461,
+      "rewards/format_reward": 0.5416666809469461,
+      "step": 47
+    },
+    {
+      "completion_length": 2735.2292251586914,
+      "epoch": 0.054857142857142854,
+      "grad_norm": 0.2746092677116394,
+      "kl": 0.0001163184642791748,
+      "learning_rate": 9.6e-07,
+      "loss": 0.0,
+      "reward": 0.02801407827064395,
+      "reward_std": 0.8248592298477888,
+      "rewards/cosine_scaled_reward": -0.05238310806453228,
+      "rewards/format_reward": 0.3958333358168602,
+      "step": 48
+    },
+    {
+      "completion_length": 2386.5416984558105,
+      "epoch": 0.056,
+      "grad_norm": 0.2179473340511322,
+      "kl": 5.142390727996826e-05,
+      "learning_rate": 9.8e-07,
+      "loss": 0.0,
+      "reward": 0.3018168299458921,
+      "reward_std": 0.9894729033112526,
+      "rewards/cosine_scaled_reward": 0.020154984667897224,
+      "rewards/format_reward": 0.5833333376795053,
+      "step": 49
+    },
+    {
+      "completion_length": 2974.583354949951,
+      "epoch": 0.05714285714285714,
+      "grad_norm": 0.19161240756511688,
+      "kl": 8.691102266311646e-05,
+      "learning_rate": 1e-06,
+      "loss": 0.0,
+      "reward": -0.04128398001194,
+      "reward_std": 0.6844424605369568,
+      "rewards/cosine_scaled_reward": -0.014122288441285491,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 50
+    },
+    {
+      "completion_length": 2252.375045776367,
+      "epoch": 0.05828571428571429,
+      "grad_norm": 0.22708459198474884,
+      "kl": 0.0002549290657043457,
+      "learning_rate": 9.999890338174275e-07,
+      "loss": 0.0,
+      "reward": 0.13623822387307882,
+      "reward_std": 0.7222620993852615,
+      "rewards/cosine_scaled_reward": -0.04494801629334688,
+      "rewards/format_reward": 0.5416666772216558,
+      "step": 51
+    },
+    {
+      "completion_length": 2870.2291870117188,
+      "epoch": 0.05942857142857143,
+      "grad_norm": 0.21368886530399323,
+      "kl": 0.0001485683023929596,
+      "learning_rate": 9.999561358041868e-07,
+      "loss": 0.0,
+      "reward": 0.015072201727889478,
+      "reward_std": 0.9205258414149284,
+      "rewards/cosine_scaled_reward": -0.06388110015541315,
+      "rewards/format_reward": 0.37500000186264515,
+      "step": 52
+    },
+    {
+      "completion_length": 2698.0000762939453,
+      "epoch": 0.060571428571428575,
+      "grad_norm": 0.2288151979446411,
+      "kl": 0.00016783177852630615,
+      "learning_rate": 9.999013075636804e-07,
+      "loss": 0.0,
+      "reward": 0.10895399999571964,
+      "reward_std": 0.8917515203356743,
+      "rewards/cosine_scaled_reward": -0.04874301888048649,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 53
+    },
+    {
+      "completion_length": 2874.979248046875,
+      "epoch": 0.061714285714285715,
+      "grad_norm": 0.16582155227661133,
+      "kl": 4.6576838940382004e-05,
+      "learning_rate": 9.998245517681593e-07,
+      "loss": 0.0,
+      "reward": 0.31593899679137394,
+      "reward_std": 0.9704541265964508,
+      "rewards/cosine_scaled_reward": 0.09179376903921366,
+      "rewards/format_reward": 0.5000000149011612,
+      "step": 54
+    },
+    {
+      "completion_length": 3011.875045776367,
+      "epoch": 0.06285714285714286,
+      "grad_norm": 0.16874325275421143,
+      "kl": 0.0001902114599943161,
+      "learning_rate": 9.997258721585931e-07,
+      "loss": 0.0,
+      "reward": 0.1326095201075077,
+      "reward_std": 0.8541341703385115,
+      "rewards/cosine_scaled_reward": 0.022567307110875845,
+      "rewards/format_reward": 0.45833333767950535,
+      "step": 55
+    },
+    {
+      "completion_length": 2914.1458892822266,
+      "epoch": 0.064,
+      "grad_norm": 0.17759209871292114,
+      "kl": 8.147954940795898e-05,
+      "learning_rate": 9.996052735444862e-07,
+      "loss": 0.0,
+      "reward": 0.021647373214364052,
+      "reward_std": 0.5984874460846186,
+      "rewards/cosine_scaled_reward": -0.05477431882172823,
+      "rewards/format_reward": 0.4375000111758709,
+      "step": 56
+    },
+    {
+      "completion_length": 3319.916717529297,
+      "epoch": 0.06514285714285714,
+      "grad_norm": 0.13513678312301636,
+      "kl": 6.243959069252014e-05,
+      "learning_rate": 9.994627618036452e-07,
+      "loss": 0.0,
+      "reward": -0.10464489553123713,
+      "reward_std": 0.9139657057821751,
+      "rewards/cosine_scaled_reward": -0.12102228635922074,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 57
+    },
+    {
+      "completion_length": 2223.3750534057617,
+      "epoch": 0.06628571428571428,
+      "grad_norm": 0.19678883254528046,
+      "kl": 0.0008740425109863281,
+      "learning_rate": 9.992983438818915e-07,
+      "loss": 0.0,
+      "reward": 0.3077916601905599,
+      "reward_std": 0.7720399703830481,
+      "rewards/cosine_scaled_reward": 0.0032341796904802322,
+      "rewards/format_reward": 0.6875000111758709,
+      "step": 58
+    },
+    {
+      "completion_length": 2857.1041870117188,
+      "epoch": 0.06742857142857143,
+      "grad_norm": 0.16988952457904816,
+      "kl": 7.66068696975708e-05,
+      "learning_rate": 9.991120277927223e-07,
+      "loss": 0.0,
+      "reward": -0.12906377390027046,
+      "reward_std": 0.5678401403129101,
+      "rewards/cosine_scaled_reward": -0.09338383004069328,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 59
+    },
+    {
+      "completion_length": 3053.0208740234375,
+      "epoch": 0.06857142857142857,
+      "grad_norm": 0.19087883830070496,
+      "kl": 0.00012252479791641235,
+      "learning_rate": 9.989038226169207e-07,
+      "loss": 0.0,
+      "reward": -0.33987870812416077,
+      "reward_std": 0.4983965791761875,
+      "rewards/cosine_scaled_reward": -0.22157840942963958,
+      "rewards/format_reward": 0.33333334140479565,
+      "step": 60
+    },
+    {
+      "completion_length": 2953.0625762939453,
+      "epoch": 0.06971428571428571,
+      "grad_norm": 0.1577925682067871,
+      "kl": 0.0002728104591369629,
+      "learning_rate": 9.98673738502114e-07,
+      "loss": 0.0,
+      "reward": 0.10354057513177395,
+      "reward_std": 0.8005717396736145,
+      "rewards/cosine_scaled_reward": -0.0372257842682302,
+      "rewards/format_reward": 0.4791666753590107,
+      "step": 61
+    },
+    {
+      "completion_length": 2714.104217529297,
+      "epoch": 0.07085714285714285,
+      "grad_norm": 0.1799556165933609,
+      "kl": 0.0007392987608909607,
+      "learning_rate": 9.98421786662277e-07,
+      "loss": 0.0,
+      "reward": 0.32593785908829886,
+      "reward_std": 0.9482499547302723,
+      "rewards/cosine_scaled_reward": 0.08002315112389624,
+      "rewards/format_reward": 0.562500013038516,
+      "step": 62
+    },
+    {
+      "completion_length": 2463.5625762939453,
+      "epoch": 0.072,
+      "grad_norm": 0.19159279763698578,
+      "kl": 0.0007972661405801773,
+      "learning_rate": 9.981479793771866e-07,
+      "loss": 0.0,
+      "reward": 0.4637246737256646,
+      "reward_std": 1.0199245177209377,
+      "rewards/cosine_scaled_reward": 0.1016120407730341,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 63
+    },
+    {
+      "completion_length": 2957.2709045410156,
+      "epoch": 0.07314285714285715,
+      "grad_norm": 0.19695256650447845,
+      "kl": 0.0005731135606765747,
+      "learning_rate": 9.97852329991824e-07,
+      "loss": 0.0,
+      "reward": -0.09675957635045052,
+      "reward_std": 0.7647779621183872,
+      "rewards/cosine_scaled_reward": -0.11780054681003094,
+      "rewards/format_reward": 0.35416667722165585,
+      "step": 64
+    },
+    {
+      "completion_length": 2748.2083473205566,
+      "epoch": 0.07428571428571429,
+      "grad_norm": 0.19534841179847717,
+      "kl": 0.0003612041473388672,
+      "learning_rate": 9.975348529157229e-07,
+      "loss": 0.0,
+      "reward": -0.20825218525715172,
+      "reward_std": 0.6059501096606255,
+      "rewards/cosine_scaled_reward": -0.18784814700484276,
+      "rewards/format_reward": 0.4166666679084301,
+      "step": 65
+    },
+    {
+      "completion_length": 2097.4375076293945,
+      "epoch": 0.07542857142857143,
+      "grad_norm": 0.250161349773407,
+      "kl": 0.0006571710109710693,
+      "learning_rate": 9.971955636222684e-07,
+      "loss": 0.0,
+      "reward": 0.0786643698811531,
+      "reward_std": 0.6597852855920792,
+      "rewards/cosine_scaled_reward": -0.006518872454762459,
+      "rewards/format_reward": 0.5,
+      "step": 66
+    },
+    {
+      "completion_length": 3430.4791870117188,
+      "epoch": 0.07657142857142857,
+      "grad_norm": 0.14664356410503387,
+      "kl": 0.0005048699676990509,
+      "learning_rate": 9.968344786479415e-07,
+      "loss": 0.0,
+      "reward": -0.5055637508630753,
+      "reward_std": 0.4412064775824547,
+      "rewards/cosine_scaled_reward": -0.2427280293777585,
+      "rewards/format_reward": 0.12500000558793545,
+      "step": 67
+    },
+    {
+      "completion_length": 2271.250045776367,
+      "epoch": 0.07771428571428571,
+      "grad_norm": 0.27284932136535645,
+      "kl": 0.0013322830200195312,
+      "learning_rate": 9.964516155915151e-07,
+      "loss": 0.0001,
+      "reward": 0.2981271520256996,
+      "reward_std": 0.9507227130234241,
+      "rewards/cosine_scaled_reward": 0.0009464251343160868,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 68
+    },
+    {
+      "completion_length": 2530.2708587646484,
+      "epoch": 0.07885714285714286,
+      "grad_norm": 0.22157438099384308,
+      "kl": 0.0010530054569244385,
+      "learning_rate": 9.960469931131936e-07,
+      "loss": 0.0,
+      "reward": -0.22882681945338845,
+      "reward_std": 0.6600410342216492,
+      "rewards/cosine_scaled_reward": -0.20334339328110218,
+      "rewards/format_reward": 0.41666667722165585,
+      "step": 69
+    },
+    {
+      "completion_length": 3004.7708587646484,
+      "epoch": 0.08,
+      "grad_norm": 0.21289457380771637,
+      "kl": 0.001432761549949646,
+      "learning_rate": 9.956206309337066e-07,
+      "loss": 0.0001,
+      "reward": -0.3504838487133384,
+      "reward_std": 0.5108464825898409,
+      "rewards/cosine_scaled_reward": -0.24259378435090184,
+      "rewards/format_reward": 0.3541666716337204,
+      "step": 70
+    },
+    {
+      "completion_length": 2526.937515258789,
+      "epoch": 0.08114285714285714,
+      "grad_norm": 0.21181254088878632,
+      "kl": 0.0007251240313053131,
+      "learning_rate": 9.951725498333448e-07,
+      "loss": 0.0,
+      "reward": -0.016672035679221153,
+      "reward_std": 0.5613718032836914,
+      "rewards/cosine_scaled_reward": -0.044779783114790916,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 71
+    },
+    {
+      "completion_length": 3029.979217529297,
+      "epoch": 0.08228571428571428,
+      "grad_norm": 0.20094312727451324,
+      "kl": 0.000997304916381836,
+      "learning_rate": 9.947027716509488e-07,
+      "loss": 0.0,
+      "reward": -0.21065808949060738,
+      "reward_std": 0.6206581741571426,
+      "rewards/cosine_scaled_reward": -0.12715522898361087,
+      "rewards/format_reward": 0.29166666977107525,
+      "step": 72
+    },
+    {
+      "completion_length": 3455.625,
+      "epoch": 0.08342857142857144,
+      "grad_norm": 0.13940556347370148,
+      "kl": 0.00026963651180267334,
+      "learning_rate": 9.942113192828444e-07,
+      "loss": 0.0,
+      "reward": -0.3948047012090683,
+      "reward_std": 0.596671599894762,
+      "rewards/cosine_scaled_reward": -0.1924935569986701,
+      "rewards/format_reward": 0.1458333395421505,
+      "step": 73
+    },
+    {
+      "completion_length": 3041.666717529297,
+      "epoch": 0.08457142857142858,
+      "grad_norm": 0.2121056467294693,
+      "kl": 0.0018717050552368164,
+      "learning_rate": 9.93698216681727e-07,
+      "loss": 0.0001,
+      "reward": -0.10725430864840746,
+      "reward_std": 0.786587443202734,
+      "rewards/cosine_scaled_reward": -0.08628973411396146,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 74
+    },
+    {
+      "completion_length": 3010.2708740234375,
+      "epoch": 0.08571428571428572,
+      "grad_norm": 0.17574620246887207,
+      "kl": 0.0008899793028831482,
+      "learning_rate": 9.931634888554935e-07,
+      "loss": 0.0,
+      "reward": 0.17169499211013317,
+      "reward_std": 0.7550366073846817,
+      "rewards/cosine_scaled_reward": 0.06438030861318111,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 75
+    },
+    {
+      "completion_length": 3000.0208587646484,
+      "epoch": 0.08685714285714285,
+      "grad_norm": 0.1713106483221054,
+      "kl": 0.0001423284411430359,
+      "learning_rate": 9.926071618660237e-07,
+      "loss": 0.0,
+      "reward": -0.2900051809847355,
+      "reward_std": 0.5428481921553612,
+      "rewards/cosine_scaled_reward": -0.17083199694752693,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 76
+    },
+    {
+      "completion_length": 3030.854232788086,
+      "epoch": 0.088,
+      "grad_norm": 0.15809102356433868,
+      "kl": 0.00039126724004745483,
+      "learning_rate": 9.9202926282791e-07,
+      "loss": 0.0,
+      "reward": -0.12223898246884346,
+      "reward_std": 0.5839939434081316,
+      "rewards/cosine_scaled_reward": -0.12459814921021461,
+      "rewards/format_reward": 0.39583334513008595,
+      "step": 77
+    },
+    {
+      "completion_length": 3080.5208740234375,
+      "epoch": 0.08914285714285715,
+      "grad_norm": 0.1908382624387741,
+      "kl": 0.00018369778990745544,
+      "learning_rate": 9.91429819907136e-07,
+      "loss": 0.0,
+      "reward": 0.24677438661456108,
+      "reward_std": 0.8652092255651951,
+      "rewards/cosine_scaled_reward": 0.060313327237963676,
+      "rewards/format_reward": 0.45833334140479565,
+      "step": 78
+    },
+    {
+      "completion_length": 2289.7916870117188,
+      "epoch": 0.09028571428571429,
+      "grad_norm": 0.2242053598165512,
+      "kl": 0.000986546277999878,
+      "learning_rate": 9.908088623197048e-07,
+      "loss": 0.0,
+      "reward": 0.043350703082978725,
+      "reward_std": 0.7450110893696547,
+      "rewards/cosine_scaled_reward": -0.10156810469925404,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 79
+    },
+    {
+      "completion_length": 3217.6458740234375,
+      "epoch": 0.09142857142857143,
+      "grad_norm": 0.16875122487545013,
+      "kl": 0.0006685960106551647,
+      "learning_rate": 9.901664203302124e-07,
+      "loss": 0.0,
+      "reward": -0.21108814515173435,
+      "reward_std": 0.6316388584673405,
+      "rewards/cosine_scaled_reward": -0.141633331310004,
+      "rewards/format_reward": 0.2916666716337204,
+      "step": 80
+    },
+    {
+      "completion_length": 3053.0417098999023,
+      "epoch": 0.09257142857142857,
+      "grad_norm": 0.21555641293525696,
+      "kl": 0.001847386360168457,
+      "learning_rate": 9.895025252503755e-07,
+      "loss": 0.0001,
+      "reward": -0.08620672597317025,
+      "reward_std": 0.6757525354623795,
+      "rewards/cosine_scaled_reward": -0.08836011588573456,
+      "rewards/format_reward": 0.37500000186264515,
+      "step": 81
+    },
+    {
+      "completion_length": 2864.375030517578,
+      "epoch": 0.09371428571428571,
+      "grad_norm": 0.18830879032611847,
+      "kl": 0.0011097192764282227,
+      "learning_rate": 9.888172094375033e-07,
+      "loss": 0.0,
+      "reward": 0.05295779928565025,
+      "reward_std": 0.6998694110661745,
+      "rewards/cosine_scaled_reward": -0.003920115530490875,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 82
+    },
+    {
+      "completion_length": 2812.312515258789,
+      "epoch": 0.09485714285714286,
+      "grad_norm": 0.22218850255012512,
+      "kl": 0.0006138831377029419,
+      "learning_rate": 9.881105062929221e-07,
+      "loss": 0.0,
+      "reward": -0.2050288449972868,
+      "reward_std": 0.6761703044176102,
+      "rewards/cosine_scaled_reward": -0.15050649549812078,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 83
+    },
+    {
+      "completion_length": 3117.25,
+      "epoch": 0.096,
+      "grad_norm": 0.16899633407592773,
+      "kl": 0.00044608116149902344,
+      "learning_rate": 9.873824502603459e-07,
+      "loss": 0.0,
+      "reward": 0.03697575815021992,
+      "reward_std": 0.8468833193182945,
+      "rewards/cosine_scaled_reward": -0.044630819000303745,
+      "rewards/format_reward": 0.35416667349636555,
+      "step": 84
+    },
+    {
+      "completion_length": 3147.687530517578,
+      "epoch": 0.09714285714285714,
+      "grad_norm": 0.14954747259616852,
+      "kl": 0.00039239227771759033,
+      "learning_rate": 9.866330768241983e-07,
+      "loss": 0.0,
+      "reward": -0.10085274185985327,
+      "reward_std": 0.7591249234974384,
+      "rewards/cosine_scaled_reward": -0.12476183846592903,
+      "rewards/format_reward": 0.37500000931322575,
+      "step": 85
+    },
+    {
+      "completion_length": 2752.958351135254,
+      "epoch": 0.09828571428571428,
+      "grad_norm": 0.1987195760011673,
+      "kl": 0.0008227825164794922,
+      "learning_rate": 9.85862422507884e-07,
+      "loss": 0.0,
+      "reward": 0.022698190063238144,
+      "reward_std": 0.6212068926542997,
+      "rewards/cosine_scaled_reward": -0.04935761634260416,
+      "rewards/format_reward": 0.4166666679084301,
+      "step": 86
+    },
+    {
+      "completion_length": 2598.500030517578,
+      "epoch": 0.09942857142857142,
+      "grad_norm": 0.196201354265213,
+      "kl": 0.0009197257459163666,
+      "learning_rate": 9.850705248720068e-07,
+      "loss": 0.0,
+      "reward": 0.1782613815739751,
+      "reward_std": 0.8598120957612991,
+      "rewards/cosine_scaled_reward": -0.03767361585050821,
+      "rewards/format_reward": 0.5416666809469461,
+      "step": 87
+    },
+    {
+      "completion_length": 2744.875045776367,
+      "epoch": 0.10057142857142858,
+      "grad_norm": 0.1994200497865677,
+      "kl": 0.0016593635082244873,
+      "learning_rate": 9.8425742251254e-07,
+      "loss": 0.0001,
+      "reward": 0.38837052757298807,
+      "reward_std": 1.1653981991112232,
+      "rewards/cosine_scaled_reward": 0.04098894074559212,
+      "rewards/format_reward": 0.5416666865348816,
+      "step": 88
+    },
+    {
+      "completion_length": 2835.041732788086,
+      "epoch": 0.10171428571428572,
+      "grad_norm": 0.17567898333072662,
+      "kl": 0.0010609626770019531,
+      "learning_rate": 9.83423155058946e-07,
+      "loss": 0.0,
+      "reward": -0.0007169158197939396,
+      "reward_std": 0.7810570430010557,
+      "rewards/cosine_scaled_reward": -0.0879359629470855,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 89
+    },
+    {
+      "completion_length": 2648.2916870117188,
+      "epoch": 0.10285714285714286,
+      "grad_norm": 0.32330945134162903,
+      "kl": 0.0009630322456359863,
+      "learning_rate": 9.825677631722435e-07,
+      "loss": 0.0,
+      "reward": -0.35989359859377146,
+      "reward_std": 0.5252104215323925,
+      "rewards/cosine_scaled_reward": -0.2538211215287447,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 90
+    },
+    {
+      "completion_length": 2993.0208740234375,
+      "epoch": 0.104,
+      "grad_norm": 0.19204914569854736,
+      "kl": 0.0005503743886947632,
+      "learning_rate": 9.816912885430258e-07,
+      "loss": 0.0,
+      "reward": 0.10479497350752354,
+      "reward_std": 0.8132709451019764,
+      "rewards/cosine_scaled_reward": 0.023820115253329277,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 91
+    },
+    {
+      "completion_length": 2592.6875228881836,
+      "epoch": 0.10514285714285715,
+      "grad_norm": 0.22138415277004242,
+      "kl": 0.0012896955013275146,
+      "learning_rate": 9.807937738894303e-07,
+      "loss": 0.0001,
+      "reward": -0.160959305241704,
+      "reward_std": 0.616182116791606,
+      "rewards/cosine_scaled_reward": -0.17551955580711365,
+      "rewards/format_reward": 0.4583333469927311,
+      "step": 92
+    },
+    {
+      "completion_length": 3457.7083435058594,
+      "epoch": 0.10628571428571429,
+      "grad_norm": 0.21576857566833496,
+      "kl": 0.0012897849082946777,
+      "learning_rate": 9.798752629550546e-07,
+      "loss": 0.0001,
+      "reward": -0.5375550724565983,
+      "reward_std": 0.46420392021536827,
+      "rewards/cosine_scaled_reward": -0.24901489913463593,
+      "rewards/format_reward": 0.10416666977107525,
+      "step": 93
+    },
+    {
+      "completion_length": 3101.8750228881836,
+      "epoch": 0.10742857142857143,
+      "grad_norm": 0.18881510198116302,
+      "kl": 0.0011924207210540771,
+      "learning_rate": 9.78935800506826e-07,
+      "loss": 0.0,
+      "reward": -0.3899065591394901,
+      "reward_std": 0.47088516876101494,
+      "rewards/cosine_scaled_reward": -0.1563735492527485,
+      "rewards/format_reward": 0.2083333358168602,
+      "step": 94
+    },
+    {
+      "completion_length": 3354.0833435058594,
+      "epoch": 0.10857142857142857,
+      "grad_norm": 0.1480981707572937,
+      "kl": 0.0006138160824775696,
+      "learning_rate": 9.779754323328192e-07,
+      "loss": 0.0,
+      "reward": -0.16396450996398926,
+      "reward_std": 0.8173685595393181,
+      "rewards/cosine_scaled_reward": -0.13610992138274014,
+      "rewards/format_reward": 0.2916666679084301,
+      "step": 95
+    },
+    {
+      "completion_length": 2596.770896911621,
+      "epoch": 0.10971428571428571,
+      "grad_norm": 0.2216062992811203,
+      "kl": 0.0016668587923049927,
+      "learning_rate": 9.769942052400235e-07,
+      "loss": 0.0001,
+      "reward": -0.06530294334515929,
+      "reward_std": 0.6829965673387051,
+      "rewards/cosine_scaled_reward": -0.08926075106137432,
+      "rewards/format_reward": 0.43750000186264515,
+      "step": 96
+    },
+    {
+      "completion_length": 3140.104217529297,
+      "epoch": 0.11085714285714286,
+      "grad_norm": 0.20360398292541504,
+      "kl": 0.0009580925107002258,
+      "learning_rate": 9.759921670520634e-07,
+      "loss": 0.0,
+      "reward": -0.06862857192754745,
+      "reward_std": 0.7287838291376829,
+      "rewards/cosine_scaled_reward": -0.07030826597474515,
+      "rewards/format_reward": 0.2916666753590107,
+      "step": 97
+    },
+    {
+      "completion_length": 2493.2291870117188,
+      "epoch": 0.112,
+      "grad_norm": 0.2129816859960556,
+      "kl": 0.0004929900169372559,
+      "learning_rate": 9.749693666068663e-07,
+      "loss": 0.0,
+      "reward": -0.014337641187012196,
+      "reward_std": 0.6026619412004948,
+      "rewards/cosine_scaled_reward": -0.1173677199985832,
+      "rewards/format_reward": 0.5416666846722364,
+      "step": 98
+    },
+    {
+      "completion_length": 2910.2708435058594,
+      "epoch": 0.11314285714285714,
+      "grad_norm": 0.2219778299331665,
+      "kl": 0.0009305477142333984,
+      "learning_rate": 9.739258537542835e-07,
+      "loss": 0.0,
+      "reward": -0.1526417959248647,
+      "reward_std": 0.6645006220787764,
+      "rewards/cosine_scaled_reward": -0.11239583557471633,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 99
+    },
+    {
+      "completion_length": 2669.1875228881836,
+      "epoch": 0.11428571428571428,
+      "grad_norm": 0.18922168016433716,
+      "kl": 0.0009844303131103516,
+      "learning_rate": 9.728616793536587e-07,
+      "loss": 0.0,
+      "reward": 0.36958627274725586,
+      "reward_std": 0.8694315142929554,
+      "rewards/cosine_scaled_reward": 0.11309660819824785,
+      "rewards/format_reward": 0.5000000111758709,
+      "step": 100
+    },
+    {
+      "completion_length": 2726.729202270508,
+      "epoch": 0.11542857142857142,
+      "grad_norm": 0.17301318049430847,
+      "kl": 0.0010145902633666992,
+      "learning_rate": 9.717768952713511e-07,
+      "loss": 0.0,
+      "reward": -0.10664987377822399,
+      "reward_std": 0.550342533737421,
+      "rewards/cosine_scaled_reward": -0.07740492327138782,
+      "rewards/format_reward": 0.3958333358168602,
+      "step": 101
+    },
+    {
+      "completion_length": 2206.812530517578,
+      "epoch": 0.11657142857142858,
+      "grad_norm": 0.3242272138595581,
+      "kl": 0.001733243465423584,
+      "learning_rate": 9.706715543782064e-07,
+      "loss": 0.0001,
+      "reward": -0.04840894846711308,
+      "reward_std": 0.5605612080544233,
+      "rewards/cosine_scaled_reward": -0.1783417221158743,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 102
+    },
+    {
+      "completion_length": 2822.145927429199,
+      "epoch": 0.11771428571428572,
+      "grad_norm": 0.20194919407367706,
+      "kl": 0.0008286237716674805,
+      "learning_rate": 9.695457105469804e-07,
+      "loss": 0.0,
+      "reward": 0.10302554164081812,
+      "reward_std": 0.7938342466950417,
+      "rewards/cosine_scaled_reward": -0.058811694383621216,
+      "rewards/format_reward": 0.5000000111758709,
+      "step": 103
+    },
+    {
+      "completion_length": 2794.2291870117188,
+      "epoch": 0.11885714285714286,
+      "grad_norm": 0.21426032483577728,
+      "kl": 0.0017331838607788086,
+      "learning_rate": 9.683994186497132e-07,
+      "loss": 0.0001,
+      "reward": -0.15357419941574335,
+      "reward_std": 0.7048191353678703,
+      "rewards/cosine_scaled_reward": -0.14375380612909794,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 104
+    },
+    {
+      "completion_length": 2595.7500381469727,
+      "epoch": 0.12,
+      "grad_norm": 0.1870584338903427,
+      "kl": 0.0007082223892211914,
+      "learning_rate": 9.672327345550543e-07,
+      "loss": 0.0,
+      "reward": 0.0526086570462212,
+      "reward_std": 1.013668704777956,
+      "rewards/cosine_scaled_reward": -0.10153029090724885,
+      "rewards/format_reward": 0.47916667349636555,
+      "step": 105
+    },
+    {
+      "completion_length": 2181.2291717529297,
+      "epoch": 0.12114285714285715,
+      "grad_norm": 0.193389892578125,
+      "kl": 0.0015556812286376953,
+      "learning_rate": 9.66045715125541e-07,
+      "loss": 0.0001,
+      "reward": 0.20659764064475894,
+      "reward_std": 0.7706484608352184,
+      "rewards/cosine_scaled_reward": 0.0427999310195446,
+      "rewards/format_reward": 0.5625,
+      "step": 106
+    },
+    {
+      "completion_length": 2629.104217529297,
+      "epoch": 0.12228571428571429,
+      "grad_norm": 0.2650865614414215,
+      "kl": 0.001280069351196289,
+      "learning_rate": 9.648384182148252e-07,
+      "loss": 0.0001,
+      "reward": 0.047421048395335674,
+      "reward_std": 0.8096052818000317,
+      "rewards/cosine_scaled_reward": -0.08784432336688042,
+      "rewards/format_reward": 0.5208333469927311,
+      "step": 107
+    },
+    {
+      "completion_length": 2760.3125610351562,
+      "epoch": 0.12342857142857143,
+      "grad_norm": 0.22029048204421997,
+      "kl": 0.0016175508499145508,
+      "learning_rate": 9.636109026648554e-07,
+      "loss": 0.0001,
+      "reward": 0.20211763679981232,
+      "reward_std": 0.9046534672379494,
+      "rewards/cosine_scaled_reward": 0.006914107128977776,
+      "rewards/format_reward": 0.4791666828095913,
+      "step": 108
+    },
+    {
+      "completion_length": 3071.2083587646484,
+      "epoch": 0.12457142857142857,
+      "grad_norm": 0.16153673827648163,
+      "kl": 0.00049591064453125,
+      "learning_rate": 9.623632283030077e-07,
+      "loss": 0.0,
+      "reward": -0.13390080258250237,
+      "reward_std": 0.6916838400065899,
+      "rewards/cosine_scaled_reward": -0.10345052601769567,
+      "rewards/format_reward": 0.3958333358168602,
+      "step": 109
+    },
+    {
+      "completion_length": 2738.8125534057617,
+      "epoch": 0.12571428571428572,
+      "grad_norm": 0.20852850377559662,
+      "kl": 0.0008258223533630371,
+      "learning_rate": 9.610954559391704e-07,
+      "loss": 0.0,
+      "reward": -0.033169424161314964,
+      "reward_std": 0.9115067571401596,
+      "rewards/cosine_scaled_reward": -0.14516241010278463,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 110
+    },
+    {
+      "completion_length": 2979.416702270508,
+      "epoch": 0.12685714285714286,
+      "grad_norm": 0.20923613011837006,
+      "kl": 0.0016465187072753906,
+      "learning_rate": 9.598076473627796e-07,
+      "loss": 0.0001,
+      "reward": -0.1917986012995243,
+      "reward_std": 0.6587283834815025,
+      "rewards/cosine_scaled_reward": -0.1627930011600256,
+      "rewards/format_reward": 0.35416667349636555,
+      "step": 111
+    },
+    {
+      "completion_length": 3034.604202270508,
+      "epoch": 0.128,
+      "grad_norm": 0.1812065690755844,
+      "kl": 0.0010587647557258606,
+      "learning_rate": 9.58499865339809e-07,
+      "loss": 0.0,
+      "reward": 0.042024691589176655,
+      "reward_std": 0.7155229933559895,
+      "rewards/cosine_scaled_reward": -0.007911409251391888,
+      "rewards/format_reward": 0.3958333395421505,
+      "step": 112
+    },
+    {
+      "completion_length": 2602.041702270508,
+      "epoch": 0.12914285714285714,
+      "grad_norm": 0.23084139823913574,
+      "kl": 0.005176067352294922,
+      "learning_rate": 9.571721736097088e-07,
+      "loss": 0.0002,
+      "reward": 0.09520017961040139,
+      "reward_std": 0.7830863744020462,
+      "rewards/cosine_scaled_reward": -0.06248902215156704,
+      "rewards/format_reward": 0.5000000149011612,
+      "step": 113
+    },
+    {
+      "completion_length": 2630.812526702881,
+      "epoch": 0.13028571428571428,
+      "grad_norm": 0.22400474548339844,
+      "kl": 0.002625703811645508,
+      "learning_rate": 9.55824636882301e-07,
+      "loss": 0.0001,
+      "reward": -0.18021708587184548,
+      "reward_std": 0.5372883807867765,
+      "rewards/cosine_scaled_reward": -0.19902520813047886,
+      "rewards/format_reward": 0.5208333432674408,
+      "step": 114
+    },
+    {
+      "completion_length": 2778.3333587646484,
+      "epoch": 0.13142857142857142,
+      "grad_norm": 0.2168537676334381,
+      "kl": 0.0024030208587646484,
+      "learning_rate": 9.54457320834625e-07,
+      "loss": 0.0001,
+      "reward": -0.23857925506308675,
+      "reward_std": 0.4751722030341625,
+      "rewards/cosine_scaled_reward": -0.12388885580003262,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 115
+    },
+    {
+      "completion_length": 3442.6458740234375,
+      "epoch": 0.13257142857142856,
+      "grad_norm": 0.16627182066440582,
+      "kl": 0.0018346309661865234,
+      "learning_rate": 9.530702921077358e-07,
+      "loss": 0.0001,
+      "reward": -0.1470964252948761,
+      "reward_std": 0.609551090747118,
+      "rewards/cosine_scaled_reward": -0.060838012024760246,
+      "rewards/format_reward": 0.2083333395421505,
+      "step": 116
+    },
+    {
+      "completion_length": 3086.3541870117188,
+      "epoch": 0.1337142857142857,
+      "grad_norm": 0.1960468739271164,
+      "kl": 0.0021970272064208984,
+      "learning_rate": 9.516636183034564e-07,
+      "loss": 0.0001,
+      "reward": -0.35040562483482063,
+      "reward_std": 0.566862914711237,
+      "rewards/cosine_scaled_reward": -0.23000845714705065,
+      "rewards/format_reward": 0.3125000037252903,
+      "step": 117
+    },
+    {
+      "completion_length": 2862.125045776367,
+      "epoch": 0.13485714285714287,
+      "grad_norm": 0.19369491934776306,
+      "kl": 0.0014324188232421875,
+      "learning_rate": 9.502373679810839e-07,
+      "loss": 0.0001,
+      "reward": 0.23708336800336838,
+      "reward_std": 0.8551704436540604,
+      "rewards/cosine_scaled_reward": 0.059158104471862316,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 118
+    },
+    {
+      "completion_length": 2329.145881652832,
+      "epoch": 0.136,
+      "grad_norm": 0.2726350724697113,
+      "kl": 0.006384849548339844,
+      "learning_rate": 9.487916106540465e-07,
+      "loss": 0.0003,
+      "reward": 0.06812019762583077,
+      "reward_std": 0.6704130079597235,
+      "rewards/cosine_scaled_reward": -0.1190024558454752,
+      "rewards/format_reward": 0.6250000223517418,
+      "step": 119
+    },
+    {
+      "completion_length": 2332.5208740234375,
+      "epoch": 0.13714285714285715,
+      "grad_norm": 0.2544117867946625,
+      "kl": 0.002226591110229492,
+      "learning_rate": 9.473264167865171e-07,
+      "loss": 0.0001,
+      "reward": 0.13008875958621502,
+      "reward_std": 0.7615821734070778,
+      "rewards/cosine_scaled_reward": -0.029063436202704906,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 120
+    },
+    {
+      "completion_length": 1837.375015258789,
+      "epoch": 0.1382857142857143,
+      "grad_norm": 0.25729382038116455,
+      "kl": 0.007863402366638184,
+      "learning_rate": 9.458418577899774e-07,
+      "loss": 0.0003,
+      "reward": 0.3155789945740253,
+      "reward_std": 0.7411515153944492,
+      "rewards/cosine_scaled_reward": -0.02028810605406761,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 121
+    },
+    {
+      "completion_length": 2958.145866394043,
+      "epoch": 0.13942857142857143,
+      "grad_norm": 0.20917245745658875,
+      "kl": 0.001485586166381836,
+      "learning_rate": 9.443380060197385e-07,
+      "loss": 0.0001,
+      "reward": 0.05837233364582062,
+      "reward_std": 0.964107995852828,
+      "rewards/cosine_scaled_reward": -0.043140748515725136,
+      "rewards/format_reward": 0.37500000558793545,
+      "step": 122
+    },
+    {
+      "completion_length": 2625.7083892822266,
+      "epoch": 0.14057142857142857,
+      "grad_norm": 0.1879587173461914,
+      "kl": 0.0017619132995605469,
+      "learning_rate": 9.428149347714143e-07,
+      "loss": 0.0001,
+      "reward": 0.08997525461018085,
+      "reward_std": 0.7339164912700653,
+      "rewards/cosine_scaled_reward": -0.0870713610202074,
+      "rewards/format_reward": 0.5625000093132257,
+      "step": 123
+    },
+    {
+      "completion_length": 2327.4792404174805,
+      "epoch": 0.1417142857142857,
+      "grad_norm": 0.19856718182563782,
+      "kl": 0.002035379409790039,
+      "learning_rate": 9.412727182773486e-07,
+      "loss": 0.0001,
+      "reward": 0.33785169292241335,
+      "reward_std": 0.8534054830670357,
+      "rewards/cosine_scaled_reward": 0.03106366191059351,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 124
+    },
+    {
+      "completion_length": 2897.2083587646484,
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.1509743332862854,
+      "kl": 0.0013265609741210938,
+      "learning_rate": 9.397114317029974e-07,
+      "loss": 0.0001,
+      "reward": -0.1488229539245367,
+      "reward_std": 0.5740789603441954,
+      "rewards/cosine_scaled_reward": -0.06105211656540632,
+      "rewards/format_reward": 0.27083333395421505,
+      "step": 125
+    },
+    {
+      "completion_length": 2956.0208740234375,
+      "epoch": 0.144,
+      "grad_norm": 0.18172477185726166,
+      "kl": 0.0009171962738037109,
+      "learning_rate": 9.381311511432658e-07,
+      "loss": 0.0,
+      "reward": -0.04263794468715787,
+      "reward_std": 0.8349475301802158,
+      "rewards/cosine_scaled_reward": -0.10641535092145205,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 126
+    },
+    {
+      "completion_length": 2777.229202270508,
+      "epoch": 0.14514285714285713,
+      "grad_norm": 0.2586840093135834,
+      "kl": 0.0015873908996582031,
+      "learning_rate": 9.36531953618799e-07,
+      "loss": 0.0001,
+      "reward": -0.25305760465562344,
+      "reward_std": 0.6229905411601067,
+      "rewards/cosine_scaled_reward": -0.22235668450593948,
+      "rewards/format_reward": 0.41666667722165585,
+      "step": 127
+    },
+    {
+      "completion_length": 2836.7708892822266,
+      "epoch": 0.1462857142857143,
+      "grad_norm": 0.21033713221549988,
+      "kl": 0.0029465854167938232,
+      "learning_rate": 9.34913917072228e-07,
+      "loss": 0.0001,
+      "reward": 0.31079221796244383,
+      "reward_std": 0.9646506570279598,
+      "rewards/cosine_scaled_reward": 0.121150006307289,
+      "rewards/format_reward": 0.43750000186264515,
+      "step": 128
+    },
+    {
+      "completion_length": 3317.375030517578,
+      "epoch": 0.14742857142857144,
+      "grad_norm": 0.19819480180740356,
+      "kl": 0.0024137496948242188,
+      "learning_rate": 9.332771203643714e-07,
+      "loss": 0.0001,
+      "reward": -0.364932868629694,
+      "reward_std": 0.47471798583865166,
+      "rewards/cosine_scaled_reward": -0.18228841945528984,
+      "rewards/format_reward": 0.2083333395421505,
+      "step": 129
+    },
+    {
+      "completion_length": 2750.1041717529297,
+      "epoch": 0.14857142857142858,
+      "grad_norm": 0.19781967997550964,
+      "kl": 0.0014467239379882812,
+      "learning_rate": 9.316216432703916e-07,
+      "loss": 0.0001,
+      "reward": -0.2264688154682517,
+      "reward_std": 0.6045875921845436,
+      "rewards/cosine_scaled_reward": -0.16815861780196428,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 130
+    },
+    {
+      "completion_length": 2835.3333740234375,
+      "epoch": 0.14971428571428572,
+      "grad_norm": 0.23097459971904755,
+      "kl": 0.0031147003173828125,
+      "learning_rate": 9.299475664759068e-07,
+      "loss": 0.0001,
+      "reward": 0.24509642273187637,
+      "reward_std": 0.6866675093770027,
+      "rewards/cosine_scaled_reward": 0.16801240853965282,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 131
+    },
+    {
+      "completion_length": 2526.6041984558105,
+      "epoch": 0.15085714285714286,
+      "grad_norm": 0.19051432609558105,
+      "kl": 0.0015637874603271484,
+      "learning_rate": 9.282549715730579e-07,
+      "loss": 0.0001,
+      "reward": 0.17219684056180995,
+      "reward_std": 0.8952484987676144,
+      "rewards/cosine_scaled_reward": -0.009550546063110232,
+      "rewards/format_reward": 0.4791666679084301,
+      "step": 132
+    },
+    {
+      "completion_length": 3378.1041870117188,
+      "epoch": 0.152,
+      "grad_norm": 0.22686553001403809,
+      "kl": 0.0022031068801879883,
+      "learning_rate": 9.265439410565328e-07,
+      "loss": 0.0001,
+      "reward": -0.4034382812678814,
+      "reward_std": 0.49673712253570557,
+      "rewards/cosine_scaled_reward": -0.20925658009946346,
+      "rewards/format_reward": 0.18750000186264515,
+      "step": 133
+    },
+    {
+      "completion_length": 2480.2083892822266,
+      "epoch": 0.15314285714285714,
+      "grad_norm": 0.2461298406124115,
+      "kl": 0.0034208297729492188,
+      "learning_rate": 9.248145583195447e-07,
+      "loss": 0.0001,
+      "reward": -0.010700544342398643,
+      "reward_std": 0.6441561691462994,
+      "rewards/cosine_scaled_reward": -0.10502434149384499,
+      "rewards/format_reward": 0.5625000055879354,
+      "step": 134
+    },
+    {
+      "completion_length": 1805.4583473205566,
+      "epoch": 0.15428571428571428,
+      "grad_norm": 0.26353919506073,
+      "kl": 0.0027773380279541016,
+      "learning_rate": 9.230669076497687e-07,
+      "loss": 0.0001,
+      "reward": 0.5292202904820442,
+      "reward_std": 0.849658913910389,
+      "rewards/cosine_scaled_reward": 0.17550375685095787,
+      "rewards/format_reward": 0.708333333954215,
+      "step": 135
+    },
+    {
+      "completion_length": 2799.7291870117188,
+      "epoch": 0.15542857142857142,
+      "grad_norm": 0.20877666771411896,
+      "kl": 0.0029191970825195312,
+      "learning_rate": 9.213010742252327e-07,
+      "loss": 0.0001,
+      "reward": 0.07902248203754425,
+      "reward_std": 0.8653755076229572,
+      "rewards/cosine_scaled_reward": 0.008886766619980335,
+      "rewards/format_reward": 0.37500000931322575,
+      "step": 136
+    },
+    {
+      "completion_length": 2720.2083587646484,
+      "epoch": 0.15657142857142858,
+      "grad_norm": 0.1975402683019638,
+      "kl": 0.002421855926513672,
+      "learning_rate": 9.195171441101668e-07,
+      "loss": 0.0001,
+      "reward": -0.21935221180319786,
+      "reward_std": 0.518220279365778,
+      "rewards/cosine_scaled_reward": -0.16422798670828342,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 137
+    },
+    {
+      "completion_length": 2631.229217529297,
+      "epoch": 0.15771428571428572,
+      "grad_norm": 0.22529752552509308,
+      "kl": 0.0021669864654541016,
+      "learning_rate": 9.177152042508077e-07,
+      "loss": 0.0001,
+      "reward": -0.1003083037212491,
+      "reward_std": 0.5785807222127914,
+      "rewards/cosine_scaled_reward": -0.13203439861536026,
+      "rewards/format_reward": 0.5208333376795053,
+      "step": 138
+    },
+    {
+      "completion_length": 3252.3125610351562,
+      "epoch": 0.15885714285714286,
+      "grad_norm": 0.16783365607261658,
+      "kl": 0.0036573410034179688,
+      "learning_rate": 9.158953424711624e-07,
+      "loss": 0.0001,
+      "reward": -0.16235242877155542,
+      "reward_std": 0.6854680478572845,
+      "rewards/cosine_scaled_reward": -0.1317966803908348,
+      "rewards/format_reward": 0.3541666716337204,
+      "step": 139
+    },
+    {
+      "completion_length": 2871.6875610351562,
+      "epoch": 0.16,
+      "grad_norm": 0.23042802512645721,
+      "kl": 0.004063129425048828,
+      "learning_rate": 9.140576474687263e-07,
+      "loss": 0.0002,
+      "reward": -0.0632519107311964,
+      "reward_std": 0.7613547593355179,
+      "rewards/cosine_scaled_reward": -0.076241385191679,
+      "rewards/format_reward": 0.37500000931322575,
+      "step": 140
+    },
+    {
+      "completion_length": 2480.291702270508,
+      "epoch": 0.16114285714285714,
+      "grad_norm": 0.2392415851354599,
+      "kl": 0.003994464874267578,
+      "learning_rate": 9.122022088101613e-07,
+      "loss": 0.0002,
+      "reward": 0.05139289842918515,
+      "reward_std": 1.0145743787288666,
+      "rewards/cosine_scaled_reward": -0.13529899902641773,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 141
+    },
+    {
+      "completion_length": 2873.666717529297,
+      "epoch": 0.16228571428571428,
+      "grad_norm": 0.17095746099948883,
+      "kl": 0.0033861398696899414,
+      "learning_rate": 9.103291169269299e-07,
+      "loss": 0.0001,
+      "reward": 0.11978689953684807,
+      "reward_std": 0.6609016172587872,
+      "rewards/cosine_scaled_reward": -0.06103468872606754,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 142
+    },
+    {
+      "completion_length": 2451.541702270508,
+      "epoch": 0.16342857142857142,
+      "grad_norm": 0.2901478111743927,
+      "kl": 0.0036554336547851562,
+      "learning_rate": 9.084384631108882e-07,
+      "loss": 0.0001,
+      "reward": -0.13543805526569486,
+      "reward_std": 0.5446794554591179,
+      "rewards/cosine_scaled_reward": -0.18383409455418587,
+      "rewards/format_reward": 0.5208333469927311,
+      "step": 143
+    },
+    {
+      "completion_length": 2912.7083740234375,
+      "epoch": 0.16457142857142856,
+      "grad_norm": 0.21996816992759705,
+      "kl": 0.004309415817260742,
+      "learning_rate": 9.065303395098358e-07,
+      "loss": 0.0002,
+      "reward": 0.015518264845013618,
+      "reward_std": 0.9661316499114037,
+      "rewards/cosine_scaled_reward": -0.05092207749839872,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 144
+    },
+    {
+      "completion_length": 2094.041706085205,
+      "epoch": 0.1657142857142857,
+      "grad_norm": 0.29601243138313293,
+      "kl": 0.003927946090698242,
+      "learning_rate": 9.046048391230247e-07,
+      "loss": 0.0002,
+      "reward": 0.2274817731231451,
+      "reward_std": 0.6735235303640366,
+      "rewards/cosine_scaled_reward": 0.019177459180355072,
+      "rewards/format_reward": 0.6250000055879354,
+      "step": 145
+    },
+    {
+      "completion_length": 2216.541717529297,
+      "epoch": 0.16685714285714287,
+      "grad_norm": 0.25898125767707825,
+      "kl": 0.002621889114379883,
+      "learning_rate": 9.026620557966279e-07,
+      "loss": 0.0001,
+      "reward": -0.13925540121272206,
+      "reward_std": 0.5450553633272648,
+      "rewards/cosine_scaled_reward": -0.2378202360123396,
+      "rewards/format_reward": 0.6875000186264515,
+      "step": 146
+    },
+    {
+      "completion_length": 2532.7916870117188,
+      "epoch": 0.168,
+      "grad_norm": 0.41119325160980225,
+      "kl": 0.0048122406005859375,
+      "learning_rate": 9.007020842191634e-07,
+      "loss": 0.0002,
+      "reward": 0.028958545066416264,
+      "reward_std": 0.9969764724373817,
+      "rewards/cosine_scaled_reward": -0.12233338970690966,
+      "rewards/format_reward": 0.4583333507180214,
+      "step": 147
+    },
+    {
+      "completion_length": 1958.9792175292969,
+      "epoch": 0.16914285714285715,
+      "grad_norm": 0.21329711377620697,
+      "kl": 0.0035009384155273438,
+      "learning_rate": 8.987250199168808e-07,
+      "loss": 0.0001,
+      "reward": 0.06930056714918464,
+      "reward_std": 0.7063222527503967,
+      "rewards/cosine_scaled_reward": -0.13013077899813652,
+      "rewards/format_reward": 0.6875000111758709,
+      "step": 148
+    },
+    {
+      "completion_length": 2652.1875610351562,
+      "epoch": 0.1702857142857143,
+      "grad_norm": 0.1904420405626297,
+      "kl": 0.0030150413513183594,
+      "learning_rate": 8.967309592491052e-07,
+      "loss": 0.0001,
+      "reward": 0.0019146008417010307,
+      "reward_std": 0.813957192003727,
+      "rewards/cosine_scaled_reward": -0.10279210843145847,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 149
+    },
+    {
+      "completion_length": 2578.625030517578,
+      "epoch": 0.17142857142857143,
+      "grad_norm": 0.1853615641593933,
+      "kl": 0.005157470703125,
+      "learning_rate": 8.9471999940354e-07,
+      "loss": 0.0002,
+      "reward": 0.13735826686024666,
+      "reward_std": 0.8738452345132828,
+      "rewards/cosine_scaled_reward": -0.027063111774623394,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 150
+    },
+    {
+      "completion_length": 2566.541732788086,
+      "epoch": 0.17257142857142857,
+      "grad_norm": 0.23767390847206116,
+      "kl": 0.0045833587646484375,
+      "learning_rate": 8.926922383915315e-07,
+      "loss": 0.0002,
+      "reward": 0.3640142543008551,
+      "reward_std": 0.9924535490572453,
+      "rewards/cosine_scaled_reward": 0.056208414025604725,
+      "rewards/format_reward": 0.6041666809469461,
+      "step": 151
+    },
+    {
+      "completion_length": 2766.5208702087402,
+      "epoch": 0.1737142857142857,
+      "grad_norm": 0.24616549909114838,
+      "kl": 0.0033342838287353516,
+      "learning_rate": 8.906477750432903e-07,
+      "loss": 0.0001,
+      "reward": -0.30499533005058765,
+      "reward_std": 0.5299477484077215,
+      "rewards/cosine_scaled_reward": -0.2434069886803627,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 152
+    },
+    {
+      "completion_length": 2895.229202270508,
+      "epoch": 0.17485714285714285,
+      "grad_norm": 0.2707502543926239,
+      "kl": 0.007465362548828125,
+      "learning_rate": 8.88586709003076e-07,
+      "loss": 0.0003,
+      "reward": -0.28449683357030153,
+      "reward_std": 0.5924058370292187,
+      "rewards/cosine_scaled_reward": -0.19268111791461706,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 153
+    },
+    {
+      "completion_length": 3004.7709045410156,
+      "epoch": 0.176,
+      "grad_norm": 0.1523827612400055,
+      "kl": 0.00289154052734375,
+      "learning_rate": 8.865091407243394e-07,
+      "loss": 0.0001,
+      "reward": 0.22956039011478424,
+      "reward_std": 0.9176076352596283,
+      "rewards/cosine_scaled_reward": 0.006945975736016408,
+      "rewards/format_reward": 0.479166679084301,
+      "step": 154
+    },
+    {
+      "completion_length": 2524.5000762939453,
+      "epoch": 0.17714285714285713,
+      "grad_norm": 0.24581067264080048,
+      "kl": 0.004633903503417969,
+      "learning_rate": 8.844151714648274e-07,
+      "loss": 0.0002,
+      "reward": 0.3532795161008835,
+      "reward_std": 0.9012942314147949,
+      "rewards/cosine_scaled_reward": 0.0729524465277791,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 155
+    },
+    {
+      "completion_length": 2804.3958587646484,
+      "epoch": 0.1782857142857143,
+      "grad_norm": 0.17449386417865753,
+      "kl": 0.0034241676330566406,
+      "learning_rate": 8.823049032816478e-07,
+      "loss": 0.0001,
+      "reward": 0.0325262644328177,
+      "reward_std": 0.7823463976383209,
+      "rewards/cosine_scaled_reward": -0.03533525764942169,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 156
+    },
+    {
+      "completion_length": 2587.6458587646484,
+      "epoch": 0.17942857142857144,
+      "grad_norm": 0.25060343742370605,
+      "kl": 0.004924774169921875,
+      "learning_rate": 8.801784390262943e-07,
+      "loss": 0.0002,
+      "reward": -0.024557745084166527,
+      "reward_std": 0.6769015416502953,
+      "rewards/cosine_scaled_reward": -0.1357441581785679,
+      "rewards/format_reward": 0.5208333488553762,
+      "step": 157
+    },
+    {
+      "completion_length": 2703.291732788086,
+      "epoch": 0.18057142857142858,
+      "grad_norm": 0.22450025379657745,
+      "kl": 0.0042743682861328125,
+      "learning_rate": 8.780358823396352e-07,
+      "loss": 0.0002,
+      "reward": 0.35095504857599735,
+      "reward_std": 0.8320975676178932,
+      "rewards/cosine_scaled_reward": 0.09731243550777435,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 158
+    },
+    {
+      "completion_length": 2376.3750610351562,
+      "epoch": 0.18171428571428572,
+      "grad_norm": 0.21200759708881378,
+      "kl": 0.0042476654052734375,
+      "learning_rate": 8.758773376468604e-07,
+      "loss": 0.0002,
+      "reward": -0.21574918151600286,
+      "reward_std": 0.5446000955998898,
+      "rewards/cosine_scaled_reward": -0.2437344677746296,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 159
+    },
+    {
+      "completion_length": 2424.9583892822266,
+      "epoch": 0.18285714285714286,
+      "grad_norm": 0.2798510491847992,
+      "kl": 0.0068912506103515625,
+      "learning_rate": 8.737029101523929e-07,
+      "loss": 0.0003,
+      "reward": 0.04046674119308591,
+      "reward_std": 0.7019340619444847,
+      "rewards/cosine_scaled_reward": -0.06891408376395702,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 160
+    },
+    {
+      "completion_length": 2377.0625610351562,
+      "epoch": 0.184,
+      "grad_norm": 0.23946921527385712,
+      "kl": 0.004667758941650391,
+      "learning_rate": 8.715127058347614e-07,
+      "loss": 0.0002,
+      "reward": 0.18844679649919271,
+      "reward_std": 0.7960058376193047,
+      "rewards/cosine_scaled_reward": 0.006343178451061249,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 161
+    },
+    {
+      "completion_length": 2592.708366394043,
+      "epoch": 0.18514285714285714,
+      "grad_norm": 0.27762702107429504,
+      "kl": 0.0064907073974609375,
+      "learning_rate": 8.693068314414344e-07,
+      "loss": 0.0003,
+      "reward": -0.033426298294216394,
+      "reward_std": 0.6928768754005432,
+      "rewards/cosine_scaled_reward": -0.1188662868225947,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 162
+    },
+    {
+      "completion_length": 2438.3541717529297,
+      "epoch": 0.18628571428571428,
+      "grad_norm": 0.28610387444496155,
+      "kl": 0.005778312683105469,
+      "learning_rate": 8.670853944836176e-07,
+      "loss": 0.0002,
+      "reward": 0.2196616232395172,
+      "reward_std": 0.7138870656490326,
+      "rewards/cosine_scaled_reward": 0.04680558480322361,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 163
+    },
+    {
+      "completion_length": 1862.2292022705078,
+      "epoch": 0.18742857142857142,
+      "grad_norm": 0.22276760637760162,
+      "kl": 0.00450897216796875,
+      "learning_rate": 8.648485032310144e-07,
+      "loss": 0.0002,
+      "reward": 0.2688090084120631,
+      "reward_std": 0.662966214120388,
+      "rewards/cosine_scaled_reward": 0.05209773499518633,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 164
+    },
+    {
+      "completion_length": 2642.979202270508,
+      "epoch": 0.18857142857142858,
+      "grad_norm": 0.2338155210018158,
+      "kl": 0.006450653076171875,
+      "learning_rate": 8.625962667065487e-07,
+      "loss": 0.0003,
+      "reward": -0.11441950500011444,
+      "reward_std": 0.6914314143359661,
+      "rewards/cosine_scaled_reward": -0.15340343955904245,
+      "rewards/format_reward": 0.45833334140479565,
+      "step": 165
+    },
+    {
+      "completion_length": 2228.5208892822266,
+      "epoch": 0.18971428571428572,
+      "grad_norm": 0.19290319085121155,
+      "kl": 0.0036525726318359375,
+      "learning_rate": 8.603287946810513e-07,
+      "loss": 0.0001,
+      "reward": 0.039148006588220596,
+      "reward_std": 0.5984033793210983,
+      "rewards/cosine_scaled_reward": -0.10414127632975578,
+      "rewards/format_reward": 0.6250000093132257,
+      "step": 166
+    },
+    {
+      "completion_length": 2005.8125610351562,
+      "epoch": 0.19085714285714286,
+      "grad_norm": 0.19015978276729584,
+      "kl": 0.0037364959716796875,
+      "learning_rate": 8.580461976679099e-07,
+      "loss": 0.0001,
+      "reward": 0.278128509176895,
+      "reward_std": 0.851395096629858,
+      "rewards/cosine_scaled_reward": -0.09441124647855759,
+      "rewards/format_reward": 0.854166679084301,
+      "step": 167
+    },
+    {
+      "completion_length": 2624.979217529297,
+      "epoch": 0.192,
+      "grad_norm": 0.18921926617622375,
+      "kl": 0.004267692565917969,
+      "learning_rate": 8.557485869176825e-07,
+      "loss": 0.0002,
+      "reward": 0.18450849317014217,
+      "reward_std": 0.9270573705434799,
+      "rewards/cosine_scaled_reward": -0.05149332107976079,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 168
+    },
+    {
+      "completion_length": 1482.1042022705078,
+      "epoch": 0.19314285714285714,
+      "grad_norm": 0.23450377583503723,
+      "kl": 0.004532814025878906,
+      "learning_rate": 8.534360744126753e-07,
+      "loss": 0.0002,
+      "reward": 0.7876127276103944,
+      "reward_std": 0.8578404039144516,
+      "rewards/cosine_scaled_reward": 0.2795695327222347,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 169
+    },
+    {
+      "completion_length": 2186.9167251586914,
+      "epoch": 0.19428571428571428,
+      "grad_norm": 0.30296990275382996,
+      "kl": 0.0044994354248046875,
+      "learning_rate": 8.511087728614862e-07,
+      "loss": 0.0002,
+      "reward": 0.056005215272307396,
+      "reward_std": 0.6679836474359035,
+      "rewards/cosine_scaled_reward": -0.021252445876598358,
+      "rewards/format_reward": 0.5625000018626451,
+      "step": 170
+    },
+    {
+      "completion_length": 2248.312530517578,
+      "epoch": 0.19542857142857142,
+      "grad_norm": 0.20839498937129974,
+      "kl": 0.0036230087280273438,
+      "learning_rate": 8.487667956935087e-07,
+      "loss": 0.0001,
+      "reward": 0.16125285997986794,
+      "reward_std": 0.7742661274969578,
+      "rewards/cosine_scaled_reward": -0.013224839232861996,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 171
+    },
+    {
+      "completion_length": 2516.833351135254,
+      "epoch": 0.19657142857142856,
+      "grad_norm": 0.21489036083221436,
+      "kl": 0.007071495056152344,
+      "learning_rate": 8.464102570534061e-07,
+      "loss": 0.0003,
+      "reward": 0.3356585130095482,
+      "reward_std": 0.7922957856208086,
+      "rewards/cosine_scaled_reward": 0.13084825314581394,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 172
+    },
+    {
+      "completion_length": 1844.708381652832,
+      "epoch": 0.1977142857142857,
+      "grad_norm": 0.23371165990829468,
+      "kl": 0.00569915771484375,
+      "learning_rate": 8.440392717955475e-07,
+      "loss": 0.0002,
+      "reward": 0.0007524143438786268,
+      "reward_std": 0.7251704446971416,
+      "rewards/cosine_scaled_reward": -0.1511296879616566,
+      "rewards/format_reward": 0.6458333395421505,
+      "step": 173
+    },
+    {
+      "completion_length": 1865.3958740234375,
+      "epoch": 0.19885714285714284,
+      "grad_norm": 0.24605944752693176,
+      "kl": 0.0073699951171875,
+      "learning_rate": 8.416539554784089e-07,
+      "loss": 0.0003,
+      "reward": 0.40005480125546455,
+      "reward_std": 0.8749304339289665,
+      "rewards/cosine_scaled_reward": 0.05308325891382992,
+      "rewards/format_reward": 0.7916666753590107,
+      "step": 174
+    },
+    {
+      "completion_length": 2525.270896911621,
+      "epoch": 0.2,
+      "grad_norm": 0.20796766877174377,
+      "kl": 0.005358695983886719,
+      "learning_rate": 8.392544243589427e-07,
+      "loss": 0.0002,
+      "reward": 0.15992992464452982,
+      "reward_std": 0.6595781818032265,
+      "rewards/cosine_scaled_reward": 0.004710111767053604,
+      "rewards/format_reward": 0.5208333376795053,
+      "step": 175
+    },
+    {
+      "completion_length": 2078.895896911621,
+      "epoch": 0.20114285714285715,
+      "grad_norm": 0.28462517261505127,
+      "kl": 0.005040168762207031,
+      "learning_rate": 8.368407953869103e-07,
+      "loss": 0.0002,
+      "reward": 0.25200897455215454,
+      "reward_std": 0.9856929145753384,
+      "rewards/cosine_scaled_reward": -0.05302844103425741,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 176
+    },
+    {
+      "completion_length": 2306.2083740234375,
+      "epoch": 0.2022857142857143,
+      "grad_norm": 0.24023154377937317,
+      "kl": 0.005183219909667969,
+      "learning_rate": 8.344131861991828e-07,
+      "loss": 0.0002,
+      "reward": 0.28276925068348646,
+      "reward_std": 0.6319316327571869,
+      "rewards/cosine_scaled_reward": 0.06355313770473003,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 177
+    },
+    {
+      "completion_length": 2121.791717529297,
+      "epoch": 0.20342857142857143,
+      "grad_norm": 0.2661464214324951,
+      "kl": 0.008258819580078125,
+      "learning_rate": 8.319717151140072e-07,
+      "loss": 0.0003,
+      "reward": 0.013979046139866114,
+      "reward_std": 0.7351427339017391,
+      "rewards/cosine_scaled_reward": -0.14257426233962178,
+      "rewards/format_reward": 0.6250000055879354,
+      "step": 178
+    },
+    {
+      "completion_length": 2433.708351135254,
+      "epoch": 0.20457142857142857,
+      "grad_norm": 0.20986828207969666,
+      "kl": 0.004935264587402344,
+      "learning_rate": 8.295165011252396e-07,
+      "loss": 0.0002,
+      "reward": -0.25348505191504955,
+      "reward_std": 0.48908819630742073,
+      "rewards/cosine_scaled_reward": -0.21158038638532162,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 179
+    },
+    {
+      "completion_length": 1877.1458892822266,
+      "epoch": 0.2057142857142857,
+      "grad_norm": 0.3029525876045227,
+      "kl": 0.007785797119140625,
+      "learning_rate": 8.270476638965461e-07,
+      "loss": 0.0003,
+      "reward": 0.3592034715693444,
+      "reward_std": 1.002089962363243,
+      "rewards/cosine_scaled_reward": 0.08408734039403498,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 180
+    },
+    {
+      "completion_length": 2741.729248046875,
+      "epoch": 0.20685714285714285,
+      "grad_norm": 0.2602676451206207,
+      "kl": 0.00711822509765625,
+      "learning_rate": 8.245653237555705e-07,
+      "loss": 0.0003,
+      "reward": -0.08426681905984879,
+      "reward_std": 0.5513291470706463,
+      "rewards/cosine_scaled_reward": -0.07528127636760473,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 181
+    },
+    {
+      "completion_length": 1988.2292175292969,
+      "epoch": 0.208,
+      "grad_norm": 0.17226605117321014,
+      "kl": 0.0025339126586914062,
+      "learning_rate": 8.220696016880687e-07,
+      "loss": 0.0001,
+      "reward": 0.23337539401836693,
+      "reward_std": 0.824427492916584,
+      "rewards/cosine_scaled_reward": -0.0713941128924489,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 182
+    },
+    {
+      "completion_length": 1507.3958892822266,
+      "epoch": 0.20914285714285713,
+      "grad_norm": 0.2777194082736969,
+      "kl": 0.006999969482421875,
+      "learning_rate": 8.195606193320136e-07,
+      "loss": 0.0003,
+      "reward": 0.5963715696707368,
+      "reward_std": 0.8888205997645855,
+      "rewards/cosine_scaled_reward": 0.07703271105128806,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 183
+    },
+    {
+      "completion_length": 2085.979202270508,
+      "epoch": 0.2102857142857143,
+      "grad_norm": 0.2736140489578247,
+      "kl": 0.0066013336181640625,
+      "learning_rate": 8.170384989716657e-07,
+      "loss": 0.0003,
+      "reward": -0.11966793239116669,
+      "reward_std": 0.4983285814523697,
+      "rewards/cosine_scaled_reward": -0.20256702601909637,
+      "rewards/format_reward": 0.6458333395421505,
+      "step": 184
+    },
+    {
+      "completion_length": 2048.5208854675293,
+      "epoch": 0.21142857142857144,
+      "grad_norm": 0.24372698366641998,
+      "kl": 0.0058803558349609375,
+      "learning_rate": 8.145033635316128e-07,
+      "loss": 0.0002,
+      "reward": 0.06693027447909117,
+      "reward_std": 0.6862058416008949,
+      "rewards/cosine_scaled_reward": -0.10436064226087183,
+      "rewards/format_reward": 0.6250000093132257,
+      "step": 185
+    },
+    {
+      "completion_length": 2159.791702270508,
+      "epoch": 0.21257142857142858,
+      "grad_norm": 0.20706257224082947,
+      "kl": 0.006072998046875,
+      "learning_rate": 8.119553365707802e-07,
+      "loss": 0.0002,
+      "reward": 0.0768028711900115,
+      "reward_std": 0.5925656575709581,
+      "rewards/cosine_scaled_reward": -0.05905535398051143,
+      "rewards/format_reward": 0.5625,
+      "step": 186
+    },
+    {
+      "completion_length": 1804.8542022705078,
+      "epoch": 0.21371428571428572,
+      "grad_norm": 0.2685582637786865,
+      "kl": 0.0061969757080078125,
+      "learning_rate": 8.093945422764069e-07,
+      "loss": 0.0002,
+      "reward": 0.12330089835450053,
+      "reward_std": 0.606924245133996,
+      "rewards/cosine_scaled_reward": -0.09539542999118567,
+      "rewards/format_reward": 0.7500000111758709,
+      "step": 187
+    },
+    {
+      "completion_length": 2442.166702270508,
+      "epoch": 0.21485714285714286,
+      "grad_norm": 0.2364640235900879,
+      "kl": 0.0075626373291015625,
+      "learning_rate": 8.068211054579943e-07,
+      "loss": 0.0003,
+      "reward": -0.09115996491163969,
+      "reward_std": 0.6131051359698176,
+      "rewards/cosine_scaled_reward": -0.1099636135622859,
+      "rewards/format_reward": 0.4791666753590107,
+      "step": 188
+    },
+    {
+      "completion_length": 1640.333366394043,
+      "epoch": 0.216,
+      "grad_norm": 0.6100494861602783,
+      "kl": 0.006476402282714844,
+      "learning_rate": 8.04235151541222e-07,
+      "loss": 0.0003,
+      "reward": 0.12896017776802182,
+      "reward_std": 0.6158286519348621,
+      "rewards/cosine_scaled_reward": -0.10562538355588913,
+      "rewards/format_reward": 0.7291666828095913,
+      "step": 189
+    },
+    {
+      "completion_length": 1453.6042022705078,
+      "epoch": 0.21714285714285714,
+      "grad_norm": 0.2732960283756256,
+      "kl": 0.006473541259765625,
+      "learning_rate": 8.01636806561836e-07,
+      "loss": 0.0003,
+      "reward": 0.2690247750142589,
+      "reward_std": 0.7748389020562172,
+      "rewards/cosine_scaled_reward": -0.04900714522227645,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 190
+    },
+    {
+      "completion_length": 1478.0000305175781,
+      "epoch": 0.21828571428571428,
+      "grad_norm": 0.25885748863220215,
+      "kl": 0.0072765350341796875,
+      "learning_rate": 7.990261971595048e-07,
+      "loss": 0.0003,
+      "reward": 0.5228147888556123,
+      "reward_std": 0.9636592417955399,
+      "rewards/cosine_scaled_reward": 0.07218710612505674,
+      "rewards/format_reward": 0.812500013038516,
+      "step": 191
+    },
+    {
+      "completion_length": 1954.8542251586914,
+      "epoch": 0.21942857142857142,
+      "grad_norm": 0.22047950327396393,
+      "kl": 0.0056095123291015625,
+      "learning_rate": 7.964034505716476e-07,
+      "loss": 0.0002,
+      "reward": 0.0916488622315228,
+      "reward_std": 0.666417833417654,
+      "rewards/cosine_scaled_reward": -0.13260145671665668,
+      "rewards/format_reward": 0.7083333488553762,
+      "step": 192
+    },
+    {
+      "completion_length": 2805.520866394043,
+      "epoch": 0.22057142857142858,
+      "grad_norm": 0.22538112103939056,
+      "kl": 0.0061187744140625,
+      "learning_rate": 7.93768694627233e-07,
+      "loss": 0.0002,
+      "reward": -0.33769641164690256,
+      "reward_std": 0.507177896797657,
+      "rewards/cosine_scaled_reward": -0.24244121788069606,
+      "rewards/format_reward": 0.39583333767950535,
+      "step": 193
+    },
+    {
+      "completion_length": 2417.354202270508,
+      "epoch": 0.22171428571428572,
+      "grad_norm": 0.2549934983253479,
+      "kl": 0.007659912109375,
+      "learning_rate": 7.911220577405484e-07,
+      "loss": 0.0003,
+      "reward": 0.4917243723757565,
+      "reward_std": 1.1182497814297676,
+      "rewards/cosine_scaled_reward": 0.08968745917081833,
+      "rewards/format_reward": 0.645833345130086,
+      "step": 194
+    },
+    {
+      "completion_length": 1545.1667175292969,
+      "epoch": 0.22285714285714286,
+      "grad_norm": 0.2932279706001282,
+      "kl": 0.0074100494384765625,
+      "learning_rate": 7.884636689049422e-07,
+      "loss": 0.0003,
+      "reward": 0.40460192365571856,
+      "reward_std": 0.8929797559976578,
+      "rewards/cosine_scaled_reward": -0.016500022262334824,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 195
+    },
+    {
+      "completion_length": 2750.791717529297,
+      "epoch": 0.224,
+      "grad_norm": 0.2122720181941986,
+      "kl": 0.007846832275390625,
+      "learning_rate": 7.857936576865356e-07,
+      "loss": 0.0003,
+      "reward": -0.0444787316955626,
+      "reward_std": 0.6230567768216133,
+      "rewards/cosine_scaled_reward": -0.09410630911588669,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 196
+    },
+    {
+      "completion_length": 1153.0000381469727,
+      "epoch": 0.22514285714285714,
+      "grad_norm": 0.30193769931793213,
+      "kl": 0.00811767578125,
+      "learning_rate": 7.831121542179086e-07,
+      "loss": 0.0003,
+      "reward": 0.5208917018026114,
+      "reward_std": 0.8946940749883652,
+      "rewards/cosine_scaled_reward": 0.04951087199151516,
+      "rewards/format_reward": 0.8958333395421505,
+      "step": 197
+    },
+    {
+      "completion_length": 1570.083396911621,
+      "epoch": 0.22628571428571428,
+      "grad_norm": 0.2520270347595215,
+      "kl": 0.008256912231445312,
+      "learning_rate": 7.804192891917571e-07,
+      "loss": 0.0003,
+      "reward": 0.43910311779472977,
+      "reward_std": 0.9849786385893822,
+      "rewards/cosine_scaled_reward": -0.005096456501632929,
+      "rewards/format_reward": 0.8125000149011612,
+      "step": 198
+    },
+    {
+      "completion_length": 1278.4375305175781,
+      "epoch": 0.22742857142857142,
+      "grad_norm": 0.265682190656662,
+      "kl": 0.00727081298828125,
+      "learning_rate": 7.777151938545235e-07,
+      "loss": 0.0003,
+      "reward": 0.16469779529143125,
+      "reward_std": 0.5715500190854073,
+      "rewards/cosine_scaled_reward": -0.1407376565039158,
+      "rewards/format_reward": 0.9375,
+      "step": 199
+    },
+    {
+      "completion_length": 1395.8541946411133,
+      "epoch": 0.22857142857142856,
+      "grad_norm": 0.24266065657138824,
+      "kl": 0.0067348480224609375,
+      "learning_rate": 7.75e-07,
+      "loss": 0.0003,
+      "reward": 0.3167417113436386,
+      "reward_std": 0.7663153670728207,
+      "rewards/cosine_scaled_reward": -0.044573438179213554,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 200
+    },
+    {
+      "completion_length": 1824.4375457763672,
+      "epoch": 0.2297142857142857,
+      "grad_norm": 0.23948884010314941,
+      "kl": 0.00595855712890625,
+      "learning_rate": 7.72273839962904e-07,
+      "loss": 0.0002,
+      "reward": 0.7574443845078349,
+      "reward_std": 0.8649395480751991,
+      "rewards/cosine_scaled_reward": 0.28773305006325245,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 201
+    },
+    {
+      "completion_length": 1388.5833740234375,
+      "epoch": 0.23085714285714284,
+      "grad_norm": 0.2889217734336853,
+      "kl": 0.008331298828125,
+      "learning_rate": 7.695368466124296e-07,
+      "loss": 0.0003,
+      "reward": 0.46887177898315713,
+      "reward_std": 0.6449649855494499,
+      "rewards/cosine_scaled_reward": 0.19245607405900955,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 202
+    },
+    {
+      "completion_length": 1545.083396911621,
+      "epoch": 0.232,
+      "grad_norm": 0.24110884964466095,
+      "kl": 0.0080413818359375,
+      "learning_rate": 7.667891533457718e-07,
+      "loss": 0.0003,
+      "reward": 0.37680933251976967,
+      "reward_std": 0.8439403660595417,
+      "rewards/cosine_scaled_reward": -0.034176651388406754,
+      "rewards/format_reward": 0.8750000149011612,
+      "step": 203
+    },
+    {
+      "completion_length": 1558.5833740234375,
+      "epoch": 0.23314285714285715,
+      "grad_norm": 0.3269757628440857,
+      "kl": 0.0079345703125,
+      "learning_rate": 7.640308940816239e-07,
+      "loss": 0.0003,
+      "reward": 0.4802742376923561,
+      "reward_std": 0.7629036977887154,
+      "rewards/cosine_scaled_reward": 0.10827891901135445,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 204
+    },
+    {
+      "completion_length": 1542.5000305175781,
+      "epoch": 0.2342857142857143,
+      "grad_norm": 0.2582697570323944,
+      "kl": 0.0057811737060546875,
+      "learning_rate": 7.612622032536507e-07,
+      "loss": 0.0002,
+      "reward": 0.7918389849364758,
+      "reward_std": 0.8735720105469227,
+      "rewards/cosine_scaled_reward": 0.29279233887791634,
+      "rewards/format_reward": 0.8750000074505806,
+      "step": 205
+    },
+    {
+      "completion_length": 1852.4167098999023,
+      "epoch": 0.23542857142857143,
+      "grad_norm": 0.30814215540885925,
+      "kl": 0.007049560546875,
+      "learning_rate": 7.584832158039378e-07,
+      "loss": 0.0003,
+      "reward": -0.12371287122368813,
+      "reward_std": 0.5305570438504219,
+      "rewards/cosine_scaled_reward": -0.24367139115929604,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 206
+    },
+    {
+      "completion_length": 1706.8125610351562,
+      "epoch": 0.23657142857142857,
+      "grad_norm": 0.2674766182899475,
+      "kl": 0.009944915771484375,
+      "learning_rate": 7.556940671764124e-07,
+      "loss": 0.0004,
+      "reward": 0.018216492608189583,
+      "reward_std": 0.8229854069650173,
+      "rewards/cosine_scaled_reward": -0.2149236612021923,
+      "rewards/format_reward": 0.7500000074505806,
+      "step": 207
+    },
+    {
+      "completion_length": 1185.0833587646484,
+      "epoch": 0.2377142857142857,
+      "grad_norm": 0.24083174765110016,
+      "kl": 0.0078887939453125,
+      "learning_rate": 7.528948933102438e-07,
+      "loss": 0.0003,
+      "reward": 0.2574625021661632,
+      "reward_std": 0.5847832001745701,
+      "rewards/cosine_scaled_reward": -0.02713889814913273,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 208
+    },
+    {
+      "completion_length": 1211.3333587646484,
+      "epoch": 0.23885714285714285,
+      "grad_norm": 0.28845900297164917,
+      "kl": 0.0077152252197265625,
+      "learning_rate": 7.500858306332172e-07,
+      "loss": 0.0003,
+      "reward": 0.5239271614700556,
+      "reward_std": 0.8414975665509701,
+      "rewards/cosine_scaled_reward": 0.07757960073649883,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 209
+    },
+    {
+      "completion_length": 1571.8333892822266,
+      "epoch": 0.24,
+      "grad_norm": 0.26938140392303467,
+      "kl": 0.006866455078125,
+      "learning_rate": 7.472670160550848e-07,
+      "loss": 0.0003,
+      "reward": 0.39476348645985126,
+      "reward_std": 0.8261179774999619,
+      "rewards/cosine_scaled_reward": -0.00616603484377265,
+      "rewards/format_reward": 0.8541666753590107,
+      "step": 210
+    },
+    {
+      "completion_length": 1672.083366394043,
+      "epoch": 0.24114285714285713,
+      "grad_norm": 0.358892560005188,
+      "kl": 0.0106353759765625,
+      "learning_rate": 7.444385869608921e-07,
+      "loss": 0.0004,
+      "reward": 0.3219002881087363,
+      "reward_std": 0.750870831310749,
+      "rewards/cosine_scaled_reward": 0.013120350427925587,
+      "rewards/format_reward": 0.7916666679084301,
+      "step": 211
+    },
+    {
+      "completion_length": 1112.6250534057617,
+      "epoch": 0.2422857142857143,
+      "grad_norm": 0.27129727602005005,
+      "kl": 0.008617401123046875,
+      "learning_rate": 7.416006812042827e-07,
+      "loss": 0.0003,
+      "reward": 0.5328430655645207,
+      "reward_std": 0.671292532235384,
+      "rewards/cosine_scaled_reward": 0.12215181812644005,
+      "rewards/format_reward": 0.9375,
+      "step": 212
+    },
+    {
+      "completion_length": 1226.1666870117188,
+      "epoch": 0.24342857142857144,
+      "grad_norm": 0.36483535170555115,
+      "kl": 0.01009368896484375,
+      "learning_rate": 7.387534371007797e-07,
+      "loss": 0.0004,
+      "reward": 0.47296964284032583,
+      "reward_std": 0.7435989566147327,
+      "rewards/cosine_scaled_reward": 0.06554012396372855,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 213
+    },
+    {
+      "completion_length": 1548.1875228881836,
+      "epoch": 0.24457142857142858,
+      "grad_norm": 0.3107360303401947,
+      "kl": 0.008228302001953125,
+      "learning_rate": 7.358969934210438e-07,
+      "loss": 0.0003,
+      "reward": 0.5165067554917186,
+      "reward_std": 0.7588581070303917,
+      "rewards/cosine_scaled_reward": 0.10956975258886814,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 214
+    },
+    {
+      "completion_length": 1461.6875305175781,
+      "epoch": 0.24571428571428572,
+      "grad_norm": 0.22486445307731628,
+      "kl": 0.0059413909912109375,
+      "learning_rate": 7.330314893841101e-07,
+      "loss": 0.0002,
+      "reward": 0.07686804980039597,
+      "reward_std": 0.5298155099153519,
+      "rewards/cosine_scaled_reward": -0.14177834056317806,
+      "rewards/format_reward": 0.8333333358168602,
+      "step": 215
+    },
+    {
+      "completion_length": 1340.5208702087402,
+      "epoch": 0.24685714285714286,
+      "grad_norm": 0.2673039436340332,
+      "kl": 0.008434295654296875,
+      "learning_rate": 7.301570646506027e-07,
+      "loss": 0.0003,
+      "reward": 0.38506741262972355,
+      "reward_std": 0.6937872804701328,
+      "rewards/cosine_scaled_reward": 0.08268431574106216,
+      "rewards/format_reward": 0.8125000074505806,
+      "step": 216
+    },
+    {
+      "completion_length": 1469.6250534057617,
+      "epoch": 0.248,
+      "grad_norm": 0.3538609445095062,
+      "kl": 0.008731842041015625,
+      "learning_rate": 7.27273859315928e-07,
+      "loss": 0.0003,
+      "reward": 0.6034841773507651,
+      "reward_std": 1.0110028125345707,
+      "rewards/cosine_scaled_reward": 0.10976234765257686,
+      "rewards/format_reward": 0.8125000037252903,
+      "step": 217
+    },
+    {
+      "completion_length": 1511.458381652832,
+      "epoch": 0.24914285714285714,
+      "grad_norm": 0.2224741131067276,
+      "kl": 0.007978439331054688,
+      "learning_rate": 7.243820139034464e-07,
+      "loss": 0.0003,
+      "reward": 0.21123092295601964,
+      "reward_std": 0.7925957031548023,
+      "rewards/cosine_scaled_reward": -0.1498157843016088,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 218
+    },
+    {
+      "completion_length": 1340.4791870117188,
+      "epoch": 0.2502857142857143,
+      "grad_norm": 0.24136604368686676,
+      "kl": 0.009267807006835938,
+      "learning_rate": 7.214816693576234e-07,
+      "loss": 0.0004,
+      "reward": 0.3911650243680924,
+      "reward_std": 0.6955340765416622,
+      "rewards/cosine_scaled_reward": 0.048877415247261524,
+      "rewards/format_reward": 0.875,
+      "step": 219
+    },
+    {
+      "completion_length": 1524.354175567627,
+      "epoch": 0.25142857142857145,
+      "grad_norm": 0.2906893491744995,
+      "kl": 0.011249542236328125,
+      "learning_rate": 7.185729670371604e-07,
+      "loss": 0.0004,
+      "reward": -0.04361694771796465,
+      "reward_std": 0.5702285468578339,
+      "rewards/cosine_scaled_reward": -0.2380666360259056,
+      "rewards/format_reward": 0.8125000074505806,
+      "step": 220
+    },
+    {
+      "completion_length": 1496.2916870117188,
+      "epoch": 0.25257142857142856,
+      "grad_norm": 0.24429333209991455,
+      "kl": 0.0070476531982421875,
+      "learning_rate": 7.156560487081051e-07,
+      "loss": 0.0003,
+      "reward": 0.4626695259066764,
+      "reward_std": 0.7840724922716618,
+      "rewards/cosine_scaled_reward": 0.06611506012268364,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 221
+    },
+    {
+      "completion_length": 1576.9167022705078,
+      "epoch": 0.2537142857142857,
+      "grad_norm": 0.26980236172676086,
+      "kl": 0.009477615356445312,
+      "learning_rate": 7.127310565369415e-07,
+      "loss": 0.0004,
+      "reward": 0.2927159178070724,
+      "reward_std": 0.6861963458359241,
+      "rewards/cosine_scaled_reward": -0.02613269304856658,
+      "rewards/format_reward": 0.7916666679084301,
+      "step": 222
+    },
+    {
+      "completion_length": 1766.4583587646484,
+      "epoch": 0.25485714285714284,
+      "grad_norm": 0.3431569039821625,
+      "kl": 0.009063720703125,
+      "learning_rate": 7.097981330836616e-07,
+      "loss": 0.0004,
+      "reward": 0.24513752292841673,
+      "reward_std": 0.6672438345849514,
+      "rewards/cosine_scaled_reward": 0.0011468753218650818,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 223
+    },
+    {
+      "completion_length": 1734.6458587646484,
+      "epoch": 0.256,
+      "grad_norm": 0.2113533616065979,
+      "kl": 0.008029937744140625,
+      "learning_rate": 7.068574212948169e-07,
+      "loss": 0.0003,
+      "reward": 0.6102093638037331,
+      "reward_std": 1.0698180794715881,
+      "rewards/cosine_scaled_reward": 0.07039764476940036,
+      "rewards/format_reward": 0.8541666753590107,
+      "step": 224
+    },
+    {
+      "completion_length": 1987.958396911621,
+      "epoch": 0.2571428571428571,
+      "grad_norm": 0.3268744945526123,
+      "kl": 0.012989044189453125,
+      "learning_rate": 7.039090644965509e-07,
+      "loss": 0.0005,
+      "reward": 0.24530102079734206,
+      "reward_std": 0.9006591737270355,
+      "rewards/cosine_scaled_reward": -0.0955775510519743,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 225
+    },
+    {
+      "completion_length": 1425.6041946411133,
+      "epoch": 0.2582857142857143,
+      "grad_norm": 0.24840526282787323,
+      "kl": 0.00763702392578125,
+      "learning_rate": 7.009532063876148e-07,
+      "loss": 0.0003,
+      "reward": 0.6913954578340054,
+      "reward_std": 0.8804083652794361,
+      "rewards/cosine_scaled_reward": 0.15895199915394187,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 226
+    },
+    {
+      "completion_length": 1124.0416946411133,
+      "epoch": 0.25942857142857145,
+      "grad_norm": 0.31861069798469543,
+      "kl": 0.011066436767578125,
+      "learning_rate": 6.979899910323624e-07,
+      "loss": 0.0004,
+      "reward": 0.3930067252367735,
+      "reward_std": 0.8200523294508457,
+      "rewards/cosine_scaled_reward": -0.05645215045660734,
+      "rewards/format_reward": 0.9583333358168602,
+      "step": 227
+    },
+    {
+      "completion_length": 1212.2291984558105,
+      "epoch": 0.26057142857142856,
+      "grad_norm": 0.27715247869491577,
+      "kl": 0.008419036865234375,
+      "learning_rate": 6.950195628537299e-07,
+      "loss": 0.0003,
+      "reward": 0.5142936524935067,
+      "reward_std": 0.8248747400939465,
+      "rewards/cosine_scaled_reward": 0.05838426947593689,
+      "rewards/format_reward": 0.895833333954215,
+      "step": 228
+    },
+    {
+      "completion_length": 1521.9167175292969,
+      "epoch": 0.26171428571428573,
+      "grad_norm": 0.3403978645801544,
+      "kl": 0.012042999267578125,
+      "learning_rate": 6.920420666261961e-07,
+      "loss": 0.0005,
+      "reward": 0.19939115084707737,
+      "reward_std": 0.5617837458848953,
+      "rewards/cosine_scaled_reward": -0.031215182272717357,
+      "rewards/format_reward": 0.8125000149011612,
+      "step": 229
+    },
+    {
+      "completion_length": 1612.5000305175781,
+      "epoch": 0.26285714285714284,
+      "grad_norm": 0.27715978026390076,
+      "kl": 0.010951995849609375,
+      "learning_rate": 6.890576474687263e-07,
+      "loss": 0.0004,
+      "reward": 0.025840092916041613,
+      "reward_std": 0.5985405147075653,
+      "rewards/cosine_scaled_reward": -0.1870412821881473,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 230
+    },
+    {
+      "completion_length": 1249.7500305175781,
+      "epoch": 0.264,
+      "grad_norm": 0.2477809637784958,
+      "kl": 0.00991058349609375,
+      "learning_rate": 6.860664508377001e-07,
+      "loss": 0.0004,
+      "reward": 0.45136169949546456,
+      "reward_std": 0.7211090363562107,
+      "rewards/cosine_scaled_reward": 0.07621757127344608,
+      "rewards/format_reward": 0.8541666697710752,
+      "step": 231
+    },
+    {
+      "completion_length": 1592.9791946411133,
+      "epoch": 0.2651428571428571,
+      "grad_norm": 0.30748802423477173,
+      "kl": 0.012454986572265625,
+      "learning_rate": 6.83068622519821e-07,
+      "loss": 0.0005,
+      "reward": 0.012111502306652255,
+      "reward_std": 0.6053478866815567,
+      "rewards/cosine_scaled_reward": -0.21021990105509758,
+      "rewards/format_reward": 0.8125000018626451,
+      "step": 232
+    },
+    {
+      "completion_length": 1202.4375495910645,
+      "epoch": 0.2662857142857143,
+      "grad_norm": 0.2600504755973816,
+      "kl": 0.011119842529296875,
+      "learning_rate": 6.800643086250121e-07,
+      "loss": 0.0004,
+      "reward": 0.3125277090584859,
+      "reward_std": 0.7523941993713379,
+      "rewards/cosine_scaled_reward": -0.07805835455656052,
+      "rewards/format_reward": 0.9375000149011612,
+      "step": 233
+    },
+    {
+      "completion_length": 1664.25004196167,
+      "epoch": 0.2674285714285714,
+      "grad_norm": 0.28175073862075806,
+      "kl": 0.013034820556640625,
+      "learning_rate": 6.770536555792944e-07,
+      "loss": 0.0005,
+      "reward": 0.2759629947831854,
+      "reward_std": 0.7744503617286682,
+      "rewards/cosine_scaled_reward": -0.00288372952491045,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 234
+    },
+    {
+      "completion_length": 1031.729206085205,
+      "epoch": 0.26857142857142857,
+      "grad_norm": 0.41683852672576904,
+      "kl": 0.009319305419921875,
+      "learning_rate": 6.740368101176495e-07,
+      "loss": 0.0004,
+      "reward": 0.536057305522263,
+      "reward_std": 0.8349857330322266,
+      "rewards/cosine_scaled_reward": 0.1025073304772377,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 235
+    },
+    {
+      "completion_length": 1922.7708587646484,
+      "epoch": 0.26971428571428574,
+      "grad_norm": 0.2624496519565582,
+      "kl": 0.010082244873046875,
+      "learning_rate": 6.710139192768694e-07,
+      "loss": 0.0004,
+      "reward": 0.5450696041807532,
+      "reward_std": 1.0304477401077747,
+      "rewards/cosine_scaled_reward": 0.09541857382282615,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 236
+    },
+    {
+      "completion_length": 1460.6250305175781,
+      "epoch": 0.27085714285714285,
+      "grad_norm": 0.2655409872531891,
+      "kl": 0.0093231201171875,
+      "learning_rate": 6.679851303883891e-07,
+      "loss": 0.0004,
+      "reward": 0.4284868792165071,
+      "reward_std": 0.6683508493006229,
+      "rewards/cosine_scaled_reward": 0.04793188441544771,
+      "rewards/format_reward": 0.875,
+      "step": 237
+    },
+    {
+      "completion_length": 1020.8125305175781,
+      "epoch": 0.272,
+      "grad_norm": 0.27945178747177124,
+      "kl": 0.010372161865234375,
+      "learning_rate": 6.649505910711058e-07,
+      "loss": 0.0004,
+      "reward": 0.6645625443197787,
+      "reward_std": 0.9196898862719536,
+      "rewards/cosine_scaled_reward": 0.1358748753555119,
+      "rewards/format_reward": 0.9166666679084301,
+      "step": 238
+    },
+    {
+      "completion_length": 1431.6041870117188,
+      "epoch": 0.27314285714285713,
+      "grad_norm": 0.25624558329582214,
+      "kl": 0.00914764404296875,
+      "learning_rate": 6.619104492241847e-07,
+      "loss": 0.0004,
+      "reward": 0.584113098680973,
+      "reward_std": 0.6673776432871819,
+      "rewards/cosine_scaled_reward": 0.31309779919683933,
+      "rewards/format_reward": 0.7708333395421505,
+      "step": 239
+    },
+    {
+      "completion_length": 1502.979232788086,
+      "epoch": 0.2742857142857143,
+      "grad_norm": 0.35446247458457947,
+      "kl": 0.02126312255859375,
+      "learning_rate": 6.588648530198504e-07,
+      "loss": 0.0009,
+      "reward": 0.07019667001441121,
+      "reward_std": 0.6685795933008194,
+      "rewards/cosine_scaled_reward": -0.22959061339497566,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 240
+    },
+    {
+      "completion_length": 1754.9375534057617,
+      "epoch": 0.2754285714285714,
+      "grad_norm": 0.29134202003479004,
+      "kl": 0.0203399658203125,
+      "learning_rate": 6.558139508961654e-07,
+      "loss": 0.0008,
+      "reward": -0.14658209728077054,
+      "reward_std": 0.5397600717842579,
+      "rewards/cosine_scaled_reward": -0.26472287997603416,
+      "rewards/format_reward": 0.7500000093132257,
+      "step": 241
+    },
+    {
+      "completion_length": 1179.6041984558105,
+      "epoch": 0.2765714285714286,
+      "grad_norm": 0.3573894500732422,
+      "kl": 0.019369125366210938,
+      "learning_rate": 6.527578915497951e-07,
+      "loss": 0.0008,
+      "reward": 0.17393979895859957,
+      "reward_std": 0.6096976324915886,
+      "rewards/cosine_scaled_reward": -0.09758513886481524,
+      "rewards/format_reward": 0.8958333395421505,
+      "step": 242
+    },
+    {
+      "completion_length": 1776.333351135254,
+      "epoch": 0.2777142857142857,
+      "grad_norm": 0.26269933581352234,
+      "kl": 0.016147613525390625,
+      "learning_rate": 6.496968239287603e-07,
+      "loss": 0.0006,
+      "reward": 0.18704321165569127,
+      "reward_std": 0.727390356361866,
+      "rewards/cosine_scaled_reward": -0.07570383511483669,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 243
+    },
+    {
+      "completion_length": 1609.1041870117188,
+      "epoch": 0.27885714285714286,
+      "grad_norm": 0.42587101459503174,
+      "kl": 0.016376495361328125,
+      "learning_rate": 6.466308972251785e-07,
+      "loss": 0.0007,
+      "reward": 0.5847336421720684,
+      "reward_std": 0.9021250456571579,
+      "rewards/cosine_scaled_reward": 0.07377137243747711,
+      "rewards/format_reward": 0.8750000149011612,
+      "step": 244
+    },
+    {
+      "completion_length": 2179.520896911621,
+      "epoch": 0.28,
+      "grad_norm": 0.5839371085166931,
+      "kl": 0.019596099853515625,
+      "learning_rate": 6.435602608679916e-07,
+      "loss": 0.0008,
+      "reward": 0.45881569012999535,
+      "reward_std": 1.0688990727066994,
+      "rewards/cosine_scaled_reward": 0.05305776512250304,
+      "rewards/format_reward": 0.6666666828095913,
+      "step": 245
+    },
+    {
+      "completion_length": 1406.6667175292969,
+      "epoch": 0.28114285714285714,
+      "grad_norm": 0.28245845437049866,
+      "kl": 0.012783050537109375,
+      "learning_rate": 6.404850645156841e-07,
+      "loss": 0.0005,
+      "reward": 0.353425451554358,
+      "reward_std": 0.7201703079044819,
+      "rewards/cosine_scaled_reward": -0.04003934998763725,
+      "rewards/format_reward": 0.8958333395421505,
+      "step": 246
+    },
+    {
+      "completion_length": 2307.7500610351562,
+      "epoch": 0.2822857142857143,
+      "grad_norm": 0.48197513818740845,
+      "kl": 0.0279693603515625,
+      "learning_rate": 6.374054580489873e-07,
+      "loss": 0.0011,
+      "reward": -0.030684850877150893,
+      "reward_std": 0.7912298962473869,
+      "rewards/cosine_scaled_reward": -0.17165578715503216,
+      "rewards/format_reward": 0.5833333488553762,
+      "step": 247
+    },
+    {
+      "completion_length": 1537.020866394043,
+      "epoch": 0.2834285714285714,
+      "grad_norm": 0.5387913584709167,
+      "kl": 0.020456314086914062,
+      "learning_rate": 6.343215915635761e-07,
+      "loss": 0.0008,
+      "reward": 0.5385480709373951,
+      "reward_std": 0.7778996899724007,
+      "rewards/cosine_scaled_reward": 0.18887367472052574,
+      "rewards/format_reward": 0.7916666679084301,
+      "step": 248
+    },
+    {
+      "completion_length": 1561.6042098999023,
+      "epoch": 0.2845714285714286,
+      "grad_norm": 0.3947852551937103,
+      "kl": 0.021512985229492188,
+      "learning_rate": 6.31233615362752e-07,
+      "loss": 0.0009,
+      "reward": 0.4948454611003399,
+      "reward_std": 0.8307998143136501,
+      "rewards/cosine_scaled_reward": 0.12714037066325545,
+      "rewards/format_reward": 0.7291666734963655,
+      "step": 249
+    },
+    {
+      "completion_length": 1415.1875457763672,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 0.48303961753845215,
+      "kl": 0.01863861083984375,
+      "learning_rate": 6.281416799501187e-07,
+      "loss": 0.0007,
+      "reward": 0.28884256578749046,
+      "reward_std": 0.7405095249414444,
+      "rewards/cosine_scaled_reward": -0.05281085259048268,
+      "rewards/format_reward": 0.8750000074505806,
+      "step": 250
+    },
+    {
+      "completion_length": 1362.2292022705078,
+      "epoch": 0.28685714285714287,
+      "grad_norm": 0.4910474717617035,
+      "kl": 0.02812957763671875,
+      "learning_rate": 6.25045936022246e-07,
+      "loss": 0.0011,
+      "reward": 0.24397201603278518,
+      "reward_std": 0.8808831572532654,
+      "rewards/cosine_scaled_reward": -0.09501386666670442,
+      "rewards/format_reward": 0.7916666734963655,
+      "step": 251
+    },
+    {
+      "completion_length": 1879.4791946411133,
+      "epoch": 0.288,
+      "grad_norm": 0.5786982774734497,
+      "kl": 0.03542327880859375,
+      "learning_rate": 6.219465344613258e-07,
+      "loss": 0.0014,
+      "reward": 0.003026331774890423,
+      "reward_std": 0.5837202109396458,
+      "rewards/cosine_scaled_reward": -0.1362874787300825,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 252
+    },
+    {
+      "completion_length": 1624.1250381469727,
+      "epoch": 0.28914285714285715,
+      "grad_norm": 0.6477235555648804,
+      "kl": 0.037837982177734375,
+      "learning_rate": 6.188436263278172e-07,
+      "loss": 0.0015,
+      "reward": 0.24255571886897087,
+      "reward_std": 0.933814812451601,
+      "rewards/cosine_scaled_reward": -0.09086994710378349,
+      "rewards/format_reward": 0.7291666697710752,
+      "step": 253
+    },
+    {
+      "completion_length": 1799.5209007263184,
+      "epoch": 0.29028571428571426,
+      "grad_norm": 0.4789735972881317,
+      "kl": 0.0286407470703125,
+      "learning_rate": 6.157373628530852e-07,
+      "loss": 0.0011,
+      "reward": 0.2414424503222108,
+      "reward_std": 0.7898707538843155,
+      "rewards/cosine_scaled_reward": -0.08134954981505871,
+      "rewards/format_reward": 0.7916666865348816,
+      "step": 254
+    },
+    {
+      "completion_length": 2204.27091217041,
+      "epoch": 0.2914285714285714,
+      "grad_norm": 0.45766016840934753,
+      "kl": 0.03482818603515625,
+      "learning_rate": 6.126278954320294e-07,
+      "loss": 0.0014,
+      "reward": 0.07502584741450846,
+      "reward_std": 0.9448065534234047,
+      "rewards/cosine_scaled_reward": -0.15562699240399525,
+      "rewards/format_reward": 0.6666666753590107,
+      "step": 255
+    },
+    {
+      "completion_length": 1663.729232788086,
+      "epoch": 0.2925714285714286,
+      "grad_norm": 0.5182092785835266,
+      "kl": 0.0309906005859375,
+      "learning_rate": 6.095153756157051e-07,
+      "loss": 0.0012,
+      "reward": 0.2998895291239023,
+      "reward_std": 0.7498802877962589,
+      "rewards/cosine_scaled_reward": -0.001917465589940548,
+      "rewards/format_reward": 0.7708333395421505,
+      "step": 256
+    },
+    {
+      "completion_length": 2637.0625762939453,
+      "epoch": 0.2937142857142857,
+      "grad_norm": 0.5543068647384644,
+      "kl": 0.0585174560546875,
+      "learning_rate": 6.06399955103937e-07,
+      "loss": 0.0023,
+      "reward": 0.48659578152000904,
+      "reward_std": 1.1908883340656757,
+      "rewards/cosine_scaled_reward": 0.1260754211107269,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 257
+    },
+    {
+      "completion_length": 2023.2708892822266,
+      "epoch": 0.2948571428571429,
+      "grad_norm": 0.5329146385192871,
+      "kl": 0.03546142578125,
+      "learning_rate": 6.032817857379256e-07,
+      "loss": 0.0014,
+      "reward": 0.459288542624563,
+      "reward_std": 0.8846051767468452,
+      "rewards/cosine_scaled_reward": 0.051098582334816456,
+      "rewards/format_reward": 0.7708333544433117,
+      "step": 258
+    },
+    {
+      "completion_length": 1453.2708625793457,
+      "epoch": 0.296,
+      "grad_norm": 0.43237462639808655,
+      "kl": 0.02968597412109375,
+      "learning_rate": 6.001610194928464e-07,
+      "loss": 0.0012,
+      "reward": 0.3637466989457607,
+      "reward_std": 0.8164427168667316,
+      "rewards/cosine_scaled_reward": 0.06898940447717905,
+      "rewards/format_reward": 0.7708333414047956,
+      "step": 259
+    },
+    {
+      "completion_length": 944.4791851043701,
+      "epoch": 0.29714285714285715,
+      "grad_norm": 0.6447161436080933,
+      "kl": 0.018672943115234375,
+      "learning_rate": 5.97037808470444e-07,
+      "loss": 0.0007,
+      "reward": 0.5366823731455952,
+      "reward_std": 0.7269707396626472,
+      "rewards/cosine_scaled_reward": 0.11251152493059635,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 260
+    },
+    {
+      "completion_length": 1973.9583740234375,
+      "epoch": 0.29828571428571427,
+      "grad_norm": 0.37675243616104126,
+      "kl": 0.03509521484375,
+      "learning_rate": 5.939123048916173e-07,
+      "loss": 0.0014,
+      "reward": 0.13344457978382707,
+      "reward_std": 0.8264882601797581,
+      "rewards/cosine_scaled_reward": -0.09806014783680439,
+      "rewards/format_reward": 0.6666666734963655,
+      "step": 261
+    },
+    {
+      "completion_length": 1516.6875228881836,
+      "epoch": 0.29942857142857143,
+      "grad_norm": 0.5359493494033813,
+      "kl": 0.03139495849609375,
+      "learning_rate": 5.907846610890011e-07,
+      "loss": 0.0013,
+      "reward": -0.0372560010291636,
+      "reward_std": 0.6091209948062897,
+      "rewards/cosine_scaled_reward": -0.212333626113832,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 262
+    },
+    {
+      "completion_length": 1346.3125228881836,
+      "epoch": 0.30057142857142854,
+      "grad_norm": 0.3749285638332367,
+      "kl": 0.0174713134765625,
+      "learning_rate": 5.87655029499542e-07,
+      "loss": 0.0007,
+      "reward": 0.17701542098075151,
+      "reward_std": 0.7833218686282635,
+      "rewards/cosine_scaled_reward": -0.15012040082365274,
+      "rewards/format_reward": 0.8541666772216558,
+      "step": 263
+    },
+    {
+      "completion_length": 1433.2708740234375,
+      "epoch": 0.3017142857142857,
+      "grad_norm": 0.4387812614440918,
+      "kl": 0.03086090087890625,
+      "learning_rate": 5.845235626570683e-07,
+      "loss": 0.0012,
+      "reward": 0.2682348359376192,
+      "reward_std": 0.736188217997551,
+      "rewards/cosine_scaled_reward": -0.10726138763129711,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 264
+    },
+    {
+      "completion_length": 1286.4583587646484,
+      "epoch": 0.3028571428571429,
+      "grad_norm": 0.5124613642692566,
+      "kl": 0.029205322265625,
+      "learning_rate": 5.813904131848564e-07,
+      "loss": 0.0012,
+      "reward": 0.47514245874481276,
+      "reward_std": 0.7719651460647583,
+      "rewards/cosine_scaled_reward": 0.03089581150561571,
+      "rewards/format_reward": 0.8958333507180214,
+      "step": 265
+    },
+    {
+      "completion_length": 1764.3958892822266,
+      "epoch": 0.304,
+      "grad_norm": 0.5820686221122742,
+      "kl": 0.04436492919921875,
+      "learning_rate": 5.78255733788191e-07,
+      "loss": 0.0018,
+      "reward": 0.060105842188932,
+      "reward_std": 0.6623933054506779,
+      "rewards/cosine_scaled_reward": -0.15320268645882607,
+      "rewards/format_reward": 0.7500000074505806,
+      "step": 266
+    },
+    {
+      "completion_length": 1691.2917022705078,
+      "epoch": 0.30514285714285716,
+      "grad_norm": 0.3804147243499756,
+      "kl": 0.05069732666015625,
+      "learning_rate": 5.751196772469237e-07,
+      "loss": 0.002,
+      "reward": 0.010469182627275586,
+      "reward_std": 0.6885622590780258,
+      "rewards/cosine_scaled_reward": -0.203377990052104,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 267
+    },
+    {
+      "completion_length": 1172.3125305175781,
+      "epoch": 0.3062857142857143,
+      "grad_norm": 0.5410240888595581,
+      "kl": 0.036266326904296875,
+      "learning_rate": 5.71982396408026e-07,
+      "loss": 0.0015,
+      "reward": 0.10452710837125778,
+      "reward_std": 0.667732447385788,
+      "rewards/cosine_scaled_reward": -0.19316846132278442,
+      "rewards/format_reward": 0.895833333954215,
+      "step": 268
+    },
+    {
+      "completion_length": 1373.541706085205,
+      "epoch": 0.30742857142857144,
+      "grad_norm": 0.5952326655387878,
+      "kl": 0.033191680908203125,
+      "learning_rate": 5.688440441781398e-07,
+      "loss": 0.0013,
+      "reward": 0.33500073617324233,
+      "reward_std": 0.7064723633229733,
+      "rewards/cosine_scaled_reward": -0.04047585092484951,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 269
+    },
+    {
+      "completion_length": 1461.0000228881836,
+      "epoch": 0.30857142857142855,
+      "grad_norm": 0.3846686780452728,
+      "kl": 0.047119140625,
+      "learning_rate": 5.657047735161255e-07,
+      "loss": 0.0019,
+      "reward": 0.48025781381875277,
+      "reward_std": 0.8026834316551685,
+      "rewards/cosine_scaled_reward": 0.06730105105089024,
+      "rewards/format_reward": 0.8750000055879354,
+      "step": 270
+    },
+    {
+      "completion_length": 1104.7500190734863,
+      "epoch": 0.3097142857142857,
+      "grad_norm": 0.3378467559814453,
+      "kl": 0.02167510986328125,
+      "learning_rate": 5.625647374256061e-07,
+      "loss": 0.0009,
+      "reward": 0.512184641789645,
+      "reward_std": 0.6890225373208523,
+      "rewards/cosine_scaled_reward": 0.11897583678364754,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 271
+    },
+    {
+      "completion_length": 1595.5625610351562,
+      "epoch": 0.31085714285714283,
+      "grad_norm": 0.9565722346305847,
+      "kl": 0.0341033935546875,
+      "learning_rate": 5.594240889475106e-07,
+      "loss": 0.0014,
+      "reward": 0.2559305219911039,
+      "reward_std": 0.7282034941017628,
+      "rewards/cosine_scaled_reward": -0.0902782422490418,
+      "rewards/format_reward": 0.8750000055879354,
+      "step": 272
+    },
+    {
+      "completion_length": 1564.2500534057617,
+      "epoch": 0.312,
+      "grad_norm": 0.6573230624198914,
+      "kl": 0.06351470947265625,
+      "learning_rate": 5.562829811526154e-07,
+      "loss": 0.0025,
+      "reward": 0.33316121553070843,
+      "reward_std": 0.7983997203409672,
+      "rewards/cosine_scaled_reward": 0.00879891961812973,
+      "rewards/format_reward": 0.7916666753590107,
+      "step": 273
+    },
+    {
+      "completion_length": 908.3750190734863,
+      "epoch": 0.31314285714285717,
+      "grad_norm": 0.3975994288921356,
+      "kl": 0.020572662353515625,
+      "learning_rate": 5.531415671340826e-07,
+      "loss": 0.0008,
+      "reward": 0.490811045630835,
+      "reward_std": 0.7964842580258846,
+      "rewards/cosine_scaled_reward": 0.027651330456137657,
+      "rewards/format_reward": 0.9791666716337204,
+      "step": 274
+    },
+    {
+      "completion_length": 1273.416706085205,
+      "epoch": 0.3142857142857143,
+      "grad_norm": 0.5143330693244934,
+      "kl": 0.0457763671875,
+      "learning_rate": 5.5e-07,
+      "loss": 0.0018,
+      "reward": 0.5196955967694521,
+      "reward_std": 0.8949318751692772,
+      "rewards/cosine_scaled_reward": 0.0634711142629385,
+      "rewards/format_reward": 0.9166666865348816,
+      "step": 275
+    },
+    {
+      "completion_length": 1216.2917175292969,
+      "epoch": 0.31542857142857145,
+      "grad_norm": 0.5758770704269409,
+      "kl": 0.04427337646484375,
+      "learning_rate": 5.468584328659172e-07,
+      "loss": 0.0018,
+      "reward": 0.45866466453298926,
+      "reward_std": 0.7997590340673923,
+      "rewards/cosine_scaled_reward": 0.004157306393608451,
+      "rewards/format_reward": 0.9166666716337204,
+      "step": 276
+    },
+    {
+      "completion_length": 1405.8542022705078,
+      "epoch": 0.31657142857142856,
+      "grad_norm": 0.7709969878196716,
+      "kl": 0.07879257202148438,
+      "learning_rate": 5.437170188473847e-07,
+      "loss": 0.0032,
+      "reward": 0.3928624112159014,
+      "reward_std": 0.8682146407663822,
+      "rewards/cosine_scaled_reward": -0.0433911276049912,
+      "rewards/format_reward": 0.8958333358168602,
+      "step": 277
+    },
+    {
+      "completion_length": 1251.5834045410156,
+      "epoch": 0.3177142857142857,
+      "grad_norm": 0.4815487265586853,
+      "kl": 0.028911590576171875,
+      "learning_rate": 5.405759110524894e-07,
+      "loss": 0.0012,
+      "reward": 0.602864139713347,
+      "reward_std": 0.75666194409132,
+      "rewards/cosine_scaled_reward": 0.1303448430262506,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 278
+    },
+    {
+      "completion_length": 1584.2292098999023,
+      "epoch": 0.31885714285714284,
+      "grad_norm": 0.676331102848053,
+      "kl": 0.04915618896484375,
+      "learning_rate": 5.37435262574394e-07,
+      "loss": 0.002,
+      "reward": 0.1920575883705169,
+      "reward_std": 0.5915602222084999,
+      "rewards/cosine_scaled_reward": -0.08263814821839333,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 279
+    },
+    {
+      "completion_length": 1887.104232788086,
+      "epoch": 0.32,
+      "grad_norm": 0.5340350270271301,
+      "kl": 0.09076309204101562,
+      "learning_rate": 5.342952264838747e-07,
+      "loss": 0.0036,
+      "reward": 0.554422979708761,
+      "reward_std": 1.0056494362652302,
+      "rewards/cosine_scaled_reward": 0.14068329893052578,
+      "rewards/format_reward": 0.7291666753590107,
+      "step": 280
+    },
+    {
+      "completion_length": 2119.9583892822266,
+      "epoch": 0.3211428571428571,
+      "grad_norm": 0.901451587677002,
+      "kl": 0.10712432861328125,
+      "learning_rate": 5.311559558218603e-07,
+      "loss": 0.0043,
+      "reward": -0.04842189947521547,
+      "reward_std": 0.735725924372673,
+      "rewards/cosine_scaled_reward": -0.18730824999511242,
+      "rewards/format_reward": 0.6041666772216558,
+      "step": 281
+    },
+    {
+      "completion_length": 1546.7708625793457,
+      "epoch": 0.3222857142857143,
+      "grad_norm": 0.5774481892585754,
+      "kl": 0.08298110961914062,
+      "learning_rate": 5.28017603591974e-07,
+      "loss": 0.0033,
+      "reward": 0.4459398053586483,
+      "reward_std": 0.8270345889031887,
+      "rewards/cosine_scaled_reward": 0.038561356253921986,
+      "rewards/format_reward": 0.875,
+      "step": 282
+    },
+    {
+      "completion_length": 1432.2292175292969,
+      "epoch": 0.32342857142857145,
+      "grad_norm": 0.6603187322616577,
+      "kl": 0.06803131103515625,
+      "learning_rate": 5.248803227530763e-07,
+      "loss": 0.0027,
+      "reward": 0.3603776376694441,
+      "reward_std": 0.8584562204778194,
+      "rewards/cosine_scaled_reward": -0.058407315984368324,
+      "rewards/format_reward": 0.8541666828095913,
+      "step": 283
+    },
+    {
+      "completion_length": 1322.5625228881836,
+      "epoch": 0.32457142857142857,
+      "grad_norm": 0.5359531044960022,
+      "kl": 0.06329727172851562,
+      "learning_rate": 5.21744266211809e-07,
+      "loss": 0.0025,
+      "reward": 0.09239194821566343,
+      "reward_std": 0.6924001406878233,
+      "rewards/cosine_scaled_reward": -0.19309467636048794,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 284
+    },
+    {
+      "completion_length": 829.2917022705078,
+      "epoch": 0.32571428571428573,
+      "grad_norm": 0.4555800259113312,
+      "kl": 0.037982940673828125,
+      "learning_rate": 5.186095868151436e-07,
+      "loss": 0.0015,
+      "reward": 0.4122108933515847,
+      "reward_std": 0.7844538278877735,
+      "rewards/cosine_scaled_reward": 0.023661921732127666,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 285
+    },
+    {
+      "completion_length": 1359.5417098999023,
+      "epoch": 0.32685714285714285,
+      "grad_norm": 0.5877644419670105,
+      "kl": 0.05910491943359375,
+      "learning_rate": 5.154764373429315e-07,
+      "loss": 0.0024,
+      "reward": 0.2864837823435664,
+      "reward_std": 0.8134515210986137,
+      "rewards/cosine_scaled_reward": -0.10266671486897394,
+      "rewards/format_reward": 0.8750000223517418,
+      "step": 286
+    },
+    {
+      "completion_length": 1420.1250343322754,
+      "epoch": 0.328,
+      "grad_norm": 1.0718910694122314,
+      "kl": 0.14565277099609375,
+      "learning_rate": 5.123449705004581e-07,
+      "loss": 0.0058,
+      "reward": 0.23910232353955507,
+      "reward_std": 0.6846515089273453,
+      "rewards/cosine_scaled_reward": -0.03220707958098501,
+      "rewards/format_reward": 0.812500013038516,
+      "step": 287
+    },
+    {
+      "completion_length": 1311.1666793823242,
+      "epoch": 0.3291428571428571,
+      "grad_norm": 0.7865566611289978,
+      "kl": 0.07602310180664062,
+      "learning_rate": 5.09215338910999e-07,
+      "loss": 0.003,
+      "reward": 0.4210634557530284,
+      "reward_std": 0.8144961148500443,
+      "rewards/cosine_scaled_reward": -0.004126264713704586,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 288
+    },
+    {
+      "completion_length": 1409.8125495910645,
+      "epoch": 0.3302857142857143,
+      "grad_norm": 1.233174204826355,
+      "kl": 0.0773468017578125,
+      "learning_rate": 5.060876951083828e-07,
+      "loss": 0.0031,
+      "reward": 0.2632339745759964,
+      "reward_std": 0.6024043373763561,
+      "rewards/cosine_scaled_reward": 0.0020552128553390503,
+      "rewards/format_reward": 0.8541666679084301,
+      "step": 289
+    },
+    {
+      "completion_length": 1338.3125381469727,
+      "epoch": 0.3314285714285714,
+      "grad_norm": 0.5929808616638184,
+      "kl": 0.119293212890625,
+      "learning_rate": 5.02962191529556e-07,
+      "loss": 0.0048,
+      "reward": 0.48768392187776044,
+      "reward_std": 0.8819490298628807,
+      "rewards/cosine_scaled_reward": -0.019131449982523918,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 290
+    },
+    {
+      "completion_length": 1393.5208892822266,
+      "epoch": 0.3325714285714286,
+      "grad_norm": 0.6517693996429443,
+      "kl": 0.109130859375,
+      "learning_rate": 4.998389805071536e-07,
+      "loss": 0.0044,
+      "reward": 0.44597443053498864,
+      "reward_std": 0.852834016084671,
+      "rewards/cosine_scaled_reward": -0.036098250187933445,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 291
+    },
+    {
+      "completion_length": 1563.4375228881836,
+      "epoch": 0.33371428571428574,
+      "grad_norm": 0.813820481300354,
+      "kl": 0.12435150146484375,
+      "learning_rate": 4.967182142620745e-07,
+      "loss": 0.005,
+      "reward": 0.1114344063680619,
+      "reward_std": 0.6936771422624588,
+      "rewards/cosine_scaled_reward": -0.18143241526558995,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 292
+    },
+    {
+      "completion_length": 1062.9166870117188,
+      "epoch": 0.33485714285714285,
+      "grad_norm": 1.081771969795227,
+      "kl": 0.08338165283203125,
+      "learning_rate": 4.93600044896063e-07,
+      "loss": 0.0033,
+      "reward": 0.35100130061618984,
+      "reward_std": 0.6911906227469444,
+      "rewards/cosine_scaled_reward": -0.036540206521749496,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 293
+    },
+    {
+      "completion_length": 1598.583396911621,
+      "epoch": 0.336,
+      "grad_norm": 0.9857237935066223,
+      "kl": 0.11139678955078125,
+      "learning_rate": 4.904846243842949e-07,
+      "loss": 0.0045,
+      "reward": 0.2396585661917925,
+      "reward_std": 0.7594392895698547,
+      "rewards/cosine_scaled_reward": -0.07188234385102987,
+      "rewards/format_reward": 0.8125000149011612,
+      "step": 294
+    },
+    {
+      "completion_length": 1559.4167251586914,
+      "epoch": 0.33714285714285713,
+      "grad_norm": 0.7534098625183105,
+      "kl": 0.13663482666015625,
+      "learning_rate": 4.873721045679706e-07,
+      "loss": 0.0055,
+      "reward": 0.35895493626594543,
+      "reward_std": 0.9453681632876396,
+      "rewards/cosine_scaled_reward": -0.029410481452941895,
+      "rewards/format_reward": 0.8333333507180214,
+      "step": 295
+    },
+    {
+      "completion_length": 1997.6042175292969,
+      "epoch": 0.3382857142857143,
+      "grad_norm": 1.1301641464233398,
+      "kl": 0.2256317138671875,
+      "learning_rate": 4.842626371469149e-07,
+      "loss": 0.009,
+      "reward": 0.21912480238825083,
+      "reward_std": 0.8136032223701477,
+      "rewards/cosine_scaled_reward": -0.07540793996304274,
+      "rewards/format_reward": 0.7500000074505806,
+      "step": 296
+    },
+    {
+      "completion_length": 2203.666717529297,
+      "epoch": 0.3394285714285714,
+      "grad_norm": 2.0461599826812744,
+      "kl": 0.2470703125,
+      "learning_rate": 4.811563736721829e-07,
+      "loss": 0.0099,
+      "reward": 0.0711959432810545,
+      "reward_std": 0.7955613285303116,
+      "rewards/cosine_scaled_reward": -0.13813064247369766,
+      "rewards/format_reward": 0.625000013038516,
+      "step": 297
+    },
+    {
+      "completion_length": 1511.0417098999023,
+      "epoch": 0.3405714285714286,
+      "grad_norm": 2.8553810119628906,
+      "kl": 0.27417755126953125,
+      "learning_rate": 4.780534655386743e-07,
+      "loss": 0.011,
+      "reward": 0.22325835039373487,
+      "reward_std": 0.7070137523114681,
+      "rewards/cosine_scaled_reward": -0.10693333297967911,
+      "rewards/format_reward": 0.8750000223517418,
+      "step": 298
+    },
+    {
+      "completion_length": 1423.8750381469727,
+      "epoch": 0.3417142857142857,
+      "grad_norm": 0.9705626368522644,
+      "kl": 0.1470947265625,
+      "learning_rate": 4.749540639777539e-07,
+      "loss": 0.0059,
+      "reward": 0.20597740169614553,
+      "reward_std": 0.7050324305891991,
+      "rewards/cosine_scaled_reward": -0.06768750678747892,
+      "rewards/format_reward": 0.7916666846722364,
+      "step": 299
+    },
+    {
+      "completion_length": 1679.583381652832,
+      "epoch": 0.34285714285714286,
+      "grad_norm": 1.6013585329055786,
+      "kl": 0.2515869140625,
+      "learning_rate": 4.7185832004988133e-07,
+      "loss": 0.0101,
+      "reward": 0.16855086106806993,
+      "reward_std": 0.7662245742976665,
+      "rewards/cosine_scaled_reward": -0.11777076427824795,
+      "rewards/format_reward": 0.7291666902601719,
+      "step": 300
+    },
+    {
+      "completion_length": 1467.8333740234375,
+      "epoch": 0.344,
+      "grad_norm": 1.4137245416641235,
+      "kl": 0.14666748046875,
+      "learning_rate": 4.68766384637248e-07,
+      "loss": 0.0059,
+      "reward": 0.34917816892266273,
+      "reward_std": 0.826218631118536,
+      "rewards/cosine_scaled_reward": -0.0683151277480647,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 301
+    },
+    {
+      "completion_length": 1356.4167175292969,
+      "epoch": 0.34514285714285714,
+      "grad_norm": 1.1879962682724,
+      "kl": 0.149932861328125,
+      "learning_rate": 4.656784084364238e-07,
+      "loss": 0.006,
+      "reward": 0.36293663922697306,
+      "reward_std": 0.8348271325230598,
+      "rewards/cosine_scaled_reward": 0.008066533133387566,
+      "rewards/format_reward": 0.8125000111758709,
+      "step": 302
+    },
+    {
+      "completion_length": 1341.3541870117188,
+      "epoch": 0.3462857142857143,
+      "grad_norm": 1.171494483947754,
+      "kl": 0.18246078491210938,
+      "learning_rate": 4.6259454195101267e-07,
+      "loss": 0.0073,
+      "reward": 0.2431361076887697,
+      "reward_std": 0.7933157682418823,
+      "rewards/cosine_scaled_reward": -0.1022418315988034,
+      "rewards/format_reward": 0.854166679084301,
+      "step": 303
+    },
+    {
+      "completion_length": 1238.6667175292969,
+      "epoch": 0.3474285714285714,
+      "grad_norm": 0.9185214042663574,
+      "kl": 0.08075714111328125,
+      "learning_rate": 4.59514935484316e-07,
+      "loss": 0.0032,
+      "reward": 0.26590197812765837,
+      "reward_std": 0.6553149335086346,
+      "rewards/cosine_scaled_reward": -0.07919650059193373,
+      "rewards/format_reward": 0.8958333358168602,
+      "step": 304
+    },
+    {
+      "completion_length": 1323.9791946411133,
+      "epoch": 0.3485714285714286,
+      "grad_norm": 1.0339250564575195,
+      "kl": 0.17450332641601562,
+      "learning_rate": 4.5643973913200837e-07,
+      "loss": 0.007,
+      "reward": 0.13226244208635762,
+      "reward_std": 0.6713650859892368,
+      "rewards/cosine_scaled_reward": -0.16428566398099065,
+      "rewards/format_reward": 0.8541666865348816,
+      "step": 305
+    },
+    {
+      "completion_length": 1336.8959045410156,
+      "epoch": 0.3497142857142857,
+      "grad_norm": 1.3460803031921387,
+      "kl": 0.196563720703125,
+      "learning_rate": 4.5336910277482155e-07,
+      "loss": 0.0079,
+      "reward": 0.6428653690963984,
+      "reward_std": 1.0474185049533844,
+      "rewards/cosine_scaled_reward": 0.14976803492754698,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 306
+    },
+    {
+      "completion_length": 1476.3542098999023,
+      "epoch": 0.35085714285714287,
+      "grad_norm": 2.0747716426849365,
+      "kl": 0.23199462890625,
+      "learning_rate": 4.503031760712397e-07,
+      "loss": 0.0093,
+      "reward": 0.3243725663051009,
+      "reward_std": 0.7664116658270359,
+      "rewards/cosine_scaled_reward": -0.016992317512631416,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 307
+    },
+    {
+      "completion_length": 2029.166732788086,
+      "epoch": 0.352,
+      "grad_norm": 1.3981530666351318,
+      "kl": 0.3675079345703125,
+      "learning_rate": 4.4724210845020494e-07,
+      "loss": 0.0147,
+      "reward": 0.14206439611734822,
+      "reward_std": 0.7617513313889503,
+      "rewards/cosine_scaled_reward": -0.12085138214752078,
+      "rewards/format_reward": 0.7500000111758709,
+      "step": 308
+    },
+    {
+      "completion_length": 1717.7292175292969,
+      "epoch": 0.35314285714285715,
+      "grad_norm": 1.3776211738586426,
+      "kl": 0.27923583984375,
+      "learning_rate": 4.441860491038345e-07,
+      "loss": 0.0112,
+      "reward": 0.26929385494440794,
+      "reward_std": 0.8059471026062965,
+      "rewards/cosine_scaled_reward": -0.06185213173739612,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 309
+    },
+    {
+      "completion_length": 1247.7917098999023,
+      "epoch": 0.35428571428571426,
+      "grad_norm": 1.195373773574829,
+      "kl": 0.13482666015625,
+      "learning_rate": 4.4113514698014953e-07,
+      "loss": 0.0054,
+      "reward": 0.20025933103170246,
+      "reward_std": 0.6624983102083206,
+      "rewards/cosine_scaled_reward": -0.13693573721684515,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 310
+    },
+    {
+      "completion_length": 1230.8333702087402,
+      "epoch": 0.3554285714285714,
+      "grad_norm": 1.7214908599853516,
+      "kl": 0.16667938232421875,
+      "learning_rate": 4.3808955077581546e-07,
+      "loss": 0.0067,
+      "reward": 0.48247593361884356,
+      "reward_std": 0.8122721910476685,
+      "rewards/cosine_scaled_reward": 0.02081843838095665,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 311
+    },
+    {
+      "completion_length": 1083.6875228881836,
+      "epoch": 0.3565714285714286,
+      "grad_norm": 1.3621957302093506,
+      "kl": 0.15818023681640625,
+      "learning_rate": 4.350494089288943e-07,
+      "loss": 0.0063,
+      "reward": 0.5721160881221294,
+      "reward_std": 0.6696533262729645,
+      "rewards/cosine_scaled_reward": 0.23700525425374508,
+      "rewards/format_reward": 0.8750000111758709,
+      "step": 312
+    },
+    {
+      "completion_length": 1498.3750228881836,
+      "epoch": 0.3577142857142857,
+      "grad_norm": 1.5137181282043457,
+      "kl": 0.31603240966796875,
+      "learning_rate": 4.3201486961161093e-07,
+      "loss": 0.0126,
+      "reward": 0.4493098706007004,
+      "reward_std": 0.8899626843631268,
+      "rewards/cosine_scaled_reward": 0.060429781675338745,
+      "rewards/format_reward": 0.7916666734963655,
+      "step": 313
+    },
+    {
+      "completion_length": 1152.7083702087402,
+      "epoch": 0.3588571428571429,
+      "grad_norm": 1.3737326860427856,
+      "kl": 0.178466796875,
+      "learning_rate": 4.2898608072313045e-07,
+      "loss": 0.0071,
+      "reward": 0.42835053242743015,
+      "reward_std": 0.7351183108985424,
+      "rewards/cosine_scaled_reward": 0.03823569389896875,
+      "rewards/format_reward": 0.8750000223517418,
+      "step": 314
+    },
+    {
+      "completion_length": 1325.7709045410156,
+      "epoch": 0.36,
+      "grad_norm": 4.047604560852051,
+      "kl": 0.39301300048828125,
+      "learning_rate": 4.2596318988235037e-07,
+      "loss": 0.0157,
+      "reward": 0.33133709616959095,
+      "reward_std": 0.82825917750597,
+      "rewards/cosine_scaled_reward": 0.003983840346336365,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 315
+    },
+    {
+      "completion_length": 1725.6042137145996,
+      "epoch": 0.36114285714285715,
+      "grad_norm": 2.519155979156494,
+      "kl": 0.63311767578125,
+      "learning_rate": 4.2294634442070553e-07,
+      "loss": 0.0253,
+      "reward": 0.09543975442647934,
+      "reward_std": 0.8387586250901222,
+      "rewards/cosine_scaled_reward": -0.15274116187356412,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 316
+    },
+    {
+      "completion_length": 1519.8333740234375,
+      "epoch": 0.36228571428571427,
+      "grad_norm": 2.768705368041992,
+      "kl": 0.483062744140625,
+      "learning_rate": 4.1993569137498776e-07,
+      "loss": 0.0193,
+      "reward": 0.35648738220334053,
+      "reward_std": 0.8188385404646397,
+      "rewards/cosine_scaled_reward": -0.0025814222171902657,
+      "rewards/format_reward": 0.7708333469927311,
+      "step": 317
+    },
+    {
+      "completion_length": 1004.2500343322754,
+      "epoch": 0.36342857142857143,
+      "grad_norm": 0.982016921043396,
+      "kl": 0.1404876708984375,
+      "learning_rate": 4.1693137748017915e-07,
+      "loss": 0.0056,
+      "reward": 0.17383845895528793,
+      "reward_std": 0.699553694576025,
+      "rewards/cosine_scaled_reward": -0.15935352514497936,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 318
+    },
+    {
+      "completion_length": 1440.8750305175781,
+      "epoch": 0.36457142857142855,
+      "grad_norm": 2.472060203552246,
+      "kl": 0.6479034423828125,
+      "learning_rate": 4.1393354916230005e-07,
+      "loss": 0.0259,
+      "reward": 0.10745102865621448,
+      "reward_std": 0.8851590566337109,
+      "rewards/cosine_scaled_reward": -0.18445847602561116,
+      "rewards/format_reward": 0.7500000186264515,
+      "step": 319
+    },
+    {
+      "completion_length": 872.4791946411133,
+      "epoch": 0.3657142857142857,
+      "grad_norm": 1.969278335571289,
+      "kl": 0.3460540771484375,
+      "learning_rate": 4.1094235253127374e-07,
+      "loss": 0.0139,
+      "reward": 0.4431753905955702,
+      "reward_std": 0.7728302180767059,
+      "rewards/cosine_scaled_reward": 0.03887226711958647,
+      "rewards/format_reward": 0.8958333507180214,
+      "step": 320
+    },
+    {
+      "completion_length": 1001.3541946411133,
+      "epoch": 0.3668571428571429,
+      "grad_norm": 1.5442945957183838,
+      "kl": 0.4463653564453125,
+      "learning_rate": 4.079579333738039e-07,
+      "loss": 0.0179,
+      "reward": 0.5132068395614624,
+      "reward_std": 0.9250943809747696,
+      "rewards/cosine_scaled_reward": 0.06521461345255375,
+      "rewards/format_reward": 0.8750000074505806,
+      "step": 321
+    },
+    {
+      "completion_length": 1230.1666946411133,
+      "epoch": 0.368,
+      "grad_norm": 4.6210222244262695,
+      "kl": 0.856231689453125,
+      "learning_rate": 4.0498043714627006e-07,
+      "loss": 0.0342,
+      "reward": 0.21194136049598455,
+      "reward_std": 0.8229392170906067,
+      "rewards/cosine_scaled_reward": -0.1281059831380844,
+      "rewards/format_reward": 0.8125000298023224,
+      "step": 322
+    },
+    {
+      "completion_length": 1046.5833740234375,
+      "epoch": 0.36914285714285716,
+      "grad_norm": 2.3179831504821777,
+      "kl": 0.600982666015625,
+      "learning_rate": 4.020100089676376e-07,
+      "loss": 0.0241,
+      "reward": 0.17017683573067188,
+      "reward_std": 0.6333070918917656,
+      "rewards/cosine_scaled_reward": -0.09792062174528837,
+      "rewards/format_reward": 0.8333333507180214,
+      "step": 323
+    },
+    {
+      "completion_length": 1352.1666946411133,
+      "epoch": 0.3702857142857143,
+      "grad_norm": 4.481067657470703,
+      "kl": 1.1396102905273438,
+      "learning_rate": 3.9904679361238526e-07,
+      "loss": 0.0455,
+      "reward": 0.1608973522670567,
+      "reward_std": 0.7919485196471214,
+      "rewards/cosine_scaled_reward": -0.13527273340150714,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 324
+    },
+    {
+      "completion_length": 1022.145866394043,
+      "epoch": 0.37142857142857144,
+      "grad_norm": 3.168888807296753,
+      "kl": 0.7562255859375,
+      "learning_rate": 3.9609093550344907e-07,
+      "loss": 0.0303,
+      "reward": 0.255460548796691,
+      "reward_std": 0.714014045894146,
+      "rewards/cosine_scaled_reward": -0.0987097217439441,
+      "rewards/format_reward": 0.8750000149011612,
+      "step": 325
+    },
+    {
+      "completion_length": 1067.8541946411133,
+      "epoch": 0.37257142857142855,
+      "grad_norm": 6.944228172302246,
+      "kl": 0.607666015625,
+      "learning_rate": 3.931425787051832e-07,
+      "loss": 0.0243,
+      "reward": 0.34114629309624434,
+      "reward_std": 0.7636858969926834,
+      "rewards/cosine_scaled_reward": -0.050329115241765976,
+      "rewards/format_reward": 0.8958333507180214,
+      "step": 326
+    },
+    {
+      "completion_length": 1227.4375610351562,
+      "epoch": 0.3737142857142857,
+      "grad_norm": 1.7565664052963257,
+      "kl": 0.2918243408203125,
+      "learning_rate": 3.902018669163384e-07,
+      "loss": 0.0117,
+      "reward": 0.4599468493834138,
+      "reward_std": 0.6929560974240303,
+      "rewards/cosine_scaled_reward": 0.0536233875900507,
+      "rewards/format_reward": 0.8958333507180214,
+      "step": 327
+    },
+    {
+      "completion_length": 1539.4792556762695,
+      "epoch": 0.37485714285714283,
+      "grad_norm": 7.016115188598633,
+      "kl": 1.123626708984375,
+      "learning_rate": 3.872689434630585e-07,
+      "loss": 0.045,
+      "reward": 0.051082022255286574,
+      "reward_std": 0.7887684628367424,
+      "rewards/cosine_scaled_reward": -0.199990039691329,
+      "rewards/format_reward": 0.7500000186264515,
+      "step": 328
+    },
+    {
+      "completion_length": 960.0208778381348,
+      "epoch": 0.376,
+      "grad_norm": 2.32981014251709,
+      "kl": 0.53167724609375,
+      "learning_rate": 3.843439512918949e-07,
+      "loss": 0.0212,
+      "reward": 0.3326158430427313,
+      "reward_std": 0.6615541912615299,
+      "rewards/cosine_scaled_reward": 0.0014219898730516434,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 329
+    },
+    {
+      "completion_length": 1015.9583740234375,
+      "epoch": 0.37714285714285717,
+      "grad_norm": 2.246696949005127,
+      "kl": 0.37122344970703125,
+      "learning_rate": 3.8142703296283953e-07,
+      "loss": 0.0149,
+      "reward": 0.3060060928110033,
+      "reward_std": 0.7870642617344856,
+      "rewards/cosine_scaled_reward": -0.09269801783375442,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 330
+    },
+    {
+      "completion_length": 1435.8333587646484,
+      "epoch": 0.3782857142857143,
+      "grad_norm": 2.061875104904175,
+      "kl": 1.0668792724609375,
+      "learning_rate": 3.785183306423767e-07,
+      "loss": 0.0428,
+      "reward": 0.12969208881258965,
+      "reward_std": 0.6653396300971508,
+      "rewards/cosine_scaled_reward": -0.09254590002819896,
+      "rewards/format_reward": 0.7708333432674408,
+      "step": 331
+    },
+    {
+      "completion_length": 1591.333366394043,
+      "epoch": 0.37942857142857145,
+      "grad_norm": 3.716142177581787,
+      "kl": 0.729095458984375,
+      "learning_rate": 3.7561798609655373e-07,
+      "loss": 0.0291,
+      "reward": 0.08031724044121802,
+      "reward_std": 0.6911140829324722,
+      "rewards/cosine_scaled_reward": -0.18430401291698217,
+      "rewards/format_reward": 0.7916666939854622,
+      "step": 332
+    },
+    {
+      "completion_length": 1069.1875343322754,
+      "epoch": 0.38057142857142856,
+      "grad_norm": 4.158648490905762,
+      "kl": 0.4372711181640625,
+      "learning_rate": 3.72726140684072e-07,
+      "loss": 0.0175,
+      "reward": 0.406515009701252,
+      "reward_std": 0.9121326096355915,
+      "rewards/cosine_scaled_reward": -0.03547548362985253,
+      "rewards/format_reward": 0.8541666865348816,
+      "step": 333
+    },
+    {
+      "completion_length": 1548.5625305175781,
+      "epoch": 0.38171428571428573,
+      "grad_norm": 2.0228302478790283,
+      "kl": 1.1025390625,
+      "learning_rate": 3.6984293534939737e-07,
+      "loss": 0.0441,
+      "reward": 0.0005341523792594671,
+      "reward_std": 0.6353342607617378,
+      "rewards/cosine_scaled_reward": -0.2071850085631013,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 334
+    },
+    {
+      "completion_length": 1218.1458740234375,
+      "epoch": 0.38285714285714284,
+      "grad_norm": 1.6079083681106567,
+      "kl": 0.4420585632324219,
+      "learning_rate": 3.6696851061588994e-07,
+      "loss": 0.0177,
+      "reward": 0.5085492568905465,
+      "reward_std": 0.9739002659916878,
+      "rewards/cosine_scaled_reward": 0.027598066721111536,
+      "rewards/format_reward": 0.8750000223517418,
+      "step": 335
+    },
+    {
+      "completion_length": 1167.2917175292969,
+      "epoch": 0.384,
+      "grad_norm": 2.6033425331115723,
+      "kl": 0.45883941650390625,
+      "learning_rate": 3.641030065789562e-07,
+      "loss": 0.0184,
+      "reward": 0.43186818808317184,
+      "reward_std": 0.8531835786998272,
+      "rewards/cosine_scaled_reward": 0.05448053032159805,
+      "rewards/format_reward": 0.8125000111758709,
+      "step": 336
+    },
+    {
+      "completion_length": 1327.1042022705078,
+      "epoch": 0.3851428571428571,
+      "grad_norm": 3.292928695678711,
+      "kl": 0.3748321533203125,
+      "learning_rate": 3.612465628992203e-07,
+      "loss": 0.015,
+      "reward": 0.2253723087196704,
+      "reward_std": 0.7758054882287979,
+      "rewards/cosine_scaled_reward": -0.13347763079218566,
+      "rewards/format_reward": 0.895833358168602,
+      "step": 337
+    },
+    {
+      "completion_length": 1171.1875190734863,
+      "epoch": 0.3862857142857143,
+      "grad_norm": 6.154543876647949,
+      "kl": 1.41033935546875,
+      "learning_rate": 3.5839931879571725e-07,
+      "loss": 0.0567,
+      "reward": 0.21605271194130182,
+      "reward_std": 0.6188852116465569,
+      "rewards/cosine_scaled_reward": -0.02803803514689207,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 338
+    },
+    {
+      "completion_length": 1353.3750381469727,
+      "epoch": 0.38742857142857146,
+      "grad_norm": 4.424708843231201,
+      "kl": 0.518829345703125,
+      "learning_rate": 3.555614130391079e-07,
+      "loss": 0.0208,
+      "reward": 0.09727495489642024,
+      "reward_std": 0.6290086731314659,
+      "rewards/cosine_scaled_reward": -0.1305898940190673,
+      "rewards/format_reward": 0.7291666772216558,
+      "step": 339
+    },
+    {
+      "completion_length": 1321.708381652832,
+      "epoch": 0.38857142857142857,
+      "grad_norm": 1.7365025281906128,
+      "kl": 1.028076171875,
+      "learning_rate": 3.5273298394491515e-07,
+      "loss": 0.0411,
+      "reward": 0.2937823599204421,
+      "reward_std": 0.857489574700594,
+      "rewards/cosine_scaled_reward": -0.07948380801826715,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 340
+    },
+    {
+      "completion_length": 1008.5208740234375,
+      "epoch": 0.38971428571428574,
+      "grad_norm": 2.895829677581787,
+      "kl": 0.6253280639648438,
+      "learning_rate": 3.4991416936678276e-07,
+      "loss": 0.025,
+      "reward": 0.5628593787550926,
+      "reward_std": 0.7620139196515083,
+      "rewards/cosine_scaled_reward": 0.1886910004541278,
+      "rewards/format_reward": 0.7500000074505806,
+      "step": 341
+    },
+    {
+      "completion_length": 1325.9792175292969,
+      "epoch": 0.39085714285714285,
+      "grad_norm": 3.778715133666992,
+      "kl": 1.060302734375,
+      "learning_rate": 3.471051066897562e-07,
+      "loss": 0.0424,
+      "reward": 0.49639496579766273,
+      "reward_std": 0.9629452601075172,
+      "rewards/cosine_scaled_reward": 0.022836442454718053,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 342
+    },
+    {
+      "completion_length": 1301.9792098999023,
+      "epoch": 0.392,
+      "grad_norm": 4.3889546394348145,
+      "kl": 0.91851806640625,
+      "learning_rate": 3.4430593282358777e-07,
+      "loss": 0.0368,
+      "reward": 0.5952606732025743,
+      "reward_std": 0.8778381422162056,
+      "rewards/cosine_scaled_reward": 0.0880544763058424,
+      "rewards/format_reward": 0.8750000223517418,
+      "step": 343
+    },
+    {
+      "completion_length": 1580.520866394043,
+      "epoch": 0.3931428571428571,
+      "grad_norm": 2.815673589706421,
+      "kl": 1.52044677734375,
+      "learning_rate": 3.4151678419606233e-07,
+      "loss": 0.0607,
+      "reward": 0.5273411017842591,
+      "reward_std": 0.7953666783869267,
+      "rewards/cosine_scaled_reward": 0.1287881238386035,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 344
+    },
+    {
+      "completion_length": 1205.958381652832,
+      "epoch": 0.3942857142857143,
+      "grad_norm": 4.648168563842773,
+      "kl": 1.01983642578125,
+      "learning_rate": 3.387377967463493e-07,
+      "loss": 0.0409,
+      "reward": 0.3727841805666685,
+      "reward_std": 0.866691779345274,
+      "rewards/cosine_scaled_reward": -0.026965959696099162,
+      "rewards/format_reward": 0.8125000223517418,
+      "step": 345
+    },
+    {
+      "completion_length": 1131.4375381469727,
+      "epoch": 0.3954285714285714,
+      "grad_norm": 1.843777060508728,
+      "kl": 0.547515869140625,
+      "learning_rate": 3.359691059183761e-07,
+      "loss": 0.0219,
+      "reward": 0.3929372038692236,
+      "reward_std": 0.8282114900648594,
+      "rewards/cosine_scaled_reward": -0.039126552641391754,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 346
+    },
+    {
+      "completion_length": 1396.1250381469727,
+      "epoch": 0.3965714285714286,
+      "grad_norm": 2.772951126098633,
+      "kl": 0.83648681640625,
+      "learning_rate": 3.3321084665422803e-07,
+      "loss": 0.0334,
+      "reward": 0.04932975070551038,
+      "reward_std": 0.7110629379749298,
+      "rewards/cosine_scaled_reward": -0.2151939356699586,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 347
+    },
+    {
+      "completion_length": 1094.2500381469727,
+      "epoch": 0.3977142857142857,
+      "grad_norm": 3.0713284015655518,
+      "kl": 0.55780029296875,
+      "learning_rate": 3.3046315338757026e-07,
+      "loss": 0.0223,
+      "reward": 0.3814494190737605,
+      "reward_std": 0.7161810956895351,
+      "rewards/cosine_scaled_reward": -0.050093160942196846,
+      "rewards/format_reward": 0.9583333432674408,
+      "step": 348
+    },
+    {
+      "completion_length": 1494.020866394043,
+      "epoch": 0.39885714285714285,
+      "grad_norm": 7.005665302276611,
+      "kl": 1.7324066162109375,
+      "learning_rate": 3.2772616003709616e-07,
+      "loss": 0.0694,
+      "reward": 0.3296292170416564,
+      "reward_std": 0.8451166786253452,
+      "rewards/cosine_scaled_reward": -0.04683339223265648,
+      "rewards/format_reward": 0.8125000260770321,
+      "step": 349
+    },
+    {
+      "completion_length": 1154.4167022705078,
+      "epoch": 0.4,
+      "grad_norm": 1.4422372579574585,
+      "kl": 0.5752105712890625,
+      "learning_rate": 3.250000000000001e-07,
+      "loss": 0.023,
+      "reward": 0.2720920806750655,
+      "reward_std": 0.9464740082621574,
+      "rewards/cosine_scaled_reward": -0.13292736560106277,
+      "rewards/format_reward": 0.854166679084301,
+      "step": 350
+    },
+    {
+      "completion_length": 1267.2708892822266,
+      "epoch": 0.40114285714285713,
+      "grad_norm": 2.480191946029663,
+      "kl": 0.645751953125,
+      "learning_rate": 3.222848061454764e-07,
+      "loss": 0.0258,
+      "reward": 0.17013419978320599,
+      "reward_std": 0.7632426917552948,
+      "rewards/cosine_scaled_reward": -0.08551971893757582,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 351
+    },
+    {
+      "completion_length": 1286.5000228881836,
+      "epoch": 0.4022857142857143,
+      "grad_norm": 2.153872489929199,
+      "kl": 0.580810546875,
+      "learning_rate": 3.195807108082429e-07,
+      "loss": 0.0232,
+      "reward": 0.12328234064625576,
+      "reward_std": 0.8743564449250698,
+      "rewards/cosine_scaled_reward": -0.14927133545279503,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 352
+    },
+    {
+      "completion_length": 1413.1042251586914,
+      "epoch": 0.4034285714285714,
+      "grad_norm": 2.458132266998291,
+      "kl": 0.6356201171875,
+      "learning_rate": 3.168878457820915e-07,
+      "loss": 0.0255,
+      "reward": 0.47827470442280173,
+      "reward_std": 0.9372801259160042,
+      "rewards/cosine_scaled_reward": 0.06890620663762093,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 353
+    },
+    {
+      "completion_length": 1377.5000457763672,
+      "epoch": 0.4045714285714286,
+      "grad_norm": 1.8596700429916382,
+      "kl": 0.601806640625,
+      "learning_rate": 3.142063423134644e-07,
+      "loss": 0.0241,
+      "reward": 0.3115939125418663,
+      "reward_std": 0.9930433630943298,
+      "rewards/cosine_scaled_reward": -0.01708403415977955,
+      "rewards/format_reward": 0.6458333544433117,
+      "step": 354
+    },
+    {
+      "completion_length": 1212.645881652832,
+      "epoch": 0.4057142857142857,
+      "grad_norm": 1.9313182830810547,
+      "kl": 0.514892578125,
+      "learning_rate": 3.115363310950578e-07,
+      "loss": 0.0206,
+      "reward": 0.17402092670090497,
+      "reward_std": 0.8653297163546085,
+      "rewards/cosine_scaled_reward": -0.06491286307573318,
+      "rewards/format_reward": 0.6250000149011612,
+      "step": 355
+    },
+    {
+      "completion_length": 1633.4584197998047,
+      "epoch": 0.40685714285714286,
+      "grad_norm": 1.906638503074646,
+      "kl": 0.5458984375,
+      "learning_rate": 3.0887794225945143e-07,
+      "loss": 0.0218,
+      "reward": 0.2535629828926176,
+      "reward_std": 0.8886833339929581,
+      "rewards/cosine_scaled_reward": -0.06057508382946253,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 356
+    },
+    {
+      "completion_length": 1881.8751029968262,
+      "epoch": 0.408,
+      "grad_norm": 2.3647892475128174,
+      "kl": 0.819091796875,
+      "learning_rate": 3.062313053727671e-07,
+      "loss": 0.0328,
+      "reward": -0.11886521428823471,
+      "reward_std": 0.6593221351504326,
+      "rewards/cosine_scaled_reward": -0.2341746687889099,
+      "rewards/format_reward": 0.6250000186264515,
+      "step": 357
+    },
+    {
+      "completion_length": 1829.6250305175781,
+      "epoch": 0.40914285714285714,
+      "grad_norm": 1.856693148612976,
+      "kl": 0.70489501953125,
+      "learning_rate": 3.0359654942835247e-07,
+      "loss": 0.0282,
+      "reward": 0.1773190926760435,
+      "reward_std": 0.8244731090962887,
+      "rewards/cosine_scaled_reward": -0.05620474135503173,
+      "rewards/format_reward": 0.5833333544433117,
+      "step": 358
+    },
+    {
+      "completion_length": 1053.9375305175781,
+      "epoch": 0.4102857142857143,
+      "grad_norm": 3.442803382873535,
+      "kl": 0.8038330078125,
+      "learning_rate": 3.0097380284049523e-07,
+      "loss": 0.0322,
+      "reward": 0.16565631795674562,
+      "reward_std": 0.8398986533284187,
+      "rewards/cosine_scaled_reward": -0.12338497827295214,
+      "rewards/format_reward": 0.7291666939854622,
+      "step": 359
+    },
+    {
+      "completion_length": 1288.1875305175781,
+      "epoch": 0.4114285714285714,
+      "grad_norm": 1.7091789245605469,
+      "kl": 0.526123046875,
+      "learning_rate": 2.9836319343816397e-07,
+      "loss": 0.0211,
+      "reward": 0.2598306848667562,
+      "reward_std": 0.8984776921570301,
+      "rewards/cosine_scaled_reward": -0.05199693702161312,
+      "rewards/format_reward": 0.6875000223517418,
+      "step": 360
+    },
+    {
+      "completion_length": 1407.9583435058594,
+      "epoch": 0.4125714285714286,
+      "grad_norm": 3.6575090885162354,
+      "kl": 0.920166015625,
+      "learning_rate": 2.9576484845877793e-07,
+      "loss": 0.0368,
+      "reward": 0.022162297973409295,
+      "reward_std": 0.6297386400401592,
+      "rewards/cosine_scaled_reward": -0.17034196108579636,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 361
+    },
+    {
+      "completion_length": 1246.812572479248,
+      "epoch": 0.4137142857142857,
+      "grad_norm": 3.3759262561798096,
+      "kl": 0.9285125732421875,
+      "learning_rate": 2.931788945420058e-07,
+      "loss": 0.0371,
+      "reward": 0.17949877493083477,
+      "reward_std": 0.7008433230221272,
+      "rewards/cosine_scaled_reward": -0.06406344473361969,
+      "rewards/format_reward": 0.6875000204890966,
+      "step": 362
+    },
+    {
+      "completion_length": 935.9166870117188,
+      "epoch": 0.41485714285714287,
+      "grad_norm": 2.3449277877807617,
+      "kl": 0.5015869140625,
+      "learning_rate": 2.9060545772359305e-07,
+      "loss": 0.02,
+      "reward": 0.462602804065682,
+      "reward_std": 0.8693399466574192,
+      "rewards/cosine_scaled_reward": 0.1225482877343893,
+      "rewards/format_reward": 0.687500013038516,
+      "step": 363
+    },
+    {
+      "completion_length": 1301.6042098999023,
+      "epoch": 0.416,
+      "grad_norm": 2.7286217212677,
+      "kl": 0.5523681640625,
+      "learning_rate": 2.8804466342921987e-07,
+      "loss": 0.0221,
+      "reward": -0.18497540964744985,
+      "reward_std": 0.596361830830574,
+      "rewards/cosine_scaled_reward": -0.2775970436632633,
+      "rewards/format_reward": 0.6458333507180214,
+      "step": 364
+    },
+    {
+      "completion_length": 1725.3958587646484,
+      "epoch": 0.41714285714285715,
+      "grad_norm": 2.718860387802124,
+      "kl": 0.6279296875,
+      "learning_rate": 2.854966364683872e-07,
+      "loss": 0.0252,
+      "reward": 0.07833639718592167,
+      "reward_std": 0.783052921295166,
+      "rewards/cosine_scaled_reward": -0.14490601420402527,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 365
+    },
+    {
+      "completion_length": 1542.166690826416,
+      "epoch": 0.41828571428571426,
+      "grad_norm": 2.4428579807281494,
+      "kl": 0.68994140625,
+      "learning_rate": 2.829615010283344e-07,
+      "loss": 0.0276,
+      "reward": 0.2222783851902932,
+      "reward_std": 0.9212566986680031,
+      "rewards/cosine_scaled_reward": 0.022268712986260653,
+      "rewards/format_reward": 0.5416666753590107,
+      "step": 366
+    },
+    {
+      "completion_length": 1529.7708854675293,
+      "epoch": 0.41942857142857143,
+      "grad_norm": 1.7189098596572876,
+      "kl": 0.618408203125,
+      "learning_rate": 2.8043938066798645e-07,
+      "loss": 0.0247,
+      "reward": 0.13789566839113832,
+      "reward_std": 0.7518769651651382,
+      "rewards/cosine_scaled_reward": -0.08000693749636412,
+      "rewards/format_reward": 0.6458333525806665,
+      "step": 367
+    },
+    {
+      "completion_length": 1874.6250610351562,
+      "epoch": 0.4205714285714286,
+      "grad_norm": 2.6109297275543213,
+      "kl": 0.5501708984375,
+      "learning_rate": 2.7793039831193133e-07,
+      "loss": 0.022,
+      "reward": 0.1437191739678383,
+      "reward_std": 0.809885136783123,
+      "rewards/cosine_scaled_reward": -0.07925083208829165,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 368
+    },
+    {
+      "completion_length": 1595.6458892822266,
+      "epoch": 0.4217142857142857,
+      "grad_norm": 2.485452890396118,
+      "kl": 0.75439453125,
+      "learning_rate": 2.7543467624442956e-07,
+      "loss": 0.0302,
+      "reward": 0.23247148096561432,
+      "reward_std": 0.9586904980242252,
+      "rewards/cosine_scaled_reward": -0.044586887350305915,
+      "rewards/format_reward": 0.6041666809469461,
+      "step": 369
+    },
+    {
+      "completion_length": 1477.1458740234375,
+      "epoch": 0.4228571428571429,
+      "grad_norm": 2.785557508468628,
+      "kl": 0.76904296875,
+      "learning_rate": 2.729523361034538e-07,
+      "loss": 0.0308,
+      "reward": 0.12985735037364066,
+      "reward_std": 0.6629677005112171,
+      "rewards/cosine_scaled_reward": -0.04906688700430095,
+      "rewards/format_reward": 0.6250000149011612,
+      "step": 370
+    },
+    {
+      "completion_length": 1081.5417079925537,
+      "epoch": 0.424,
+      "grad_norm": 2.7538247108459473,
+      "kl": 0.563232421875,
+      "learning_rate": 2.7048349887476037e-07,
+      "loss": 0.0225,
+      "reward": 0.592208469286561,
+      "reward_std": 0.7628613486886024,
+      "rewards/cosine_scaled_reward": 0.19106208952143788,
+      "rewards/format_reward": 0.8125000186264515,
+      "step": 371
+    },
+    {
+      "completion_length": 1689.6458892822266,
+      "epoch": 0.42514285714285716,
+      "grad_norm": 2.427741050720215,
+      "kl": 0.419677734375,
+      "learning_rate": 2.6802828488599294e-07,
+      "loss": 0.0168,
+      "reward": 0.2929907846264541,
+      "reward_std": 0.8767264820635319,
+      "rewards/cosine_scaled_reward": -0.03612378612160683,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 372
+    },
+    {
+      "completion_length": 1054.0833702087402,
+      "epoch": 0.42628571428571427,
+      "grad_norm": 3.8023006916046143,
+      "kl": 0.587646484375,
+      "learning_rate": 2.655868138008171e-07,
+      "loss": 0.0235,
+      "reward": 0.05784207722172141,
+      "reward_std": 0.7628821134567261,
+      "rewards/cosine_scaled_reward": -0.17036252235993743,
+      "rewards/format_reward": 0.6875000260770321,
+      "step": 373
+    },
+    {
+      "completion_length": 1296.9375305175781,
+      "epoch": 0.42742857142857144,
+      "grad_norm": 3.30541729927063,
+      "kl": 0.861572265625,
+      "learning_rate": 2.631592046130896e-07,
+      "loss": 0.0345,
+      "reward": 0.2668555803829804,
+      "reward_std": 0.8589187040925026,
+      "rewards/cosine_scaled_reward": -0.026118143810890615,
+      "rewards/format_reward": 0.6666666753590107,
+      "step": 374
+    },
+    {
+      "completion_length": 1779.8958549499512,
+      "epoch": 0.42857142857142855,
+      "grad_norm": 3.2860774993896484,
+      "kl": 0.71484375,
+      "learning_rate": 2.6074557564105724e-07,
+      "loss": 0.0286,
+      "reward": 0.2315208874642849,
+      "reward_std": 0.9205419048666954,
+      "rewards/cosine_scaled_reward": -0.08181109488941729,
+      "rewards/format_reward": 0.6875000260770321,
+      "step": 375
+    },
+    {
+      "completion_length": 1341.812515258789,
+      "epoch": 0.4297142857142857,
+      "grad_norm": 1.9312739372253418,
+      "kl": 0.6658935546875,
+      "learning_rate": 2.583460445215911e-07,
+      "loss": 0.0266,
+      "reward": 0.11426342278718948,
+      "reward_std": 0.7130968421697617,
+      "rewards/cosine_scaled_reward": -0.13826850580517203,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 376
+    },
+    {
+      "completion_length": 1541.6042098999023,
+      "epoch": 0.4308571428571429,
+      "grad_norm": 1.676692008972168,
+      "kl": 0.5164794921875,
+      "learning_rate": 2.5596072820445254e-07,
+      "loss": 0.0206,
+      "reward": 0.11265636514872313,
+      "reward_std": 0.6777131371200085,
+      "rewards/cosine_scaled_reward": -0.12696546595543623,
+      "rewards/format_reward": 0.7291666939854622,
+      "step": 377
+    },
+    {
+      "completion_length": 1453.2917251586914,
+      "epoch": 0.432,
+      "grad_norm": 2.0128226280212402,
+      "kl": 0.51177978515625,
+      "learning_rate": 2.5358974294659373e-07,
+      "loss": 0.0205,
+      "reward": 0.31238897051662207,
+      "reward_std": 1.0008442774415016,
+      "rewards/cosine_scaled_reward": -0.07301739603281021,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 378
+    },
+    {
+      "completion_length": 1567.0000457763672,
+      "epoch": 0.43314285714285716,
+      "grad_norm": 2.777541160583496,
+      "kl": 0.682373046875,
+      "learning_rate": 2.512332043064913e-07,
+      "loss": 0.0273,
+      "reward": 0.006588555872440338,
+      "reward_std": 0.7485722936689854,
+      "rewards/cosine_scaled_reward": -0.18877192959189415,
+      "rewards/format_reward": 0.6666666846722364,
+      "step": 379
+    },
+    {
+      "completion_length": 1324.5208854675293,
+      "epoch": 0.4342857142857143,
+      "grad_norm": 3.0375523567199707,
+      "kl": 0.670166015625,
+      "learning_rate": 2.488912271385139e-07,
+      "loss": 0.0268,
+      "reward": 0.2578076588688418,
+      "reward_std": 0.9591740146279335,
+      "rewards/cosine_scaled_reward": -0.012080159038305283,
+      "rewards/format_reward": 0.5416666846722364,
+      "step": 380
+    },
+    {
+      "completion_length": 1730.3125762939453,
+      "epoch": 0.43542857142857144,
+      "grad_norm": 1.9336341619491577,
+      "kl": 0.7171630859375,
+      "learning_rate": 2.465639255873246e-07,
+      "loss": 0.0287,
+      "reward": -0.1746607469394803,
+      "reward_std": 0.5531054511666298,
+      "rewards/cosine_scaled_reward": -0.2578210327774286,
+      "rewards/format_reward": 0.6458333469927311,
+      "step": 381
+    },
+    {
+      "completion_length": 1355.208366394043,
+      "epoch": 0.43657142857142855,
+      "grad_norm": 2.0603504180908203,
+      "kl": 0.6038818359375,
+      "learning_rate": 2.4425141308231765e-07,
+      "loss": 0.0242,
+      "reward": 0.16119742300361395,
+      "reward_std": 0.8973702676594257,
+      "rewards/cosine_scaled_reward": -0.13088632840663195,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 382
+    },
+    {
+      "completion_length": 1379.7292098999023,
+      "epoch": 0.4377142857142857,
+      "grad_norm": 1.975783348083496,
+      "kl": 0.513427734375,
+      "learning_rate": 2.4195380233209006e-07,
+      "loss": 0.0206,
+      "reward": 0.5386483520269394,
+      "reward_std": 0.9303784146904945,
+      "rewards/cosine_scaled_reward": 0.09457700047641993,
+      "rewards/format_reward": 0.7500000111758709,
+      "step": 383
+    },
+    {
+      "completion_length": 1441.4375305175781,
+      "epoch": 0.43885714285714283,
+      "grad_norm": 2.0518715381622314,
+      "kl": 0.517333984375,
+      "learning_rate": 2.3967120531894857e-07,
+      "loss": 0.0207,
+      "reward": 0.5782037973403931,
+      "reward_std": 1.07766717299819,
+      "rewards/cosine_scaled_reward": 0.11610157322138548,
+      "rewards/format_reward": 0.6875000260770321,
+      "step": 384
+    },
+    {
+      "completion_length": 1418.8958740234375,
+      "epoch": 0.44,
+      "grad_norm": 2.584573745727539,
+      "kl": 0.51806640625,
+      "learning_rate": 2.374037332934512e-07,
+      "loss": 0.0207,
+      "reward": 0.05193836707621813,
+      "reward_std": 0.8466444984078407,
+      "rewards/cosine_scaled_reward": -0.1831620568409562,
+      "rewards/format_reward": 0.6875000149011612,
+      "step": 385
+    },
+    {
+      "completion_length": 1452.9166946411133,
+      "epoch": 0.44114285714285717,
+      "grad_norm": 2.1661787033081055,
+      "kl": 0.517578125,
+      "learning_rate": 2.3515149676898552e-07,
+      "loss": 0.0207,
+      "reward": 0.14932426065206528,
+      "reward_std": 0.8432199582457542,
+      "rewards/cosine_scaled_reward": -0.10539527935907245,
+      "rewards/format_reward": 0.6458333544433117,
+      "step": 386
+    },
+    {
+      "completion_length": 1647.145866394043,
+      "epoch": 0.4422857142857143,
+      "grad_norm": 1.8875645399093628,
+      "kl": 0.56982421875,
+      "learning_rate": 2.3291460551638237e-07,
+      "loss": 0.0228,
+      "reward": 0.01697136198345106,
+      "reward_std": 0.7178105190396309,
+      "rewards/cosine_scaled_reward": -0.11740265972912312,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 387
+    },
+    {
+      "completion_length": 1262.020866394043,
+      "epoch": 0.44342857142857145,
+      "grad_norm": 4.029469966888428,
+      "kl": 0.7529296875,
+      "learning_rate": 2.306931685585657e-07,
+      "loss": 0.0301,
+      "reward": 0.3093376103788614,
+      "reward_std": 0.9565748050808907,
+      "rewards/cosine_scaled_reward": -0.05382521077990532,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 388
+    },
+    {
+      "completion_length": 1679.1667404174805,
+      "epoch": 0.44457142857142856,
+      "grad_norm": 1.9148039817810059,
+      "kl": 0.68115234375,
+      "learning_rate": 2.2848729416523859e-07,
+      "loss": 0.0272,
+      "reward": 0.03406955860555172,
+      "reward_std": 0.8620793931186199,
+      "rewards/cosine_scaled_reward": -0.1516010407358408,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 389
+    },
+    {
+      "completion_length": 1607.0625305175781,
+      "epoch": 0.44571428571428573,
+      "grad_norm": 2.3997583389282227,
+      "kl": 0.6044921875,
+      "learning_rate": 2.2629708984760706e-07,
+      "loss": 0.0242,
+      "reward": -0.03216620907187462,
+      "reward_std": 0.7698984518647194,
+      "rewards/cosine_scaled_reward": -0.18137890007346869,
+      "rewards/format_reward": 0.6041666865348816,
+      "step": 390
+    },
+    {
+      "completion_length": 1295.2916870117188,
+      "epoch": 0.44685714285714284,
+      "grad_norm": 2.192011833190918,
+      "kl": 0.499267578125,
+      "learning_rate": 2.2412266235313973e-07,
+      "loss": 0.02,
+      "reward": 0.3369876742362976,
+      "reward_std": 0.9241867884993553,
+      "rewards/cosine_scaled_reward": -0.05256127379834652,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 391
+    },
+    {
+      "completion_length": 1388.0625305175781,
+      "epoch": 0.448,
+      "grad_norm": 1.6250756978988647,
+      "kl": 0.4912109375,
+      "learning_rate": 2.2196411766036487e-07,
+      "loss": 0.0196,
+      "reward": 0.3453047815710306,
+      "reward_std": 0.924341045320034,
+      "rewards/cosine_scaled_reward": -0.037261209450662136,
+      "rewards/format_reward": 0.7708333432674408,
+      "step": 392
+    },
+    {
+      "completion_length": 1362.3333587646484,
+      "epoch": 0.4491428571428571,
+      "grad_norm": 2.632045269012451,
+      "kl": 0.560302734375,
+      "learning_rate": 2.1982156097370557e-07,
+      "loss": 0.0224,
+      "reward": 0.3152189594693482,
+      "reward_std": 0.8564588874578476,
+      "rewards/cosine_scaled_reward": -0.05929289385676384,
+      "rewards/format_reward": 0.7916666865348816,
+      "step": 393
+    },
+    {
+      "completion_length": 1506.1666870117188,
+      "epoch": 0.4502857142857143,
+      "grad_norm": 2.0749998092651367,
+      "kl": 0.65478515625,
+      "learning_rate": 2.1769509671835223e-07,
+      "loss": 0.0262,
+      "reward": -0.0516182022402063,
+      "reward_std": 0.6332247443497181,
+      "rewards/cosine_scaled_reward": -0.17348778434097767,
+      "rewards/format_reward": 0.6041666865348816,
+      "step": 394
+    },
+    {
+      "completion_length": 1105.4583587646484,
+      "epoch": 0.4514285714285714,
+      "grad_norm": 3.7922682762145996,
+      "kl": 0.60986328125,
+      "learning_rate": 2.1558482853517253e-07,
+      "loss": 0.0244,
+      "reward": 0.3151353672146797,
+      "reward_std": 0.7997763678431511,
+      "rewards/cosine_scaled_reward": 0.014608239755034447,
+      "rewards/format_reward": 0.6666666828095913,
+      "step": 395
+    },
+    {
+      "completion_length": 1559.6875762939453,
+      "epoch": 0.45257142857142857,
+      "grad_norm": 3.7929108142852783,
+      "kl": 0.51507568359375,
+      "learning_rate": 2.134908592756607e-07,
+      "loss": 0.0206,
+      "reward": 0.1322829071432352,
+      "reward_std": 0.8961255550384521,
+      "rewards/cosine_scaled_reward": -0.12251237966120243,
+      "rewards/format_reward": 0.6458333507180214,
+      "step": 396
+    },
+    {
+      "completion_length": 1392.083381652832,
+      "epoch": 0.45371428571428574,
+      "grad_norm": 1.7970480918884277,
+      "kl": 0.5240478515625,
+      "learning_rate": 2.1141329099692406e-07,
+      "loss": 0.0209,
+      "reward": 0.24729618662968278,
+      "reward_std": 0.7044042460620403,
+      "rewards/cosine_scaled_reward": -0.03716139169409871,
+      "rewards/format_reward": 0.7291666902601719,
+      "step": 397
+    },
+    {
+      "completion_length": 1375.2500228881836,
+      "epoch": 0.45485714285714285,
+      "grad_norm": 2.7973999977111816,
+      "kl": 0.5516357421875,
+      "learning_rate": 2.0935222495670968e-07,
+      "loss": 0.0221,
+      "reward": 0.05064563453197479,
+      "reward_std": 0.7481582798063755,
+      "rewards/cosine_scaled_reward": -0.16217901837080717,
+      "rewards/format_reward": 0.6666666828095913,
+      "step": 398
+    },
+    {
+      "completion_length": 1417.9375381469727,
+      "epoch": 0.456,
+      "grad_norm": 3.365849733352661,
+      "kl": 0.652587890625,
+      "learning_rate": 2.0730776160846853e-07,
+      "loss": 0.0261,
+      "reward": 0.24118162877857685,
+      "reward_std": 0.9246044531464577,
+      "rewards/cosine_scaled_reward": -0.05636600544676185,
+      "rewards/format_reward": 0.6666666865348816,
+      "step": 399
+    },
+    {
+      "completion_length": 835.3958549499512,
+      "epoch": 0.45714285714285713,
+      "grad_norm": 2.1656346321105957,
+      "kl": 0.446533203125,
+      "learning_rate": 2.0528000059645995e-07,
+      "loss": 0.0178,
+      "reward": 0.66196015663445,
+      "reward_std": 1.0285310856997967,
+      "rewards/cosine_scaled_reward": 0.17172618986387533,
+      "rewards/format_reward": 0.8541666939854622,
+      "step": 400
+    },
+    {
+      "completion_length": 1314.083396911621,
+      "epoch": 0.4582857142857143,
+      "grad_norm": 1.6506296396255493,
+      "kl": 0.44915771484375,
+      "learning_rate": 2.032690407508949e-07,
+      "loss": 0.018,
+      "reward": 0.157606887165457,
+      "reward_std": 0.7822442874312401,
+      "rewards/cosine_scaled_reward": -0.11768818949349225,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 401
+    },
+    {
+      "completion_length": 1331.9583892822266,
+      "epoch": 0.4594285714285714,
+      "grad_norm": 3.1099698543548584,
+      "kl": 0.589599609375,
+      "learning_rate": 2.0127498008311922e-07,
+      "loss": 0.0236,
+      "reward": 0.2424939200282097,
+      "reward_std": 0.7927302867174149,
+      "rewards/cosine_scaled_reward": -0.0522354356944561,
+      "rewards/format_reward": 0.6875000260770321,
+      "step": 402
+    },
+    {
+      "completion_length": 1337.020881652832,
+      "epoch": 0.4605714285714286,
+      "grad_norm": 2.6694633960723877,
+      "kl": 0.537109375,
+      "learning_rate": 1.9929791578083655e-07,
+      "loss": 0.0215,
+      "reward": 0.2716318762395531,
+      "reward_std": 0.8227218054234982,
+      "rewards/cosine_scaled_reward": -0.02993021416477859,
+      "rewards/format_reward": 0.6666666902601719,
+      "step": 403
+    },
+    {
+      "completion_length": 1489.7708740234375,
+      "epoch": 0.4617142857142857,
+      "grad_norm": 2.208977460861206,
+      "kl": 0.66632080078125,
+      "learning_rate": 1.9733794420337213e-07,
+      "loss": 0.0267,
+      "reward": 0.28225214779376984,
+      "reward_std": 0.8597016483545303,
+      "rewards/cosine_scaled_reward": -0.004551528720185161,
+      "rewards/format_reward": 0.7083333395421505,
+      "step": 404
+    },
+    {
+      "completion_length": 1259.8333587646484,
+      "epoch": 0.46285714285714286,
+      "grad_norm": 4.63955545425415,
+      "kl": 0.81591796875,
+      "learning_rate": 1.9539516087697517e-07,
+      "loss": 0.0327,
+      "reward": 0.35530247224960476,
+      "reward_std": 0.8299554735422134,
+      "rewards/cosine_scaled_reward": 0.037398045882582664,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 405
+    },
+    {
+      "completion_length": 1513.1250610351562,
+      "epoch": 0.464,
+      "grad_norm": 2.6123645305633545,
+      "kl": 0.635986328125,
+      "learning_rate": 1.934696604901642e-07,
+      "loss": 0.0254,
+      "reward": 0.06231710687279701,
+      "reward_std": 0.8518500626087189,
+      "rewards/cosine_scaled_reward": -0.1575743369758129,
+      "rewards/format_reward": 0.6458333507180214,
+      "step": 406
+    },
+    {
+      "completion_length": 1208.5000228881836,
+      "epoch": 0.46514285714285714,
+      "grad_norm": 3.125258445739746,
+      "kl": 0.557861328125,
+      "learning_rate": 1.915615368891117e-07,
+      "loss": 0.0224,
+      "reward": 0.28017069818452,
+      "reward_std": 0.7779465243220329,
+      "rewards/cosine_scaled_reward": -0.040886467322707176,
+      "rewards/format_reward": 0.7291666939854622,
+      "step": 407
+    },
+    {
+      "completion_length": 1217.5208587646484,
+      "epoch": 0.4662857142857143,
+      "grad_norm": 2.7784037590026855,
+      "kl": 0.427734375,
+      "learning_rate": 1.8967088307307e-07,
+      "loss": 0.0171,
+      "reward": 0.3413910511881113,
+      "reward_std": 0.6852421574294567,
+      "rewards/cosine_scaled_reward": -0.017510855570435524,
+      "rewards/format_reward": 0.8958333432674408,
+      "step": 408
+    },
+    {
+      "completion_length": 1725.041748046875,
+      "epoch": 0.4674285714285714,
+      "grad_norm": 1.8275662660598755,
+      "kl": 0.676513671875,
+      "learning_rate": 1.8779779118983867e-07,
+      "loss": 0.0271,
+      "reward": 0.21645420044660568,
+      "reward_std": 0.8697847276926041,
+      "rewards/cosine_scaled_reward": -0.08236874872818589,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 409
+    },
+    {
+      "completion_length": 1496.8333892822266,
+      "epoch": 0.4685714285714286,
+      "grad_norm": 3.2171754837036133,
+      "kl": 0.934356689453125,
+      "learning_rate": 1.8594235253127372e-07,
+      "loss": 0.0374,
+      "reward": 0.17047240026295185,
+      "reward_std": 0.7880502119660378,
+      "rewards/cosine_scaled_reward": -0.04659413266927004,
+      "rewards/format_reward": 0.583333345130086,
+      "step": 410
+    },
+    {
+      "completion_length": 1500.4792098999023,
+      "epoch": 0.4697142857142857,
+      "grad_norm": 1.7640888690948486,
+      "kl": 0.608154296875,
+      "learning_rate": 1.8410465752883758e-07,
+      "loss": 0.0243,
+      "reward": 0.3746693143621087,
+      "reward_std": 1.0237346589565277,
+      "rewards/cosine_scaled_reward": -0.025670517061371356,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 411
+    },
+    {
+      "completion_length": 1243.1042098999023,
+      "epoch": 0.47085714285714286,
+      "grad_norm": 3.035127878189087,
+      "kl": 0.6143798828125,
+      "learning_rate": 1.822847957491922e-07,
+      "loss": 0.0246,
+      "reward": 0.37103684339672327,
+      "reward_std": 0.9396983981132507,
+      "rewards/cosine_scaled_reward": -0.03727317973971367,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 412
+    },
+    {
+      "completion_length": 1486.4166946411133,
+      "epoch": 0.472,
+      "grad_norm": 2.925623655319214,
+      "kl": 0.7296142578125,
+      "learning_rate": 1.804828558898332e-07,
+      "loss": 0.0292,
+      "reward": 0.19466528482735157,
+      "reward_std": 0.7697472274303436,
+      "rewards/cosine_scaled_reward": -0.08936256961897016,
+      "rewards/format_reward": 0.6875000149011612,
+      "step": 413
+    },
+    {
+      "completion_length": 1481.3125381469727,
+      "epoch": 0.47314285714285714,
+      "grad_norm": 2.3652820587158203,
+      "kl": 0.717041015625,
+      "learning_rate": 1.7869892577476722e-07,
+      "loss": 0.0286,
+      "reward": 0.06787084229290485,
+      "reward_std": 0.8079018853604794,
+      "rewards/cosine_scaled_reward": -0.15581453032791615,
+      "rewards/format_reward": 0.6666666865348816,
+      "step": 414
+    },
+    {
+      "completion_length": 1480.1875610351562,
+      "epoch": 0.4742857142857143,
+      "grad_norm": 1.5522794723510742,
+      "kl": 0.653564453125,
+      "learning_rate": 1.7693309235023127e-07,
+      "loss": 0.0261,
+      "reward": 0.2685818700119853,
+      "reward_std": 0.9294859580695629,
+      "rewards/cosine_scaled_reward": -0.06137389224022627,
+      "rewards/format_reward": 0.6875000186264515,
+      "step": 415
+    },
+    {
+      "completion_length": 1191.3125534057617,
+      "epoch": 0.4754285714285714,
+      "grad_norm": 2.506166696548462,
+      "kl": 0.4866943359375,
+      "learning_rate": 1.7518544168045524e-07,
+      "loss": 0.0195,
+      "reward": 0.540099716745317,
+      "reward_std": 0.9752370566129684,
+      "rewards/cosine_scaled_reward": 0.078727146377787,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 416
+    },
+    {
+      "completion_length": 1577.3750686645508,
+      "epoch": 0.4765714285714286,
+      "grad_norm": 2.541682004928589,
+      "kl": 0.90380859375,
+      "learning_rate": 1.7345605894346726e-07,
+      "loss": 0.0361,
+      "reward": 0.19305734895169735,
+      "reward_std": 0.8487963303923607,
+      "rewards/cosine_scaled_reward": -0.08683300111442804,
+      "rewards/format_reward": 0.6666666902601719,
+      "step": 417
+    },
+    {
+      "completion_length": 1273.8958473205566,
+      "epoch": 0.4777142857142857,
+      "grad_norm": 2.7441353797912598,
+      "kl": 0.631591796875,
+      "learning_rate": 1.7174502842694212e-07,
+      "loss": 0.0253,
+      "reward": 0.5500797647982836,
+      "reward_std": 0.9426254630088806,
+      "rewards/cosine_scaled_reward": 0.08637175487820059,
+      "rewards/format_reward": 0.8333333507180214,
+      "step": 418
+    },
+    {
+      "completion_length": 1354.8542098999023,
+      "epoch": 0.47885714285714287,
+      "grad_norm": 2.3867642879486084,
+      "kl": 0.687744140625,
+      "learning_rate": 1.7005243352409333e-07,
+      "loss": 0.0275,
+      "reward": 0.3600006675114855,
+      "reward_std": 0.9771365597844124,
+      "rewards/cosine_scaled_reward": -0.004668326582759619,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 419
+    },
+    {
+      "completion_length": 1028.6250381469727,
+      "epoch": 0.48,
+      "grad_norm": 10.020596504211426,
+      "kl": 0.8775634765625,
+      "learning_rate": 1.6837835672960831e-07,
+      "loss": 0.0351,
+      "reward": 0.07369395159184933,
+      "reward_std": 0.7157322019338608,
+      "rewards/cosine_scaled_reward": -0.15065189078450203,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 420
+    },
+    {
+      "completion_length": 1321.0833587646484,
+      "epoch": 0.48114285714285715,
+      "grad_norm": 3.3951587677001953,
+      "kl": 0.703125,
+      "learning_rate": 1.6672287963562852e-07,
+      "loss": 0.0281,
+      "reward": -0.07291397266089916,
+      "reward_std": 0.6696721352636814,
+      "rewards/cosine_scaled_reward": -0.20483622467145324,
+      "rewards/format_reward": 0.6458333469927311,
+      "step": 421
+    },
+    {
+      "completion_length": 1193.3541946411133,
+      "epoch": 0.48228571428571426,
+      "grad_norm": 2.8352770805358887,
+      "kl": 0.587646484375,
+      "learning_rate": 1.6508608292777203e-07,
+      "loss": 0.0235,
+      "reward": 0.2525853253901005,
+      "reward_std": 0.8423508293926716,
+      "rewards/cosine_scaled_reward": -0.07235960848629475,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 422
+    },
+    {
+      "completion_length": 1172.8333740234375,
+      "epoch": 0.48342857142857143,
+      "grad_norm": 2.012563467025757,
+      "kl": 0.49560546875,
+      "learning_rate": 1.6346804638120098e-07,
+      "loss": 0.0198,
+      "reward": 0.09193800436332822,
+      "reward_std": 0.7620077319443226,
+      "rewards/cosine_scaled_reward": -0.18725344724953175,
+      "rewards/format_reward": 0.7916666939854622,
+      "step": 423
+    },
+    {
+      "completion_length": 1772.6875534057617,
+      "epoch": 0.4845714285714286,
+      "grad_norm": 1.4459457397460938,
+      "kl": 0.5943603515625,
+      "learning_rate": 1.6186884885673413e-07,
+      "loss": 0.0238,
+      "reward": 0.2721530678682029,
+      "reward_std": 0.9363379552960396,
+      "rewards/cosine_scaled_reward": -0.06505887769162655,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 424
+    },
+    {
+      "completion_length": 1682.208366394043,
+      "epoch": 0.4857142857142857,
+      "grad_norm": 2.2990784645080566,
+      "kl": 0.73602294921875,
+      "learning_rate": 1.6028856829700258e-07,
+      "loss": 0.0294,
+      "reward": 0.5255491202697158,
+      "reward_std": 1.0602496266365051,
+      "rewards/cosine_scaled_reward": 0.10401645209640265,
+      "rewards/format_reward": 0.6666666772216558,
+      "step": 425
+    },
+    {
+      "completion_length": 1195.916690826416,
+      "epoch": 0.4868571428571429,
+      "grad_norm": 2.467822790145874,
+      "kl": 0.5208740234375,
+      "learning_rate": 1.5872728172265146e-07,
+      "loss": 0.0208,
+      "reward": 0.11827224772423506,
+      "reward_std": 0.6336887441575527,
+      "rewards/cosine_scaled_reward": -0.1465412126854062,
+      "rewards/format_reward": 0.7916666939854622,
+      "step": 426
+    },
+    {
+      "completion_length": 1604.0208587646484,
+      "epoch": 0.488,
+      "grad_norm": 2.8577768802642822,
+      "kl": 0.614990234375,
+      "learning_rate": 1.5718506522858572e-07,
+      "loss": 0.0246,
+      "reward": 0.27578693721443415,
+      "reward_std": 1.0030243545770645,
+      "rewards/cosine_scaled_reward": -0.04830464324913919,
+      "rewards/format_reward": 0.6666666865348816,
+      "step": 427
+    },
+    {
+      "completion_length": 1337.7916946411133,
+      "epoch": 0.48914285714285716,
+      "grad_norm": 1.5771373510360718,
+      "kl": 0.516357421875,
+      "learning_rate": 1.5566199398026147e-07,
+      "loss": 0.0206,
+      "reward": 0.31163547467440367,
+      "reward_std": 0.8827511817216873,
+      "rewards/cosine_scaled_reward": -0.05672437600696867,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 428
+    },
+    {
+      "completion_length": 1092.6458854675293,
+      "epoch": 0.49028571428571427,
+      "grad_norm": 7.517470836639404,
+      "kl": 0.735107421875,
+      "learning_rate": 1.5415814221002265e-07,
+      "loss": 0.0294,
+      "reward": -0.04257449973374605,
+      "reward_std": 0.6279182583093643,
+      "rewards/cosine_scaled_reward": -0.2530031790956855,
+      "rewards/format_reward": 0.7916666865348816,
+      "step": 429
+    },
+    {
+      "completion_length": 1677.7500686645508,
+      "epoch": 0.49142857142857144,
+      "grad_norm": 2.1630637645721436,
+      "kl": 0.6884765625,
+      "learning_rate": 1.5267358321348285e-07,
+      "loss": 0.0276,
+      "reward": 0.20026828069239855,
+      "reward_std": 0.8897528201341629,
+      "rewards/cosine_scaled_reward": -0.10384058998897672,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 430
+    },
+    {
+      "completion_length": 1051.2500381469727,
+      "epoch": 0.49257142857142855,
+      "grad_norm": 2.162562847137451,
+      "kl": 0.677490234375,
+      "learning_rate": 1.5120838934595337e-07,
+      "loss": 0.0271,
+      "reward": 0.0934929153881967,
+      "reward_std": 0.6808763556182384,
+      "rewards/cosine_scaled_reward": -0.1685788333415985,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 431
+    },
+    {
+      "completion_length": 1581.7917098999023,
+      "epoch": 0.4937142857142857,
+      "grad_norm": 3.096701145172119,
+      "kl": 0.9864044189453125,
+      "learning_rate": 1.4976263201891613e-07,
+      "loss": 0.0394,
+      "reward": 0.003629262908361852,
+      "reward_std": 0.6885860674083233,
+      "rewards/cosine_scaled_reward": -0.15299149602651596,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 432
+    },
+    {
+      "completion_length": 1580.2708740234375,
+      "epoch": 0.4948571428571429,
+      "grad_norm": 2.7364211082458496,
+      "kl": 0.75,
+      "learning_rate": 1.483363816965435e-07,
+      "loss": 0.03,
+      "reward": 0.15776942297816277,
+      "reward_std": 0.8655783608555794,
+      "rewards/cosine_scaled_reward": -0.08985630236566067,
+      "rewards/format_reward": 0.6041666828095913,
+      "step": 433
+    },
+    {
+      "completion_length": 1532.2291984558105,
+      "epoch": 0.496,
+      "grad_norm": 1.8213260173797607,
+      "kl": 0.5916748046875,
+      "learning_rate": 1.469297078922642e-07,
+      "loss": 0.0237,
+      "reward": -0.09767143800854683,
+      "reward_std": 0.6334349103271961,
+      "rewards/cosine_scaled_reward": -0.22124753845855594,
+      "rewards/format_reward": 0.6458333544433117,
+      "step": 434
+    },
+    {
+      "completion_length": 942.9166793823242,
+      "epoch": 0.49714285714285716,
+      "grad_norm": 2.527085542678833,
+      "kl": 0.7471923828125,
+      "learning_rate": 1.4554267916537495e-07,
+      "loss": 0.0299,
+      "reward": -0.003471766598522663,
+      "reward_std": 0.5359638221561909,
+      "rewards/cosine_scaled_reward": -0.20710261538624763,
+      "rewards/format_reward": 0.8125000223517418,
+      "step": 435
+    },
+    {
+      "completion_length": 1480.1250381469727,
+      "epoch": 0.4982857142857143,
+      "grad_norm": 2.621030569076538,
+      "kl": 0.82220458984375,
+      "learning_rate": 1.4417536311769885e-07,
+      "loss": 0.0329,
+      "reward": 0.3342234673909843,
+      "reward_std": 0.8837791383266449,
+      "rewards/cosine_scaled_reward": 0.03901571640744805,
+      "rewards/format_reward": 0.6250000149011612,
+      "step": 436
+    },
+    {
+      "completion_length": 1359.333381652832,
+      "epoch": 0.49942857142857144,
+      "grad_norm": 2.2266130447387695,
+      "kl": 0.56341552734375,
+      "learning_rate": 1.4282782639029128e-07,
+      "loss": 0.0225,
+      "reward": 0.17527570901438594,
+      "reward_std": 0.7361086085438728,
+      "rewards/cosine_scaled_reward": -0.09591093473136425,
+      "rewards/format_reward": 0.7083333618938923,
+      "step": 437
+    },
+    {
+      "completion_length": 1493.1875457763672,
+      "epoch": 0.5005714285714286,
+      "grad_norm": 1.7907952070236206,
+      "kl": 0.62255859375,
+      "learning_rate": 1.4150013466019114e-07,
+      "loss": 0.0249,
+      "reward": 0.025391742354258895,
+      "reward_std": 0.8453281559050083,
+      "rewards/cosine_scaled_reward": -0.22053607925772667,
+      "rewards/format_reward": 0.7500000260770321,
+      "step": 438
+    },
+    {
+      "completion_length": 1641.4167175292969,
+      "epoch": 0.5017142857142857,
+      "grad_norm": 2.076477527618408,
+      "kl": 0.87103271484375,
+      "learning_rate": 1.4019235263722034e-07,
+      "loss": 0.0349,
+      "reward": -0.12565073231235147,
+      "reward_std": 0.695248618721962,
+      "rewards/cosine_scaled_reward": -0.2208885379950516,
+      "rewards/format_reward": 0.583333345130086,
+      "step": 439
+    },
+    {
+      "completion_length": 1336.645866394043,
+      "epoch": 0.5028571428571429,
+      "grad_norm": 3.641127586364746,
+      "kl": 0.525146484375,
+      "learning_rate": 1.3890454406082956e-07,
+      "loss": 0.021,
+      "reward": 0.20326408464461565,
+      "reward_std": 0.780558280646801,
+      "rewards/cosine_scaled_reward": -0.13858992606401443,
+      "rewards/format_reward": 0.854166679084301,
+      "step": 440
+    },
+    {
+      "completion_length": 1554.1458740234375,
+      "epoch": 0.504,
+      "grad_norm": 2.14461612701416,
+      "kl": 0.649658203125,
+      "learning_rate": 1.3763677169699217e-07,
+      "loss": 0.026,
+      "reward": 0.04901134385727346,
+      "reward_std": 0.6753638684749603,
+      "rewards/cosine_scaled_reward": -0.1266509434208274,
+      "rewards/format_reward": 0.625000013038516,
+      "step": 441
+    },
+    {
+      "completion_length": 1134.083366394043,
+      "epoch": 0.5051428571428571,
+      "grad_norm": 2.213015079498291,
+      "kl": 0.38922119140625,
+      "learning_rate": 1.3638909733514452e-07,
+      "loss": 0.0156,
+      "reward": 0.6072201561182737,
+      "reward_std": 0.8717315904796124,
+      "rewards/cosine_scaled_reward": 0.12222992815077305,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 442
+    },
+    {
+      "completion_length": 1613.5625381469727,
+      "epoch": 0.5062857142857143,
+      "grad_norm": 1.7041293382644653,
+      "kl": 0.726806640625,
+      "learning_rate": 1.351615817851748e-07,
+      "loss": 0.0291,
+      "reward": 0.07468948839232326,
+      "reward_std": 0.6469858847558498,
+      "rewards/cosine_scaled_reward": -0.17274294421076775,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 443
+    },
+    {
+      "completion_length": 1118.9792137145996,
+      "epoch": 0.5074285714285715,
+      "grad_norm": 2.203859567642212,
+      "kl": 0.5146484375,
+      "learning_rate": 1.3395428487445914e-07,
+      "loss": 0.0205,
+      "reward": 0.109013965819031,
+      "reward_std": 0.700001485645771,
+      "rewards/cosine_scaled_reward": -0.17907634377479553,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 444
+    },
+    {
+      "completion_length": 1328.2500381469727,
+      "epoch": 0.5085714285714286,
+      "grad_norm": 2.1055400371551514,
+      "kl": 0.700439453125,
+      "learning_rate": 1.3276726544494571e-07,
+      "loss": 0.0281,
+      "reward": 0.003907807171344757,
+      "reward_std": 0.6304442547261715,
+      "rewards/cosine_scaled_reward": -0.1728717922233045,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 445
+    },
+    {
+      "completion_length": 1408.5417098999023,
+      "epoch": 0.5097142857142857,
+      "grad_norm": 3.701373338699341,
+      "kl": 1.005859375,
+      "learning_rate": 1.316005813502869e-07,
+      "loss": 0.0403,
+      "reward": 0.18983037257567048,
+      "reward_std": 0.7102062851190567,
+      "rewards/cosine_scaled_reward": -0.07104396633803844,
+      "rewards/format_reward": 0.6875000167638063,
+      "step": 446
+    },
+    {
+      "completion_length": 1182.68754196167,
+      "epoch": 0.5108571428571429,
+      "grad_norm": 2.484187126159668,
+      "kl": 0.940673828125,
+      "learning_rate": 1.3045428945301953e-07,
+      "loss": 0.0376,
+      "reward": 0.17063724854961038,
+      "reward_std": 0.6657870672643185,
+      "rewards/cosine_scaled_reward": -0.09112012386322021,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 447
+    },
+    {
+      "completion_length": 1155.3750228881836,
+      "epoch": 0.512,
+      "grad_norm": 9.17601490020752,
+      "kl": 0.546142578125,
+      "learning_rate": 1.2932844562179352e-07,
+      "loss": 0.0218,
+      "reward": 0.30222914123442024,
+      "reward_std": 0.7583190239965916,
+      "rewards/cosine_scaled_reward": -0.05191066488623619,
+      "rewards/format_reward": 0.7916666865348816,
+      "step": 448
+    },
+    {
+      "completion_length": 1077.8541984558105,
+      "epoch": 0.5131428571428571,
+      "grad_norm": 2.589733839035034,
+      "kl": 0.573486328125,
+      "learning_rate": 1.2822310472864885e-07,
+      "loss": 0.0229,
+      "reward": 0.17757831397466362,
+      "reward_std": 0.7745413295924664,
+      "rewards/cosine_scaled_reward": -0.1571638728491962,
+      "rewards/format_reward": 0.8541666865348816,
+      "step": 449
+    },
+    {
+      "completion_length": 1738.6667098999023,
+      "epoch": 0.5142857142857142,
+      "grad_norm": 3.1829066276550293,
+      "kl": 1.14892578125,
+      "learning_rate": 1.2713832064634125e-07,
+      "loss": 0.0459,
+      "reward": 0.03298312705010176,
+      "reward_std": 0.7640566490590572,
+      "rewards/cosine_scaled_reward": -0.14894709549844265,
+      "rewards/format_reward": 0.6458333469927311,
+      "step": 450
+    },
+    {
+      "completion_length": 1184.0416870117188,
+      "epoch": 0.5154285714285715,
+      "grad_norm": 3.6981794834136963,
+      "kl": 0.68731689453125,
+      "learning_rate": 1.260741462457165e-07,
+      "loss": 0.0275,
+      "reward": 0.326223655953072,
+      "reward_std": 0.9320776239037514,
+      "rewards/cosine_scaled_reward": -0.049362530931830406,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 451
+    },
+    {
+      "completion_length": 1356.0000228881836,
+      "epoch": 0.5165714285714286,
+      "grad_norm": 2.8852858543395996,
+      "kl": 0.6221923828125,
+      "learning_rate": 1.2503063339313356e-07,
+      "loss": 0.0249,
+      "reward": 0.23957047518342733,
+      "reward_std": 0.6933315098285675,
+      "rewards/cosine_scaled_reward": -0.05927852354943752,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 452
+    },
+    {
+      "completion_length": 1344.395866394043,
+      "epoch": 0.5177142857142857,
+      "grad_norm": 2.0090019702911377,
+      "kl": 0.6807861328125,
+      "learning_rate": 1.2400783294793668e-07,
+      "loss": 0.0272,
+      "reward": 0.2872171855997294,
+      "reward_std": 0.7287779673933983,
+      "rewards/cosine_scaled_reward": -0.033939655870199203,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 453
+    },
+    {
+      "completion_length": 1270.8125762939453,
+      "epoch": 0.5188571428571429,
+      "grad_norm": 3.2378604412078857,
+      "kl": 0.65252685546875,
+      "learning_rate": 1.2300579475997657e-07,
+      "loss": 0.0261,
+      "reward": 0.11756114871241152,
+      "reward_std": 0.7418475337326527,
+      "rewards/cosine_scaled_reward": -0.15102737117558718,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 454
+    },
+    {
+      "completion_length": 1492.208351135254,
+      "epoch": 0.52,
+      "grad_norm": 4.2774338722229,
+      "kl": 0.79248046875,
+      "learning_rate": 1.220245676671809e-07,
+      "loss": 0.0317,
+      "reward": -0.018868495360948145,
+      "reward_std": 0.6951228454709053,
+      "rewards/cosine_scaled_reward": -0.20306831784546375,
+      "rewards/format_reward": 0.6875000223517418,
+      "step": 455
+    },
+    {
+      "completion_length": 1657.2291870117188,
+      "epoch": 0.5211428571428571,
+      "grad_norm": 2.7355401515960693,
+      "kl": 0.7105712890625,
+      "learning_rate": 1.2106419949317388e-07,
+      "loss": 0.0284,
+      "reward": 0.18439527601003647,
+      "reward_std": 0.7989875040948391,
+      "rewards/cosine_scaled_reward": -0.12901971023529768,
+      "rewards/format_reward": 0.7916666865348816,
+      "step": 456
+    },
+    {
+      "completion_length": 1063.9166946411133,
+      "epoch": 0.5222857142857142,
+      "grad_norm": 2.7030558586120605,
+      "kl": 0.5423583984375,
+      "learning_rate": 1.2012473704494537e-07,
+      "loss": 0.0217,
+      "reward": 0.31176955718547106,
+      "reward_std": 0.8158461526036263,
+      "rewards/cosine_scaled_reward": -0.0352974534034729,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 457
+    },
+    {
+      "completion_length": 1281.9167175292969,
+      "epoch": 0.5234285714285715,
+      "grad_norm": 2.028135299682617,
+      "kl": 0.7296142578125,
+      "learning_rate": 1.1920622611056974e-07,
+      "loss": 0.0292,
+      "reward": 0.2823291067034006,
+      "reward_std": 0.8681261576712132,
+      "rewards/cosine_scaled_reward": -0.06675804499536753,
+      "rewards/format_reward": 0.7708333488553762,
+      "step": 458
+    },
+    {
+      "completion_length": 1336.5208892822266,
+      "epoch": 0.5245714285714286,
+      "grad_norm": 3.601792573928833,
+      "kl": 0.614990234375,
+      "learning_rate": 1.1830871145697412e-07,
+      "loss": 0.0246,
+      "reward": 0.2878631625790149,
+      "reward_std": 0.8964981138706207,
+      "rewards/cosine_scaled_reward": -0.07753422670066357,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 459
+    },
+    {
+      "completion_length": 1851.0833740234375,
+      "epoch": 0.5257142857142857,
+      "grad_norm": 3.7353336811065674,
+      "kl": 0.7938232421875,
+      "learning_rate": 1.1743223682775649e-07,
+      "loss": 0.0317,
+      "reward": 0.22537417088460643,
+      "reward_std": 0.8646272122859955,
+      "rewards/cosine_scaled_reward": -0.08393661957234144,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 460
+    },
+    {
+      "completion_length": 1549.5000534057617,
+      "epoch": 0.5268571428571428,
+      "grad_norm": 3.5757029056549072,
+      "kl": 0.853271484375,
+      "learning_rate": 1.1657684494105386e-07,
+      "loss": 0.0342,
+      "reward": 0.11143558099865913,
+      "reward_std": 0.7968284860253334,
+      "rewards/cosine_scaled_reward": -0.10749776661396027,
+      "rewards/format_reward": 0.6250000186264515,
+      "step": 461
+    },
+    {
+      "completion_length": 1378.3542022705078,
+      "epoch": 0.528,
+      "grad_norm": 3.368990421295166,
+      "kl": 0.8125,
+      "learning_rate": 1.1574257748745986e-07,
+      "loss": 0.0325,
+      "reward": -0.07270677981432527,
+      "reward_std": 0.5974620878696442,
+      "rewards/cosine_scaled_reward": -0.23459232598543167,
+      "rewards/format_reward": 0.708333358168602,
+      "step": 462
+    },
+    {
+      "completion_length": 1423.1250457763672,
+      "epoch": 0.5291428571428571,
+      "grad_norm": 3.6016039848327637,
+      "kl": 0.7799072265625,
+      "learning_rate": 1.1492947512799328e-07,
+      "loss": 0.0312,
+      "reward": 0.2612933642594726,
+      "reward_std": 0.7466553151607513,
+      "rewards/cosine_scaled_reward": -0.047830826602876186,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 463
+    },
+    {
+      "completion_length": 1017.833366394043,
+      "epoch": 0.5302857142857142,
+      "grad_norm": 2.5817713737487793,
+      "kl": 0.462646484375,
+      "learning_rate": 1.1413757749211602e-07,
+      "loss": 0.0185,
+      "reward": 0.4776353507768363,
+      "reward_std": 0.6480004005134106,
+      "rewards/cosine_scaled_reward": 0.10391846485435963,
+      "rewards/format_reward": 0.8958333395421505,
+      "step": 464
+    },
+    {
+      "completion_length": 1335.3542098999023,
+      "epoch": 0.5314285714285715,
+      "grad_norm": 2.435389995574951,
+      "kl": 0.8916015625,
+      "learning_rate": 1.1336692317580158e-07,
+      "loss": 0.0357,
+      "reward": -0.06275751371867955,
+      "reward_std": 0.6158505566418171,
+      "rewards/cosine_scaled_reward": -0.23374686716124415,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 465
+    },
+    {
+      "completion_length": 1214.8333587646484,
+      "epoch": 0.5325714285714286,
+      "grad_norm": 2.2548060417175293,
+      "kl": 0.5836181640625,
+      "learning_rate": 1.1261754973965422e-07,
+      "loss": 0.0233,
+      "reward": 0.37778762076050043,
+      "reward_std": 0.9365783184766769,
+      "rewards/cosine_scaled_reward": -0.0053373780101537704,
+      "rewards/format_reward": 0.7708333488553762,
+      "step": 466
+    },
+    {
+      "completion_length": 1402.68754196167,
+      "epoch": 0.5337142857142857,
+      "grad_norm": 2.9286229610443115,
+      "kl": 0.724578857421875,
+      "learning_rate": 1.1188949370707787e-07,
+      "loss": 0.029,
+      "reward": 0.11761695193126798,
+      "reward_std": 0.7316232472658157,
+      "rewards/cosine_scaled_reward": -0.15633787866681814,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 467
+    },
+    {
+      "completion_length": 1510.9791946411133,
+      "epoch": 0.5348571428571428,
+      "grad_norm": 14.900471687316895,
+      "kl": 2.21337890625,
+      "learning_rate": 1.1118279056249653e-07,
+      "loss": 0.0885,
+      "reward": 0.21817315090447664,
+      "reward_std": 0.9960628747940063,
+      "rewards/cosine_scaled_reward": -0.07175269955769181,
+      "rewards/format_reward": 0.6250000223517418,
+      "step": 468
+    },
+    {
+      "completion_length": 1016.2917098999023,
+      "epoch": 0.536,
+      "grad_norm": 6.350338459014893,
+      "kl": 0.953369140625,
+      "learning_rate": 1.1049747474962444e-07,
+      "loss": 0.0382,
+      "reward": 0.2713460554368794,
+      "reward_std": 0.7287629023194313,
+      "rewards/cosine_scaled_reward": -0.04988390300422907,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 469
+    },
+    {
+      "completion_length": 1466.6042251586914,
+      "epoch": 0.5371428571428571,
+      "grad_norm": 7.370168209075928,
+      "kl": 1.435546875,
+      "learning_rate": 1.0983357966978745e-07,
+      "loss": 0.0574,
+      "reward": 0.01497307000681758,
+      "reward_std": 0.733777578920126,
+      "rewards/cosine_scaled_reward": -0.16290199384093285,
+      "rewards/format_reward": 0.6250000149011612,
+      "step": 470
+    },
+    {
+      "completion_length": 1592.7083740234375,
+      "epoch": 0.5382857142857143,
+      "grad_norm": 7.640194892883301,
+      "kl": 1.49639892578125,
+      "learning_rate": 1.0919113768029517e-07,
+      "loss": 0.0598,
+      "reward": 0.3828916675411165,
+      "reward_std": 0.9707400016486645,
+      "rewards/cosine_scaled_reward": 0.03202553070150316,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 471
+    },
+    {
+      "completion_length": 1145.1041946411133,
+      "epoch": 0.5394285714285715,
+      "grad_norm": 2.8886444568634033,
+      "kl": 0.802978515625,
+      "learning_rate": 1.0857018009286381e-07,
+      "loss": 0.0321,
+      "reward": 0.28387897345237434,
+      "reward_std": 0.7857476621866226,
+      "rewards/cosine_scaled_reward": -0.0928277347702533,
+      "rewards/format_reward": 0.895833358168602,
+      "step": 472
+    },
+    {
+      "completion_length": 1445.7708740234375,
+      "epoch": 0.5405714285714286,
+      "grad_norm": 3.867896318435669,
+      "kl": 1.0830078125,
+      "learning_rate": 1.0797073717209013e-07,
+      "loss": 0.0433,
+      "reward": 0.11159952421439812,
+      "reward_std": 0.7589086703956127,
+      "rewards/cosine_scaled_reward": -0.1572155966423452,
+      "rewards/format_reward": 0.770833358168602,
+      "step": 473
+    },
+    {
+      "completion_length": 1613.3750686645508,
+      "epoch": 0.5417142857142857,
+      "grad_norm": 4.905575275421143,
+      "kl": 1.672607421875,
+      "learning_rate": 1.0739283813397639e-07,
+      "loss": 0.0669,
+      "reward": 0.428771385923028,
+      "reward_std": 1.020697444677353,
+      "rewards/cosine_scaled_reward": 0.044653447810560465,
+      "rewards/format_reward": 0.6458333563059568,
+      "step": 474
+    },
+    {
+      "completion_length": 1337.0417022705078,
+      "epoch": 0.5428571428571428,
+      "grad_norm": 4.179660320281982,
+      "kl": 0.6060791015625,
+      "learning_rate": 1.068365111445064e-07,
+      "loss": 0.0242,
+      "reward": 0.23191617615520954,
+      "reward_std": 0.8730170913040638,
+      "rewards/cosine_scaled_reward": -0.11908163363114,
+      "rewards/format_reward": 0.8125000223517418,
+      "step": 475
+    },
+    {
+      "completion_length": 1265.6667098999023,
+      "epoch": 0.544,
+      "grad_norm": 6.457125186920166,
+      "kl": 1.1802978515625,
+      "learning_rate": 1.063017833182728e-07,
+      "loss": 0.0473,
+      "reward": 0.32982578047085553,
+      "reward_std": 0.9863774701952934,
+      "rewards/cosine_scaled_reward": -0.03677630145102739,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 476
+    },
+    {
+      "completion_length": 998.0000381469727,
+      "epoch": 0.5451428571428572,
+      "grad_norm": 3.380866050720215,
+      "kl": 0.7977294921875,
+      "learning_rate": 1.0578868071715544e-07,
+      "loss": 0.0319,
+      "reward": 0.4656647043302655,
+      "reward_std": 0.9266739711165428,
+      "rewards/cosine_scaled_reward": 0.003877062350511551,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 477
+    },
+    {
+      "completion_length": 1333.4583892822266,
+      "epoch": 0.5462857142857143,
+      "grad_norm": 25.78436851501465,
+      "kl": 2.43603515625,
+      "learning_rate": 1.0529722834905125e-07,
+      "loss": 0.0974,
+      "reward": 0.3743471228517592,
+      "reward_std": 0.8705001547932625,
+      "rewards/cosine_scaled_reward": 0.015704313293099403,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 478
+    },
+    {
+      "completion_length": 1580.6666946411133,
+      "epoch": 0.5474285714285714,
+      "grad_norm": 5.661951541900635,
+      "kl": 1.4453125,
+      "learning_rate": 1.0482745016665526e-07,
+      "loss": 0.0579,
+      "reward": 0.039567636558786035,
+      "reward_std": 0.7153622172772884,
+      "rewards/cosine_scaled_reward": -0.17540637124329805,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 479
+    },
+    {
+      "completion_length": 1532.4792175292969,
+      "epoch": 0.5485714285714286,
+      "grad_norm": 10.698458671569824,
+      "kl": 1.906494140625,
+      "learning_rate": 1.0437936906629334e-07,
+      "loss": 0.0764,
+      "reward": 0.12518761213868856,
+      "reward_std": 0.7352484799921513,
+      "rewards/cosine_scaled_reward": -0.1592358397319913,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 480
+    },
+    {
+      "completion_length": 1801.8959045410156,
+      "epoch": 0.5497142857142857,
+      "grad_norm": 7.208052158355713,
+      "kl": 2.16015625,
+      "learning_rate": 1.0395300688680625e-07,
+      "loss": 0.0865,
+      "reward": -0.068937080912292,
+      "reward_std": 0.678013302385807,
+      "rewards/cosine_scaled_reward": -0.21540158614516258,
+      "rewards/format_reward": 0.6666666865348816,
+      "step": 481
+    },
+    {
+      "completion_length": 1408.2083587646484,
+      "epoch": 0.5508571428571428,
+      "grad_norm": 28.815509796142578,
+      "kl": 2.3358154296875,
+      "learning_rate": 1.0354838440848501e-07,
+      "loss": 0.0934,
+      "reward": 0.2200728515163064,
+      "reward_std": 0.8331051766872406,
+      "rewards/cosine_scaled_reward": -0.07284185755997896,
+      "rewards/format_reward": 0.7083333544433117,
+      "step": 482
+    },
+    {
+      "completion_length": 1403.6250457763672,
+      "epoch": 0.552,
+      "grad_norm": 4.524230480194092,
+      "kl": 0.8890380859375,
+      "learning_rate": 1.0316552135205837e-07,
+      "loss": 0.0355,
+      "reward": 0.26419115875614807,
+      "reward_std": 0.8404405452311039,
+      "rewards/cosine_scaled_reward": -0.07794593391008675,
+      "rewards/format_reward": 0.7500000298023224,
+      "step": 483
+    },
+    {
+      "completion_length": 1195.2916870117188,
+      "epoch": 0.5531428571428572,
+      "grad_norm": 3.5205888748168945,
+      "kl": 1.357666015625,
+      "learning_rate": 1.0280443637773163e-07,
+      "loss": 0.0543,
+      "reward": 0.28255582111887634,
+      "reward_std": 0.9850383549928665,
+      "rewards/cosine_scaled_reward": -0.051978057832457125,
+      "rewards/format_reward": 0.6875000260770321,
+      "step": 484
+    },
+    {
+      "completion_length": 1259.208366394043,
+      "epoch": 0.5542857142857143,
+      "grad_norm": 2.4940898418426514,
+      "kl": 0.69293212890625,
+      "learning_rate": 1.0246514708427701e-07,
+      "loss": 0.0277,
+      "reward": 0.11278902753838338,
+      "reward_std": 0.8309934213757515,
+      "rewards/cosine_scaled_reward": -0.1679881983436644,
+      "rewards/format_reward": 0.7708333432674408,
+      "step": 485
+    },
+    {
+      "completion_length": 972.7291946411133,
+      "epoch": 0.5554285714285714,
+      "grad_norm": 5.1607160568237305,
+      "kl": 1.142333984375,
+      "learning_rate": 1.0214767000817596e-07,
+      "loss": 0.0456,
+      "reward": 0.25730944075621665,
+      "reward_std": 0.7034793458878994,
+      "rewards/cosine_scaled_reward": -0.04670877754688263,
+      "rewards/format_reward": 0.8125000223517418,
+      "step": 486
+    },
+    {
+      "completion_length": 1239.770881652832,
+      "epoch": 0.5565714285714286,
+      "grad_norm": 4.1862897872924805,
+      "kl": 0.94287109375,
+      "learning_rate": 1.0185202062281336e-07,
+      "loss": 0.0377,
+      "reward": 0.5008484733989462,
+      "reward_std": 0.818550631403923,
+      "rewards/cosine_scaled_reward": 0.08279422484338284,
+      "rewards/format_reward": 0.8541666865348816,
+      "step": 487
+    },
+    {
+      "completion_length": 1284.854206085205,
+      "epoch": 0.5577142857142857,
+      "grad_norm": 3.6226584911346436,
+      "kl": 0.8634033203125,
+      "learning_rate": 1.0157821333772304e-07,
+      "loss": 0.0346,
+      "reward": 0.09361092420294881,
+      "reward_std": 0.6568006910383701,
+      "rewards/cosine_scaled_reward": -0.13768697017803788,
+      "rewards/format_reward": 0.7708333488553762,
+      "step": 488
+    },
+    {
+      "completion_length": 1505.7708930969238,
+      "epoch": 0.5588571428571428,
+      "grad_norm": 3.779109477996826,
+      "kl": 1.6417236328125,
+      "learning_rate": 1.013262614978859e-07,
+      "loss": 0.0656,
+      "reward": -0.12699575908482075,
+      "reward_std": 0.6261985003948212,
+      "rewards/cosine_scaled_reward": -0.29026195663027465,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 489
+    },
+    {
+      "completion_length": 1053.770866394043,
+      "epoch": 0.56,
+      "grad_norm": 1.6185153722763062,
+      "kl": 0.48480224609375,
+      "learning_rate": 1.0109617738307911e-07,
+      "loss": 0.0194,
+      "reward": 0.23615788342431188,
+      "reward_std": 0.704241368919611,
+      "rewards/cosine_scaled_reward": -0.062381197698414326,
+      "rewards/format_reward": 0.833333358168602,
+      "step": 490
+    },
+    {
+      "completion_length": 1363.4167022705078,
+      "epoch": 0.5611428571428572,
+      "grad_norm": 7.597695350646973,
+      "kl": 1.367431640625,
+      "learning_rate": 1.0088797220727779e-07,
+      "loss": 0.0547,
+      "reward": 0.5214339741505682,
+      "reward_std": 1.070147231221199,
+      "rewards/cosine_scaled_reward": 0.063262770883739,
+      "rewards/format_reward": 0.7500000149011612,
+      "step": 491
+    },
+    {
+      "completion_length": 1276.8542022705078,
+      "epoch": 0.5622857142857143,
+      "grad_norm": 3.7099130153656006,
+      "kl": 0.9229736328125,
+      "learning_rate": 1.0070165611810855e-07,
+      "loss": 0.037,
+      "reward": 0.25723157986067235,
+      "reward_std": 0.7029771581292152,
+      "rewards/cosine_scaled_reward": -0.017836466431617737,
+      "rewards/format_reward": 0.7291666939854622,
+      "step": 492
+    },
+    {
+      "completion_length": 1266.0625534057617,
+      "epoch": 0.5634285714285714,
+      "grad_norm": 3.5932085514068604,
+      "kl": 1.254150390625,
+      "learning_rate": 1.005372381963547e-07,
+      "loss": 0.0501,
+      "reward": 0.2146676443517208,
+      "reward_std": 0.8730643317103386,
+      "rewards/cosine_scaled_reward": -0.10545299621298909,
+      "rewards/format_reward": 0.7500000223517418,
+      "step": 493
+    },
+    {
+      "completion_length": 1115.4792022705078,
+      "epoch": 0.5645714285714286,
+      "grad_norm": 2.4263534545898438,
+      "kl": 0.810546875,
+      "learning_rate": 1.0039472645551372e-07,
+      "loss": 0.0324,
+      "reward": 0.29131725314073265,
+      "reward_std": 0.9221536330878735,
+      "rewards/cosine_scaled_reward": -0.07916020415723324,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 494
+    },
+    {
+      "completion_length": 1731.3750686645508,
+      "epoch": 0.5657142857142857,
+      "grad_norm": 15.834768295288086,
+      "kl": 2.26806640625,
+      "learning_rate": 1.002741278414069e-07,
+      "loss": 0.0906,
+      "reward": 0.2820148948812857,
+      "reward_std": 0.9001934975385666,
+      "rewards/cosine_scaled_reward": -0.013229399919509888,
+      "rewards/format_reward": 0.6458333507180214,
+      "step": 495
+    },
+    {
+      "completion_length": 1282.9583702087402,
+      "epoch": 0.5668571428571428,
+      "grad_norm": 3.1927053928375244,
+      "kl": 1.07080078125,
+      "learning_rate": 1.0017544823184055e-07,
+      "loss": 0.0428,
+      "reward": 0.2954734539380297,
+      "reward_std": 0.7768198624253273,
+      "rewards/cosine_scaled_reward": -0.05057825893163681,
+      "rewards/format_reward": 0.7708333656191826,
+      "step": 496
+    },
+    {
+      "completion_length": 1330.3541946411133,
+      "epoch": 0.568,
+      "grad_norm": 3.622467279434204,
+      "kl": 1.56591796875,
+      "learning_rate": 1.0009869243631952e-07,
+      "loss": 0.0626,
+      "reward": 0.2870326414704323,
+      "reward_std": 0.9923329427838326,
+      "rewards/cosine_scaled_reward": -0.05224178615026176,
+      "rewards/format_reward": 0.666666692122817,
+      "step": 497
+    },
+    {
+      "completion_length": 1888.2083435058594,
+      "epoch": 0.5691428571428572,
+      "grad_norm": 8.798296928405762,
+      "kl": 2.22607421875,
+      "learning_rate": 1.000438641958131e-07,
+      "loss": 0.089,
+      "reward": 0.09313779044896364,
+      "reward_std": 0.8005350790917873,
+      "rewards/cosine_scaled_reward": -0.1090406347066164,
+      "rewards/format_reward": 0.5833333488553762,
+      "step": 498
+    },
+    {
+      "completion_length": 1403.8958587646484,
+      "epoch": 0.5702857142857143,
+      "grad_norm": 3.579638957977295,
+      "kl": 1.40771484375,
+      "learning_rate": 1.0001096618257236e-07,
+      "loss": 0.0563,
+      "reward": 0.31425497168675065,
+      "reward_std": 0.9142299555242062,
+      "rewards/cosine_scaled_reward": -0.03276859223842621,
+      "rewards/format_reward": 0.7291666902601719,
+      "step": 499
+    },
+    {
+      "completion_length": 1698.9792022705078,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 178.20928955078125,
+      "kl": 7.2745361328125,
+      "learning_rate": 1e-07,
+      "loss": 0.2911,
+      "reward": 0.14990665763616562,
+      "reward_std": 0.7996302992105484,
+      "rewards/cosine_scaled_reward": -0.12167917937040329,
+      "rewards/format_reward": 0.7083333563059568,
+      "step": 500
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "step": 500,
+      "total_flos": 0.0,
+      "train_loss": 0.01018991470336914,
+      "train_runtime": 16191.746,
+      "train_samples_per_second": 1.482,
+      "train_steps_per_second": 0.031
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 6,
+  "trial_name": null,
+  "trial_params": null
+}