diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,8963 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9998655733297486,
+  "eval_steps": 500,
+  "global_step": 3719,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 459.0625,
+      "epoch": 0.0013442667025137787,
+      "grad_norm": 33.51481246948242,
+      "kl": 0.001410222053527832,
+      "learning_rate": 2.688172043010753e-07,
+      "loss": 0.0001,
+      "reward": 0.1325918197631836,
+      "reward_std": 0.02248889727052301,
+      "rewards/reward_func_1": 0.1325918197631836,
+      "step": 5
+    },
+    {
+      "completion_length": 479.13125,
+      "epoch": 0.0026885334050275574,
+      "grad_norm": 41.71512985229492,
+      "kl": 0.005267477035522461,
+      "learning_rate": 5.376344086021506e-07,
+      "loss": 0.0002,
+      "reward": 0.13215713500976561,
+      "reward_std": 0.024013734073378146,
+      "rewards/reward_func_1": 0.13215713500976561,
+      "step": 10
+    },
+    {
+      "completion_length": 454.19375,
+      "epoch": 0.0040328001075413365,
+      "grad_norm": 38.0341911315918,
+      "kl": 0.15977706909179687,
+      "learning_rate": 8.064516129032258e-07,
+      "loss": 0.0064,
+      "reward": 0.1277914047241211,
+      "reward_std": 0.023130150814540684,
+      "rewards/reward_func_1": 0.1277914047241211,
+      "step": 15
+    },
+    {
+      "completion_length": 525.4125,
+      "epoch": 0.005377066810055115,
+      "grad_norm": 22.35483169555664,
+      "kl": 0.31226425170898436,
+      "learning_rate": 1.0752688172043011e-06,
+      "loss": 0.0125,
+      "reward": 0.13453254699707032,
+      "reward_std": 0.02571835172129795,
+      "rewards/reward_func_1": 0.13453254699707032,
+      "step": 20
+    },
+    {
+      "completion_length": 537.725,
+      "epoch": 0.006721333512568894,
+      "grad_norm": 26.161535263061523,
+      "kl": 0.36166534423828123,
+      "learning_rate": 1.3440860215053765e-06,
+      "loss": 0.0145,
+      "reward": 0.14296913146972656,
+      "reward_std": 0.022099756821990012,
+      "rewards/reward_func_1": 0.14296913146972656,
+      "step": 25
+    },
+    {
+      "completion_length": 471.96875,
+      "epoch": 0.008065600215082673,
+      "grad_norm": 108.09612274169922,
+      "kl": 0.27325439453125,
+      "learning_rate": 1.6129032258064516e-06,
+      "loss": 0.0109,
+      "reward": 0.12444114685058594,
+      "reward_std": 0.025380318914540113,
+      "rewards/reward_func_1": 0.12444114685058594,
+      "step": 30
+    },
+    {
+      "completion_length": 566.39375,
+      "epoch": 0.009409866917596451,
+      "grad_norm": 28.055280685424805,
+      "kl": 0.4273193359375,
+      "learning_rate": 1.881720430107527e-06,
+      "loss": 0.0171,
+      "reward": 0.13999091386795043,
+      "reward_std": 0.023209166852757333,
+      "rewards/reward_func_1": 0.13999091386795043,
+      "step": 35
+    },
+    {
+      "completion_length": 558.03125,
+      "epoch": 0.01075413362011023,
+      "grad_norm": 92.63247680664062,
+      "kl": 0.7135009765625,
+      "learning_rate": 2.1505376344086023e-06,
+      "loss": 0.0285,
+      "reward": 0.13420333862304687,
+      "reward_std": 0.021424611564725637,
+      "rewards/reward_func_1": 0.13420333862304687,
+      "step": 40
+    },
+    {
+      "completion_length": 524.88125,
+      "epoch": 0.012098400322624008,
+      "grad_norm": 13.822972297668457,
+      "kl": 13.015373229980469,
+      "learning_rate": 2.4193548387096776e-06,
+      "loss": 0.5195,
+      "reward": 0.137255859375,
+      "reward_std": 0.02337467367760837,
+      "rewards/reward_func_1": 0.137255859375,
+      "step": 45
+    },
+    {
+      "completion_length": 554.8125,
+      "epoch": 0.013442667025137788,
+      "grad_norm": 43.86341857910156,
+      "kl": 0.65223388671875,
+      "learning_rate": 2.688172043010753e-06,
+      "loss": 0.0261,
+      "reward": 0.13823509216308594,
+      "reward_std": 0.02139872215921059,
+      "rewards/reward_func_1": 0.13823509216308594,
+      "step": 50
+    },
+    {
+      "completion_length": 453.91875,
+      "epoch": 0.014786933727651566,
+      "grad_norm": 17.72090721130371,
+      "kl": 0.695703125,
+      "learning_rate": 2.9569892473118283e-06,
+      "loss": 0.0278,
+      "reward": 0.13162574768066407,
+      "reward_std": 0.0243722494575195,
+      "rewards/reward_func_1": 0.13162574768066407,
+      "step": 55
+    },
+    {
+      "completion_length": 477.7375,
+      "epoch": 0.016131200430165346,
+      "grad_norm": 51.72431182861328,
+      "kl": 1.161859130859375,
+      "learning_rate": 3.225806451612903e-06,
+      "loss": 0.0465,
+      "reward": 0.13519821166992188,
+      "reward_std": 0.019137346441857515,
+      "rewards/reward_func_1": 0.13519821166992188,
+      "step": 60
+    },
+    {
+      "completion_length": 491.5375,
+      "epoch": 0.017475467132679123,
+      "grad_norm": 23.050928115844727,
+      "kl": 1.3964111328125,
+      "learning_rate": 3.494623655913979e-06,
+      "loss": 0.0559,
+      "reward": 0.12957611083984374,
+      "reward_std": 0.021138915204210205,
+      "rewards/reward_func_1": 0.12957611083984374,
+      "step": 65
+    },
+    {
+      "completion_length": 510.4125,
+      "epoch": 0.018819733835192903,
+      "grad_norm": 26.82096290588379,
+      "kl": 2.96982421875,
+      "learning_rate": 3.763440860215054e-06,
+      "loss": 0.1189,
+      "reward": 0.13300743103027343,
+      "reward_std": 0.024069122620858252,
+      "rewards/reward_func_1": 0.13300743103027343,
+      "step": 70
+    },
+    {
+      "completion_length": 442.7375,
+      "epoch": 0.020164000537706683,
+      "grad_norm": 26.9343318939209,
+      "kl": 3.32802734375,
+      "learning_rate": 4.032258064516129e-06,
+      "loss": 0.1332,
+      "reward": 0.12296409010887147,
+      "reward_std": 0.025652985728811473,
+      "rewards/reward_func_1": 0.12296409010887147,
+      "step": 75
+    },
+    {
+      "completion_length": 585.75625,
+      "epoch": 0.02150826724022046,
+      "grad_norm": 6.579223155975342,
+      "kl": 69.7091796875,
+      "learning_rate": 4.3010752688172045e-06,
+      "loss": 2.7885,
+      "reward": 0.1406890869140625,
+      "reward_std": 0.021016028558369725,
+      "rewards/reward_func_1": 0.1406890869140625,
+      "step": 80
+    },
+    {
+      "completion_length": 549.41875,
+      "epoch": 0.02285253394273424,
+      "grad_norm": 3.3090600967407227,
+      "kl": 1.4935546875,
+      "learning_rate": 4.56989247311828e-06,
+      "loss": 0.0597,
+      "reward": 0.13293228149414063,
+      "reward_std": 0.0250552476150915,
+      "rewards/reward_func_1": 0.13293228149414063,
+      "step": 85
+    },
+    {
+      "completion_length": 534.4625,
+      "epoch": 0.024196800645248016,
+      "grad_norm": 12.640344619750977,
+      "kl": 1.730859375,
+      "learning_rate": 4.838709677419355e-06,
+      "loss": 0.0692,
+      "reward": 0.12914085388183594,
+      "reward_std": 0.021931628661695866,
+      "rewards/reward_func_1": 0.12914085388183594,
+      "step": 90
+    },
+    {
+      "completion_length": 964.30625,
+      "epoch": 0.025541067347761796,
+      "grad_norm": 3.1176600456237793,
+      "kl": 0.46341552734375,
+      "learning_rate": 5.1075268817204305e-06,
+      "loss": 0.0185,
+      "reward": 0.12530202865600587,
+      "reward_std": 0.030988389148842544,
+      "rewards/reward_func_1": 0.12530202865600587,
+      "step": 95
+    },
+    {
+      "completion_length": 946.646875,
+      "epoch": 0.026885334050275576,
+      "grad_norm": 2.275771379470825,
+      "kl": 1.3025634765625,
+      "learning_rate": 5.376344086021506e-06,
+      "loss": 0.0521,
+      "reward": 0.11467647552490234,
+      "reward_std": 0.028262564330361784,
+      "rewards/reward_func_1": 0.11467647552490234,
+      "step": 100
+    },
+    {
+      "completion_length": 745.26875,
+      "epoch": 0.028229600752789352,
+      "grad_norm": 12.095799446105957,
+      "kl": 2.6501708984375,
+      "learning_rate": 5.645161290322582e-06,
+      "loss": 0.106,
+      "reward": 0.11658521220088006,
+      "reward_std": 0.02962974151596427,
+      "rewards/reward_func_1": 0.11658521220088006,
+      "step": 105
+    },
+    {
+      "completion_length": 602.88125,
+      "epoch": 0.029573867455303132,
+      "grad_norm": 4.594326496124268,
+      "kl": 305.4122314453125,
+      "learning_rate": 5.9139784946236566e-06,
+      "loss": 12.2179,
+      "reward": 0.12178945541381836,
+      "reward_std": 0.022939921566285194,
+      "rewards/reward_func_1": 0.12178945541381836,
+      "step": 110
+    },
+    {
+      "completion_length": 649.784375,
+      "epoch": 0.030918134157816912,
+      "grad_norm": 26.5841121673584,
+      "kl": 1.881787109375,
+      "learning_rate": 6.182795698924732e-06,
+      "loss": 0.0753,
+      "reward": 0.12521166801452638,
+      "reward_std": 0.02316317391814664,
+      "rewards/reward_func_1": 0.12521166801452638,
+      "step": 115
+    },
+    {
+      "completion_length": 720.3375,
+      "epoch": 0.03226240086033069,
+      "grad_norm": 4.994908809661865,
+      "kl": 2.0610107421875,
+      "learning_rate": 6.451612903225806e-06,
+      "loss": 0.0825,
+      "reward": 0.1228231817483902,
+      "reward_std": 0.02567218211479485,
+      "rewards/reward_func_1": 0.1228231817483902,
+      "step": 120
+    },
+    {
+      "completion_length": 675.49375,
+      "epoch": 0.033606667562844465,
+      "grad_norm": 10.518940925598145,
+      "kl": 2.297900390625,
+      "learning_rate": 6.720430107526882e-06,
+      "loss": 0.0921,
+      "reward": 0.10162264108657837,
+      "reward_std": 0.026170244067907335,
+      "rewards/reward_func_1": 0.10162264108657837,
+      "step": 125
+    },
+    {
+      "completion_length": 662.875,
+      "epoch": 0.034950934265358245,
+      "grad_norm": 6.318077564239502,
+      "kl": 2.387255859375,
+      "learning_rate": 6.989247311827958e-06,
+      "loss": 0.0955,
+      "reward": 0.11114879846572875,
+      "reward_std": 0.031876870489213616,
+      "rewards/reward_func_1": 0.11114879846572875,
+      "step": 130
+    },
+    {
+      "completion_length": 707.71875,
+      "epoch": 0.036295200967872025,
+      "grad_norm": 19.05810546875,
+      "kl": 3.068359375,
+      "learning_rate": 7.258064516129033e-06,
+      "loss": 0.1228,
+      "reward": 0.09947696328163147,
+      "reward_std": 0.030518771056085824,
+      "rewards/reward_func_1": 0.09947696328163147,
+      "step": 135
+    },
+    {
+      "completion_length": 807.725,
+      "epoch": 0.037639467670385805,
+      "grad_norm": 5.729363441467285,
+      "kl": 2.4376953125,
+      "learning_rate": 7.526881720430108e-06,
+      "loss": 0.0975,
+      "reward": 0.09508908390998841,
+      "reward_std": 0.035996314510703085,
+      "rewards/reward_func_1": 0.09508908390998841,
+      "step": 140
+    },
+    {
+      "completion_length": 921.71875,
+      "epoch": 0.038983734372899585,
+      "grad_norm": 279.34271240234375,
+      "kl": 6.6451171875,
+      "learning_rate": 7.795698924731183e-06,
+      "loss": 0.2654,
+      "reward": 0.06034855842590332,
+      "reward_std": 0.03402297935681418,
+      "rewards/reward_func_1": 0.06034855842590332,
+      "step": 145
+    },
+    {
+      "completion_length": 876.575,
+      "epoch": 0.040328001075413365,
+      "grad_norm": 198.41583251953125,
+      "kl": 6.54765625,
+      "learning_rate": 8.064516129032258e-06,
+      "loss": 0.2619,
+      "reward": 0.07277845814824105,
+      "reward_std": 0.03974553793668747,
+      "rewards/reward_func_1": 0.07277845814824105,
+      "step": 150
+    },
+    {
+      "completion_length": 780.615625,
+      "epoch": 0.04167226777792714,
+      "grad_norm": 16.293689727783203,
+      "kl": 5.5056640625,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.2202,
+      "reward": 0.0797007441520691,
+      "reward_std": 0.03856456303037703,
+      "rewards/reward_func_1": 0.0797007441520691,
+      "step": 155
+    },
+    {
+      "completion_length": 955.875,
+      "epoch": 0.04301653448044092,
+      "grad_norm": 5.261388778686523,
+      "kl": 4.3888671875,
+      "learning_rate": 8.602150537634409e-06,
+      "loss": 0.1755,
+      "reward": 0.06296098232269287,
+      "reward_std": 0.032317174156196414,
+      "rewards/reward_func_1": 0.06296098232269287,
+      "step": 160
+    },
+    {
+      "completion_length": 905.43125,
+      "epoch": 0.0443608011829547,
+      "grad_norm": 2.4820497035980225,
+      "kl": 4.9009765625,
+      "learning_rate": 8.870967741935484e-06,
+      "loss": 0.196,
+      "reward": 0.07761964425444604,
+      "reward_std": 0.031909586675465104,
+      "rewards/reward_func_1": 0.07761964425444604,
+      "step": 165
+    },
+    {
+      "completion_length": 755.90625,
+      "epoch": 0.04570506788546848,
+      "grad_norm": 3.4995851516723633,
+      "kl": 4.2828125,
+      "learning_rate": 9.13978494623656e-06,
+      "loss": 0.1714,
+      "reward": 0.0845573864877224,
+      "reward_std": 0.028807568131014705,
+      "rewards/reward_func_1": 0.0845573864877224,
+      "step": 170
+    },
+    {
+      "completion_length": 660.746875,
+      "epoch": 0.04704933458798226,
+      "grad_norm": 19.133451461791992,
+      "kl": 4.21357421875,
+      "learning_rate": 9.408602150537635e-06,
+      "loss": 0.1686,
+      "reward": 0.09643235206604003,
+      "reward_std": 0.02881598025560379,
+      "rewards/reward_func_1": 0.09643235206604003,
+      "step": 175
+    },
+    {
+      "completion_length": 651.1375,
+      "epoch": 0.04839360129049603,
+      "grad_norm": 5.1367645263671875,
+      "kl": 5.13486328125,
+      "learning_rate": 9.67741935483871e-06,
+      "loss": 0.2055,
+      "reward": 0.09348840713500976,
+      "reward_std": 0.03065957601647824,
+      "rewards/reward_func_1": 0.09348840713500976,
+      "step": 180
+    },
+    {
+      "completion_length": 799.3375,
+      "epoch": 0.04973786799300981,
+      "grad_norm": 9.540658950805664,
+      "kl": 3.68408203125,
+      "learning_rate": 9.946236559139786e-06,
+      "loss": 0.1475,
+      "reward": 0.08806414604187011,
+      "reward_std": 0.03414921889780089,
+      "rewards/reward_func_1": 0.08806414604187011,
+      "step": 185
+    },
+    {
+      "completion_length": 752.3375,
+      "epoch": 0.05108213469552359,
+      "grad_norm": 7.665460586547852,
+      "kl": 8.005517578125,
+      "learning_rate": 1.0215053763440861e-05,
+      "loss": 0.3196,
+      "reward": 0.07655536755919456,
+      "reward_std": 0.03511982869822532,
+      "rewards/reward_func_1": 0.07655536755919456,
+      "step": 190
+    },
+    {
+      "completion_length": 771.375,
+      "epoch": 0.05242640139803737,
+      "grad_norm": 10.91799259185791,
+      "kl": 5.8822265625,
+      "learning_rate": 1.0483870967741936e-05,
+      "loss": 0.2353,
+      "reward": 0.07151660919189454,
+      "reward_std": 0.03899585076142102,
+      "rewards/reward_func_1": 0.07151660919189454,
+      "step": 195
+    },
+    {
+      "completion_length": 898.54375,
+      "epoch": 0.05377066810055115,
+      "grad_norm": 8.995491027832031,
+      "kl": 26.2333984375,
+      "learning_rate": 1.0752688172043012e-05,
+      "loss": 1.0536,
+      "reward": 0.05515105128288269,
+      "reward_std": 0.037433248152956365,
+      "rewards/reward_func_1": 0.05515105128288269,
+      "step": 200
+    },
+    {
+      "completion_length": 801.6125,
+      "epoch": 0.05511493480306493,
+      "grad_norm": 2.727968454360962,
+      "kl": 3.705029296875,
+      "learning_rate": 1.1021505376344085e-05,
+      "loss": 0.1481,
+      "reward": 0.07217190265655518,
+      "reward_std": 0.03416005950421095,
+      "rewards/reward_func_1": 0.07217190265655518,
+      "step": 205
+    },
+    {
+      "completion_length": 759.43125,
+      "epoch": 0.056459201505578704,
+      "grad_norm": 2.115070343017578,
+      "kl": 4.862158203125,
+      "learning_rate": 1.1290322580645164e-05,
+      "loss": 0.1946,
+      "reward": 0.09042127132415771,
+      "reward_std": 0.032155740447342394,
+      "rewards/reward_func_1": 0.09042127132415771,
+      "step": 210
+    },
+    {
+      "completion_length": 932.6875,
+      "epoch": 0.057803468208092484,
+      "grad_norm": 2.2677536010742188,
+      "kl": 2.760693359375,
+      "learning_rate": 1.1559139784946238e-05,
+      "loss": 0.1104,
+      "reward": 0.062465869216248394,
+      "reward_std": 0.033533206372521815,
+      "rewards/reward_func_1": 0.062465869216248394,
+      "step": 215
+    },
+    {
+      "completion_length": 995.0375,
+      "epoch": 0.059147734910606264,
+      "grad_norm": 28.45672607421875,
+      "kl": 2.51044921875,
+      "learning_rate": 1.1827956989247313e-05,
+      "loss": 0.1004,
+      "reward": 0.04683060795068741,
+      "reward_std": 0.03918457605177537,
+      "rewards/reward_func_1": 0.04683060795068741,
+      "step": 220
+    },
+    {
+      "completion_length": 1012.8125,
+      "epoch": 0.060492001613120044,
+      "grad_norm": 657.45166015625,
+      "kl": 8.178271484375,
+      "learning_rate": 1.2096774193548388e-05,
+      "loss": 0.328,
+      "reward": 0.029974862933158875,
+      "reward_std": 0.029841514525469393,
+      "rewards/reward_func_1": 0.029974862933158875,
+      "step": 225
+    },
+    {
+      "completion_length": 902.15,
+      "epoch": 0.061836268315633824,
+      "grad_norm": 63.430824279785156,
+      "kl": 6.02529296875,
+      "learning_rate": 1.2365591397849464e-05,
+      "loss": 0.2418,
+      "reward": 0.03732140064239502,
+      "reward_std": 0.030012273252941667,
+      "rewards/reward_func_1": 0.03732140064239502,
+      "step": 230
+    },
+    {
+      "completion_length": 820.70625,
+      "epoch": 0.0631805350181476,
+      "grad_norm": 18.125106811523438,
+      "kl": 2.8375,
+      "learning_rate": 1.2634408602150539e-05,
+      "loss": 0.1134,
+      "reward": 0.0467583104968071,
+      "reward_std": 0.026065533305518328,
+      "rewards/reward_func_1": 0.0467583104968071,
+      "step": 235
+    },
+    {
+      "completion_length": 781.80625,
+      "epoch": 0.06452480172066138,
+      "grad_norm": 16.778675079345703,
+      "kl": 4.069482421875,
+      "learning_rate": 1.2903225806451613e-05,
+      "loss": 0.1629,
+      "reward": 0.039750583469867706,
+      "reward_std": 0.02966789968777448,
+      "rewards/reward_func_1": 0.039750583469867706,
+      "step": 240
+    },
+    {
+      "completion_length": 733.74375,
+      "epoch": 0.06586906842317516,
+      "grad_norm": 17.191570281982422,
+      "kl": 3.77578125,
+      "learning_rate": 1.3172043010752688e-05,
+      "loss": 0.151,
+      "reward": 0.031531840562820435,
+      "reward_std": 0.03298547498416156,
+      "rewards/reward_func_1": 0.031531840562820435,
+      "step": 245
+    },
+    {
+      "completion_length": 724.8125,
+      "epoch": 0.06721333512568893,
+      "grad_norm": 7.622962951660156,
+      "kl": 4.6927734375,
+      "learning_rate": 1.3440860215053763e-05,
+      "loss": 0.1877,
+      "reward": 0.028579163551330566,
+      "reward_std": 0.03135324278846383,
+      "rewards/reward_func_1": 0.028579163551330566,
+      "step": 250
+    },
+    {
+      "completion_length": 833.86875,
+      "epoch": 0.06855760182820271,
+      "grad_norm": 7.43701171875,
+      "kl": 3.25869140625,
+      "learning_rate": 1.3709677419354839e-05,
+      "loss": 0.1303,
+      "reward": 0.022608640044927596,
+      "reward_std": 0.020842469058698042,
+      "rewards/reward_func_1": 0.022608640044927596,
+      "step": 255
+    },
+    {
+      "completion_length": 954.625,
+      "epoch": 0.06990186853071649,
+      "grad_norm": 8.609543800354004,
+      "kl": 1.60927734375,
+      "learning_rate": 1.3978494623655916e-05,
+      "loss": 0.0644,
+      "reward": 0.006280577182769776,
+      "reward_std": 0.01644945718580857,
+      "rewards/reward_func_1": 0.006280577182769776,
+      "step": 260
+    },
+    {
+      "completion_length": 924.771875,
+      "epoch": 0.07124613523323027,
+      "grad_norm": 25.172861099243164,
+      "kl": 3.16962890625,
+      "learning_rate": 1.4247311827956991e-05,
+      "loss": 0.1268,
+      "reward": 0.008034330606460572,
+      "reward_std": 0.014402125729247928,
+      "rewards/reward_func_1": 0.008034330606460572,
+      "step": 265
+    },
+    {
+      "completion_length": 609.03125,
+      "epoch": 0.07259040193574405,
+      "grad_norm": 20.167495727539062,
+      "kl": 6.338671875,
+      "learning_rate": 1.4516129032258066e-05,
+      "loss": 0.2535,
+      "reward": 0.0008509188890457153,
+      "reward_std": 0.015080565505195409,
+      "rewards/reward_func_1": 0.0008509188890457153,
+      "step": 270
+    },
+    {
+      "completion_length": 669.51875,
+      "epoch": 0.07393466863825783,
+      "grad_norm": 4.5325541496276855,
+      "kl": 8.6146484375,
+      "learning_rate": 1.4784946236559142e-05,
+      "loss": 0.3447,
+      "reward": 0.0004961371421813964,
+      "reward_std": 0.018004348664544523,
+      "rewards/reward_func_1": 0.0004961371421813964,
+      "step": 275
+    },
+    {
+      "completion_length": 282.79375,
+      "epoch": 0.07527893534077161,
+      "grad_norm": 7.294330596923828,
+      "kl": 11.61015625,
+      "learning_rate": 1.5053763440860215e-05,
+      "loss": 0.4646,
+      "reward": 0.016247385740280153,
+      "reward_std": 0.020933675090782346,
+      "rewards/reward_func_1": 0.016247385740280153,
+      "step": 280
+    },
+    {
+      "completion_length": 618.675,
+      "epoch": 0.07662320204328539,
+      "grad_norm": 6.872623443603516,
+      "kl": 25.2796875,
+      "learning_rate": 1.5322580645161292e-05,
+      "loss": 1.0111,
+      "reward": 0.0221073180437088,
+      "reward_std": 0.01539291434455663,
+      "rewards/reward_func_1": 0.0221073180437088,
+      "step": 285
+    },
+    {
+      "completion_length": 71.5875,
+      "epoch": 0.07796746874579917,
+      "grad_norm": 4.054460048675537,
+      "kl": 12.71484375,
+      "learning_rate": 1.5591397849462366e-05,
+      "loss": 0.5082,
+      "reward": 0.029245705343782902,
+      "reward_std": 0.01699454879271798,
+      "rewards/reward_func_1": 0.029245705343782902,
+      "step": 290
+    },
+    {
+      "completion_length": 148.65,
+      "epoch": 0.07931173544831295,
+      "grad_norm": 5.3947062492370605,
+      "kl": 10.1796875,
+      "learning_rate": 1.586021505376344e-05,
+      "loss": 0.4074,
+      "reward": 0.03420259654521942,
+      "reward_std": 0.021775428601540626,
+      "rewards/reward_func_1": 0.03420259654521942,
+      "step": 295
+    },
+    {
+      "completion_length": 151.83125,
+      "epoch": 0.08065600215082673,
+      "grad_norm": 4.995931148529053,
+      "kl": 8.5484375,
+      "learning_rate": 1.6129032258064517e-05,
+      "loss": 0.3418,
+      "reward": 0.0415335863828659,
+      "reward_std": 0.015774094988591968,
+      "rewards/reward_func_1": 0.0415335863828659,
+      "step": 300
+    },
+    {
+      "completion_length": 249.5125,
+      "epoch": 0.0820002688533405,
+      "grad_norm": 6.766551971435547,
+      "kl": 7.60859375,
+      "learning_rate": 1.6397849462365594e-05,
+      "loss": 0.3044,
+      "reward": 0.04043524265289307,
+      "reward_std": 0.01937261049170047,
+      "rewards/reward_func_1": 0.04043524265289307,
+      "step": 305
+    },
+    {
+      "completion_length": 132.06875,
+      "epoch": 0.08334453555585428,
+      "grad_norm": 7.255300045013428,
+      "kl": 33.04140625,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 1.3153,
+      "reward": 0.04444933533668518,
+      "reward_std": 0.015411415329435841,
+      "rewards/reward_func_1": 0.04444933533668518,
+      "step": 310
+    },
+    {
+      "completion_length": 427.43125,
+      "epoch": 0.08468880225836806,
+      "grad_norm": 4.3001933097839355,
+      "kl": 7.823046875,
+      "learning_rate": 1.6935483870967744e-05,
+      "loss": 0.3131,
+      "reward": 0.036715186480432746,
+      "reward_std": 0.025042318180203436,
+      "rewards/reward_func_1": 0.036715186480432746,
+      "step": 315
+    },
+    {
+      "completion_length": 617.3875,
+      "epoch": 0.08603306896088184,
+      "grad_norm": 5.300938129425049,
+      "kl": 9.6765625,
+      "learning_rate": 1.7204301075268818e-05,
+      "loss": 0.3872,
+      "reward": 0.02903536558151245,
+      "reward_std": 0.03241579991299659,
+      "rewards/reward_func_1": 0.02903536558151245,
+      "step": 320
+    },
+    {
+      "completion_length": 480.16875,
+      "epoch": 0.08737733566339562,
+      "grad_norm": 3.9806482791900635,
+      "kl": 9.332421875,
+      "learning_rate": 1.7473118279569895e-05,
+      "loss": 0.3733,
+      "reward": 0.03327850103378296,
+      "reward_std": 0.02665868471376598,
+      "rewards/reward_func_1": 0.03327850103378296,
+      "step": 325
+    },
+    {
+      "completion_length": 624.35625,
+      "epoch": 0.0887216023659094,
+      "grad_norm": 1.2687135934829712,
+      "kl": 8.853515625,
+      "learning_rate": 1.774193548387097e-05,
+      "loss": 0.3541,
+      "reward": 0.014437276124954223,
+      "reward_std": 0.028702187002636492,
+      "rewards/reward_func_1": 0.014437276124954223,
+      "step": 330
+    },
+    {
+      "completion_length": 482.596875,
+      "epoch": 0.09006586906842318,
+      "grad_norm": 4.883694648742676,
+      "kl": 10.290625,
+      "learning_rate": 1.8010752688172042e-05,
+      "loss": 0.4117,
+      "reward": -0.00023592226207256318,
+      "reward_std": 0.02089865313610062,
+      "rewards/reward_func_1": -0.00023592226207256318,
+      "step": 335
+    },
+    {
+      "completion_length": 551.590625,
+      "epoch": 0.09141013577093696,
+      "grad_norm": 31.997989654541016,
+      "kl": 27.48359375,
+      "learning_rate": 1.827956989247312e-05,
+      "loss": 1.0984,
+      "reward": 0.017952871322631837,
+      "reward_std": 0.02710586852626875,
+      "rewards/reward_func_1": 0.017952871322631837,
+      "step": 340
+    },
+    {
+      "completion_length": 90.98125,
+      "epoch": 0.09275440247345074,
+      "grad_norm": 3.1550872325897217,
+      "kl": 6.864453125,
+      "learning_rate": 1.8548387096774193e-05,
+      "loss": 0.2747,
+      "reward": 0.0417973518371582,
+      "reward_std": 0.013006633780605625,
+      "rewards/reward_func_1": 0.0417973518371582,
+      "step": 345
+    },
+    {
+      "completion_length": 93.0875,
+      "epoch": 0.09409866917596452,
+      "grad_norm": 7.118286609649658,
+      "kl": 5.258984375,
+      "learning_rate": 1.881720430107527e-05,
+      "loss": 0.2103,
+      "reward": 0.04214920997619629,
+      "reward_std": 0.0113553161296295,
+      "rewards/reward_func_1": 0.04214920997619629,
+      "step": 350
+    },
+    {
+      "completion_length": 23.6375,
+      "epoch": 0.0954429358784783,
+      "grad_norm": 0.8202024698257446,
+      "kl": 5.026171875,
+      "learning_rate": 1.9086021505376347e-05,
+      "loss": 0.2011,
+      "reward": 0.04604549407958984,
+      "reward_std": 0.007001646169919695,
+      "rewards/reward_func_1": 0.04604549407958984,
+      "step": 355
+    },
+    {
+      "completion_length": 420.46875,
+      "epoch": 0.09678720258099206,
+      "grad_norm": 3.111360788345337,
+      "kl": 5.22421875,
+      "learning_rate": 1.935483870967742e-05,
+      "loss": 0.209,
+      "reward": 0.018204644322395325,
+      "reward_std": 0.012417513456603047,
+      "rewards/reward_func_1": 0.018204644322395325,
+      "step": 360
+    },
+    {
+      "completion_length": 1005.70625,
+      "epoch": 0.09813146928350584,
+      "grad_norm": 0.18079593777656555,
+      "kl": 2.351806640625,
+      "learning_rate": 1.9623655913978498e-05,
+      "loss": 0.0941,
+      "reward": 0.032704389095306395,
+      "reward_std": 0.01922294880496338,
+      "rewards/reward_func_1": 0.032704389095306395,
+      "step": 365
+    },
+    {
+      "completion_length": 1019.66875,
+      "epoch": 0.09947573598601962,
+      "grad_norm": 0.2214096635580063,
+      "kl": 0.1852294921875,
+      "learning_rate": 1.989247311827957e-05,
+      "loss": 0.0074,
+      "reward": 0.061572599411010745,
+      "reward_std": 0.01994357380317524,
+      "rewards/reward_func_1": 0.061572599411010745,
+      "step": 370
+    },
+    {
+      "completion_length": 995.678125,
+      "epoch": 0.1008200026885334,
+      "grad_norm": 0.5282752513885498,
+      "kl": 0.2708984375,
+      "learning_rate": 1.9999960353893115e-05,
+      "loss": 0.0108,
+      "reward": 0.055290712416172026,
+      "reward_std": 0.016450203530257567,
+      "rewards/reward_func_1": 0.055290712416172026,
+      "step": 375
+    },
+    {
+      "completion_length": 1009.3375,
+      "epoch": 0.10216426939104718,
+      "grad_norm": 0.770745575428009,
+      "kl": 0.82568359375,
+      "learning_rate": 1.9999718073267252e-05,
+      "loss": 0.0331,
+      "reward": 0.05625443458557129,
+      "reward_std": 0.01596820540144108,
+      "rewards/reward_func_1": 0.05625443458557129,
+      "step": 380
+    },
+    {
+      "completion_length": 1016.053125,
+      "epoch": 0.10350853609356096,
+      "grad_norm": 0.473776251077652,
+      "kl": 0.31748046875,
+      "learning_rate": 1.9999255542960368e-05,
+      "loss": 0.0127,
+      "reward": 0.01015063002705574,
+      "reward_std": 0.010674006148474292,
+      "rewards/reward_func_1": 0.01015063002705574,
+      "step": 385
+    },
+    {
+      "completion_length": 1001.11875,
+      "epoch": 0.10485280279607474,
+      "grad_norm": 0.7848945260047913,
+      "kl": 0.3754150390625,
+      "learning_rate": 1.999857277315996e-05,
+      "loss": 0.015,
+      "reward": 0.0617163360118866,
+      "reward_std": 0.021173181070480496,
+      "rewards/reward_func_1": 0.0617163360118866,
+      "step": 390
+    },
+    {
+      "completion_length": 1021.315625,
+      "epoch": 0.10619706949858852,
+      "grad_norm": 16.4527645111084,
+      "kl": 1.40760498046875,
+      "learning_rate": 1.9997669778904446e-05,
+      "loss": 0.0563,
+      "reward": 0.04422735869884491,
+      "reward_std": 0.022931134537793697,
+      "rewards/reward_func_1": 0.04422735869884491,
+      "step": 395
+    },
+    {
+      "completion_length": 1017.70625,
+      "epoch": 0.1075413362011023,
+      "grad_norm": 0.4751810133457184,
+      "kl": 1744830464.8029785,
+      "learning_rate": 1.9996546580082792e-05,
+      "loss": 69673728.0,
+      "reward": 0.06360023021697998,
+      "reward_std": 0.025275059579871594,
+      "rewards/reward_func_1": 0.06360023021697998,
+      "step": 400
+    },
+    {
+      "completion_length": 1012.39375,
+      "epoch": 0.10888560290361608,
+      "grad_norm": 0.8353201746940613,
+      "kl": 1.180908203125,
+      "learning_rate": 1.9995203201434124e-05,
+      "loss": 0.0472,
+      "reward": 0.054718819260597226,
+      "reward_std": 0.02895347127923742,
+      "rewards/reward_func_1": 0.054718819260597226,
+      "step": 405
+    },
+    {
+      "completion_length": 883.959375,
+      "epoch": 0.11022986960612986,
+      "grad_norm": 5.007371425628662,
+      "kl": 0.9612548828125,
+      "learning_rate": 1.9993639672547146e-05,
+      "loss": 0.0384,
+      "reward": 0.06528196483850479,
+      "reward_std": 0.028689574322197587,
+      "rewards/reward_func_1": 0.06528196483850479,
+      "step": 410
+    },
+    {
+      "completion_length": 446.45625,
+      "epoch": 0.11157413630864363,
+      "grad_norm": 1.8732314109802246,
+      "kl": 2.2693359375,
+      "learning_rate": 1.9991856027859504e-05,
+      "loss": 0.0908,
+      "reward": 0.052714601159095764,
+      "reward_std": 0.030338111356832086,
+      "rewards/reward_func_1": 0.052714601159095764,
+      "step": 415
+    },
+    {
+      "completion_length": 662.909375,
+      "epoch": 0.11291840301115741,
+      "grad_norm": 6.701618194580078,
+      "kl": 1.86259765625,
+      "learning_rate": 1.9989852306657015e-05,
+      "loss": 0.0745,
+      "reward": 0.07098124027252198,
+      "reward_std": 0.029971924761775882,
+      "rewards/reward_func_1": 0.07098124027252198,
+      "step": 420
+    },
+    {
+      "completion_length": 946.375,
+      "epoch": 0.11426266971367119,
+      "grad_norm": 0.3844849169254303,
+      "kl": 0.8337646484375,
+      "learning_rate": 1.998762855307283e-05,
+      "loss": 0.0333,
+      "reward": 0.08022915720939636,
+      "reward_std": 0.018059900577645747,
+      "rewards/reward_func_1": 0.08022915720939636,
+      "step": 425
+    },
+    {
+      "completion_length": 1011.1875,
+      "epoch": 0.11560693641618497,
+      "grad_norm": 0.3679395020008087,
+      "kl": 0.24794921875,
+      "learning_rate": 1.998518481608643e-05,
+      "loss": 0.0099,
+      "reward": 0.08296351432800293,
+      "reward_std": 0.022155718586873263,
+      "rewards/reward_func_1": 0.08296351432800293,
+      "step": 430
+    },
+    {
+      "completion_length": 1024.0,
+      "epoch": 0.11695120311869875,
+      "grad_norm": 0.5217347145080566,
+      "kl": 0.3544921875,
+      "learning_rate": 1.998252114952255e-05,
+      "loss": 0.0142,
+      "reward": 0.047972720861434934,
+      "reward_std": 0.01913239884888753,
+      "rewards/reward_func_1": 0.047972720861434934,
+      "step": 435
+    },
+    {
+      "completion_length": 738.446875,
+      "epoch": 0.11829546982121253,
+      "grad_norm": 7.95990514755249,
+      "kl": 1.81640625,
+      "learning_rate": 1.9979637612050028e-05,
+      "loss": 0.0727,
+      "reward": 0.02692788541316986,
+      "reward_std": 0.02701200459850952,
+      "rewards/reward_func_1": 0.02692788541316986,
+      "step": 440
+    },
+    {
+      "completion_length": 434.340625,
+      "epoch": 0.11963973652372631,
+      "grad_norm": 11.07094955444336,
+      "kl": 5.8614990234375,
+      "learning_rate": 1.9976534267180464e-05,
+      "loss": 0.2344,
+      "reward": 0.03783460408449173,
+      "reward_std": 0.018221309431828557,
+      "rewards/reward_func_1": 0.03783460408449173,
+      "step": 445
+    },
+    {
+      "completion_length": 38.14375,
+      "epoch": 0.12098400322624009,
+      "grad_norm": 4.362886905670166,
+      "kl": 17.2671875,
+      "learning_rate": 1.997321118326687e-05,
+      "loss": 0.6907,
+      "reward": 0.041283273696899415,
+      "reward_std": 0.011085864211781881,
+      "rewards/reward_func_1": 0.041283273696899415,
+      "step": 450
+    },
+    {
+      "completion_length": 71.421875,
+      "epoch": 0.12232826992875387,
+      "grad_norm": 16.95098876953125,
+      "kl": 14.43046875,
+      "learning_rate": 1.996966843350212e-05,
+      "loss": 0.5772,
+      "reward": 0.029994052648544312,
+      "reward_std": 0.014221129479119554,
+      "rewards/reward_func_1": 0.029994052648544312,
+      "step": 455
+    },
+    {
+      "completion_length": 162.315625,
+      "epoch": 0.12367253663126765,
+      "grad_norm": 5.00581693649292,
+      "kl": 13.87890625,
+      "learning_rate": 1.996590609591736e-05,
+      "loss": 0.5553,
+      "reward": 0.03443393409252167,
+      "reward_std": 0.015160623186966404,
+      "rewards/reward_func_1": 0.03443393409252167,
+      "step": 460
+    },
+    {
+      "completion_length": 387.49375,
+      "epoch": 0.12501680333378143,
+      "grad_norm": 34.0945930480957,
+      "kl": 25.624267578125,
+      "learning_rate": 1.99619242533803e-05,
+      "loss": 1.0272,
+      "reward": 0.020095158740878104,
+      "reward_std": 0.014898770145373419,
+      "rewards/reward_func_1": 0.020095158740878104,
+      "step": 465
+    },
+    {
+      "completion_length": 313.875,
+      "epoch": 0.1263610700362952,
+      "grad_norm": 16.63882827758789,
+      "kl": 11.6126220703125,
+      "learning_rate": 1.9957722993593365e-05,
+      "loss": 0.4642,
+      "reward": 0.03102530874311924,
+      "reward_std": 0.015879479701106904,
+      "rewards/reward_func_1": 0.03102530874311924,
+      "step": 470
+    },
+    {
+      "completion_length": 43.409375,
+      "epoch": 0.127705336738809,
+      "grad_norm": 27.554349899291992,
+      "kl": 16.75,
+      "learning_rate": 1.9953302409091773e-05,
+      "loss": 0.6699,
+      "reward": 0.033424198627471924,
+      "reward_std": 0.013823152912664227,
+      "rewards/reward_func_1": 0.033424198627471924,
+      "step": 475
+    },
+    {
+      "completion_length": 38.784375,
+      "epoch": 0.12904960344132277,
+      "grad_norm": 0.890618085861206,
+      "kl": 16.7375,
+      "learning_rate": 1.9948662597241505e-05,
+      "loss": 0.6692,
+      "reward": 0.029546657204627992,
+      "reward_std": 0.014900979267258663,
+      "rewards/reward_func_1": 0.029546657204627992,
+      "step": 480
+    },
+    {
+      "completion_length": 2.1125,
+      "epoch": 0.13039387014383655,
+      "grad_norm": 5.359586238861084,
+      "kl": 18.334375,
+      "learning_rate": 1.9943803660237146e-05,
+      "loss": 0.733,
+      "reward": 0.042370176315307616,
+      "reward_std": 0.011466928146546707,
+      "rewards/reward_func_1": 0.042370176315307616,
+      "step": 485
+    },
+    {
+      "completion_length": 7.484375,
+      "epoch": 0.13173813684635033,
+      "grad_norm": 133.7327423095703,
+      "kl": 17.6125,
+      "learning_rate": 1.9938725705099652e-05,
+      "loss": 0.7044,
+      "reward": 0.042084154486656186,
+      "reward_std": 0.014303012995515018,
+      "rewards/reward_func_1": 0.042084154486656186,
+      "step": 490
+    },
+    {
+      "completion_length": 2.0125,
+      "epoch": 0.13308240354886408,
+      "grad_norm": 5.098598480224609,
+      "kl": 17.9546875,
+      "learning_rate": 1.9933428843673968e-05,
+      "loss": 0.7184,
+      "reward": 0.047114628553390506,
+      "reward_std": 0.01080106117296964,
+      "rewards/reward_func_1": 0.047114628553390506,
+      "step": 495
+    },
+    {
+      "completion_length": 2.065625,
+      "epoch": 0.13442667025137786,
+      "grad_norm": 3.663975715637207,
+      "kl": 17.8578125,
+      "learning_rate": 1.9927913192626597e-05,
+      "loss": 0.714,
+      "reward": 0.04776406288146973,
+      "reward_std": 0.012929379957495258,
+      "rewards/reward_func_1": 0.04776406288146973,
+      "step": 500
+    },
+    {
+      "completion_length": 1.9375,
+      "epoch": 0.13577093695389164,
+      "grad_norm": 30.02657699584961,
+      "kl": 668487.05625,
+      "learning_rate": 1.9922178873442998e-05,
+      "loss": 26829.6063,
+      "reward": 0.054135143756866455,
+      "reward_std": 0.009887221396638779,
+      "rewards/reward_func_1": 0.054135143756866455,
+      "step": 505
+    },
+    {
+      "completion_length": 24.953125,
+      "epoch": 0.13711520365640542,
+      "grad_norm": 6.359250068664551,
+      "kl": 19.0421875,
+      "learning_rate": 1.9916226012424925e-05,
+      "loss": 0.7612,
+      "reward": 0.05320845246315002,
+      "reward_std": 0.010508252962245024,
+      "rewards/reward_func_1": 0.05320845246315002,
+      "step": 510
+    },
+    {
+      "completion_length": 257.546875,
+      "epoch": 0.1384594703589192,
+      "grad_norm": 3.6431725025177,
+      "kl": 14.698828125,
+      "learning_rate": 1.991005474068765e-05,
+      "loss": 0.5884,
+      "reward": 0.03921504020690918,
+      "reward_std": 0.014476554578868673,
+      "rewards/reward_func_1": 0.03921504020690918,
+      "step": 515
+    },
+    {
+      "completion_length": 110.59375,
+      "epoch": 0.13980373706143298,
+      "grad_norm": 4.781929016113281,
+      "kl": 16.16640625,
+      "learning_rate": 1.9903665194157077e-05,
+      "loss": 0.6467,
+      "reward": 0.043272508680820464,
+      "reward_std": 0.014059747860301286,
+      "rewards/reward_func_1": 0.043272508680820464,
+      "step": 520
+    },
+    {
+      "completion_length": 1.365625,
+      "epoch": 0.14114800376394676,
+      "grad_norm": 12.174643516540527,
+      "kl": 18.8140625,
+      "learning_rate": 1.989705751356672e-05,
+      "loss": 0.7527,
+      "reward": 0.040436971187591556,
+      "reward_std": 0.009100792693789116,
+      "rewards/reward_func_1": 0.040436971187591556,
+      "step": 525
+    },
+    {
+      "completion_length": 3.353125,
+      "epoch": 0.14249227046646054,
+      "grad_norm": 1.5478957891464233,
+      "kl": 15.30625,
+      "learning_rate": 1.9890231844454643e-05,
+      "loss": 0.6123,
+      "reward": 0.035584007203578946,
+      "reward_std": 0.013873677587253042,
+      "rewards/reward_func_1": 0.035584007203578946,
+      "step": 530
+    },
+    {
+      "completion_length": 1.078125,
+      "epoch": 0.14383653716897432,
+      "grad_norm": 0.8689735531806946,
+      "kl": 20.396875,
+      "learning_rate": 1.9883188337160225e-05,
+      "loss": 0.8161,
+      "reward": 0.043527424335479736,
+      "reward_std": 0.008268717869577813,
+      "rewards/reward_func_1": 0.043527424335479736,
+      "step": 535
+    },
+    {
+      "completion_length": 1.775,
+      "epoch": 0.1451808038714881,
+      "grad_norm": 2.0533359050750732,
+      "kl": 18.6109375,
+      "learning_rate": 1.9875927146820867e-05,
+      "loss": 0.7448,
+      "reward": 0.0420529842376709,
+      "reward_std": 0.008834328277953319,
+      "rewards/reward_func_1": 0.0420529842376709,
+      "step": 540
+    },
+    {
+      "completion_length": 2.91875,
+      "epoch": 0.14652507057400188,
+      "grad_norm": 7.708526611328125,
+      "kl": 17.74375,
+      "learning_rate": 1.9868448433368567e-05,
+      "loss": 0.7098,
+      "reward": 0.03905548453330994,
+      "reward_std": 0.011393586202757433,
+      "rewards/reward_func_1": 0.03905548453330994,
+      "step": 545
+    },
+    {
+      "completion_length": 26.203125,
+      "epoch": 0.14786933727651566,
+      "grad_norm": 5.5177435874938965,
+      "kl": 13.23671875,
+      "learning_rate": 1.9860752361526384e-05,
+      "loss": 0.5295,
+      "reward": 0.030779826641082763,
+      "reward_std": 0.017280431411927567,
+      "rewards/reward_func_1": 0.030779826641082763,
+      "step": 550
+    },
+    {
+      "completion_length": 16.4625,
+      "epoch": 0.14921360397902944,
+      "grad_norm": 2.438591718673706,
+      "kl": 16.7859375,
+      "learning_rate": 1.985283910080484e-05,
+      "loss": 0.6714,
+      "reward": 0.03871009349822998,
+      "reward_std": 0.012890951918961946,
+      "rewards/reward_func_1": 0.03871009349822998,
+      "step": 555
+    },
+    {
+      "completion_length": 29.490625,
+      "epoch": 0.15055787068154322,
+      "grad_norm": 11.178017616271973,
+      "kl": 16.7796875,
+      "learning_rate": 1.9844708825498163e-05,
+      "loss": 0.6712,
+      "reward": 0.03712189197540283,
+      "reward_std": 0.014122735538694541,
+      "rewards/reward_func_1": 0.03712189197540283,
+      "step": 560
+    },
+    {
+      "completion_length": 8.56875,
+      "epoch": 0.151902137384057,
+      "grad_norm": 1.3991609811782837,
+      "kl": 6695.2,
+      "learning_rate": 1.983636171468046e-05,
+      "loss": 269.0283,
+      "reward": 0.045965385437011716,
+      "reward_std": 0.011782036734803113,
+      "rewards/reward_func_1": 0.045965385437011716,
+      "step": 565
+    },
+    {
+      "completion_length": 48.490625,
+      "epoch": 0.15324640408657078,
+      "grad_norm": 8.38524055480957,
+      "kl": 16.6640625,
+      "learning_rate": 1.9827797952201756e-05,
+      "loss": 0.6669,
+      "reward": 0.04424548149108887,
+      "reward_std": 0.014852115589019377,
+      "rewards/reward_func_1": 0.04424548149108887,
+      "step": 570
+    },
+    {
+      "completion_length": 93.028125,
+      "epoch": 0.15459067078908456,
+      "grad_norm": 3.190880298614502,
+      "kl": 14.90859375,
+      "learning_rate": 1.9819017726683966e-05,
+      "loss": 0.5958,
+      "reward": 0.030410957336425782,
+      "reward_std": 0.021107864176156,
+      "rewards/reward_func_1": 0.030410957336425782,
+      "step": 575
+    },
+    {
+      "completion_length": 18.278125,
+      "epoch": 0.15593493749159834,
+      "grad_norm": 304378225360896.0,
+      "kl": 36779813791349.3,
+      "learning_rate": 1.9810021231516733e-05,
+      "loss": 1472844005376.0,
+      "reward": 0.046595031023025514,
+      "reward_std": 0.01781562084943289,
+      "rewards/reward_func_1": 0.046595031023025514,
+      "step": 580
+    },
+    {
+      "completion_length": 8.640625,
+      "epoch": 0.15727920419411212,
+      "grad_norm": 1.4716880321502686,
+      "kl": 1590480094.4328125,
+      "learning_rate": 1.9800808664853162e-05,
+      "loss": 63543705.6,
+      "reward": 0.047375273704528806,
+      "reward_std": 0.015040177796618082,
+      "rewards/reward_func_1": 0.047375273704528806,
+      "step": 585
+    },
+    {
+      "completion_length": 5.3625,
+      "epoch": 0.1586234708966259,
+      "grad_norm": 0.6876189708709717,
+      "kl": 18.7609375,
+      "learning_rate": 1.979138022960546e-05,
+      "loss": 0.7509,
+      "reward": 0.04904801845550537,
+      "reward_std": 0.013263128971448167,
+      "rewards/reward_func_1": 0.04904801845550537,
+      "step": 590
+    },
+    {
+      "completion_length": 7.621875,
+      "epoch": 0.15996773759913968,
+      "grad_norm": 1.0694186687469482,
+      "kl": 18.6953125,
+      "learning_rate": 1.9781736133440462e-05,
+      "loss": 0.748,
+      "reward": 0.050295126438140866,
+      "reward_std": 0.011883826142002363,
+      "rewards/reward_func_1": 0.050295126438140866,
+      "step": 595
+    },
+    {
+      "completion_length": 11.071875,
+      "epoch": 0.16131200430165346,
+      "grad_norm": 0.40570515394210815,
+      "kl": 197926011378090.44,
+      "learning_rate": 1.9771876588775072e-05,
+      "loss": 7919798059008.0,
+      "reward": 0.048408856987953185,
+      "reward_std": 0.013870497528841952,
+      "rewards/reward_func_1": 0.048408856987953185,
+      "step": 600
+    },
+    {
+      "completion_length": 3.678125,
+      "epoch": 0.1626562710041672,
+      "grad_norm": 29.10760498046875,
+      "kl": 1.1033819087057724e+16,
+      "learning_rate": 1.976180181277157e-05,
+      "loss": 440097890002534.4,
+      "reward": 0.05020642280578613,
+      "reward_std": 0.010890257774008205,
+      "rewards/reward_func_1": 0.05020642280578613,
+      "step": 605
+    },
+    {
+      "completion_length": 3.86875,
+      "epoch": 0.164000537706681,
+      "grad_norm": 5.49629020690918,
+      "kl": 18.6328125,
+      "learning_rate": 1.975151202733283e-05,
+      "loss": 0.7452,
+      "reward": 0.04792967140674591,
+      "reward_std": 0.013347143970895559,
+      "rewards/reward_func_1": 0.04792967140674591,
+      "step": 610
+    },
+    {
+      "completion_length": 3.88125,
+      "epoch": 0.16534480440919477,
+      "grad_norm": 0.6328467726707458,
+      "kl": 18.6640625,
+      "learning_rate": 1.974100745909744e-05,
+      "loss": 0.7466,
+      "reward": 0.048745088279247284,
+      "reward_std": 0.01318539776839316,
+      "rewards/reward_func_1": 0.048745088279247284,
+      "step": 615
+    },
+    {
+      "completion_length": 6.23125,
+      "epoch": 0.16668907111170855,
+      "grad_norm": 10.044380187988281,
+      "kl": 19.2171875,
+      "learning_rate": 1.9730288339434698e-05,
+      "loss": 0.7687,
+      "reward": 0.05019671618938446,
+      "reward_std": 0.011686628483585083,
+      "rewards/reward_func_1": 0.05019671618938446,
+      "step": 620
+    },
+    {
+      "completion_length": 12.4625,
+      "epoch": 0.16803333781422233,
+      "grad_norm": 0.5477828979492188,
+      "kl": 18.334375,
+      "learning_rate": 1.9719354904439535e-05,
+      "loss": 0.733,
+      "reward": 0.04945822358131409,
+      "reward_std": 0.01383852595463395,
+      "rewards/reward_func_1": 0.04945822358131409,
+      "step": 625
+    },
+    {
+      "completion_length": 9.596875,
+      "epoch": 0.1693776045167361,
+      "grad_norm": 0.485173761844635,
+      "kl": 19.209375,
+      "learning_rate": 1.9708207394927294e-05,
+      "loss": 0.7682,
+      "reward": 0.05124917030334473,
+      "reward_std": 0.010859370271646185,
+      "rewards/reward_func_1": 0.05124917030334473,
+      "step": 630
+    },
+    {
+      "completion_length": 17.403125,
+      "epoch": 0.1707218712192499,
+      "grad_norm": 1.2319393157958984,
+      "kl": 18.13125,
+      "learning_rate": 1.969684605642844e-05,
+      "loss": 0.7251,
+      "reward": 0.046324634552001955,
+      "reward_std": 0.01381837234366685,
+      "rewards/reward_func_1": 0.046324634552001955,
+      "step": 635
+    },
+    {
+      "completion_length": 10.29375,
+      "epoch": 0.17206613792176367,
+      "grad_norm": 3.877319574356079,
+      "kl": 3407890.8765625,
+      "learning_rate": 1.9685271139183143e-05,
+      "loss": 136448.95,
+      "reward": 0.051538944244384766,
+      "reward_std": 0.010041882294171956,
+      "rewards/reward_func_1": 0.051538944244384766,
+      "step": 640
+    },
+    {
+      "completion_length": 4.340625,
+      "epoch": 0.17341040462427745,
+      "grad_norm": 9511026688.0,
+      "kl": 79088875.28125,
+      "learning_rate": 1.9673482898135774e-05,
+      "loss": 3171008.6,
+      "reward": 0.05210127830505371,
+      "reward_std": 0.010918277798919008,
+      "rewards/reward_func_1": 0.05210127830505371,
+      "step": 645
+    },
+    {
+      "completion_length": 3.2375,
+      "epoch": 0.17475467132679123,
+      "grad_norm": 1.1192519664764404,
+      "kl": 17511264.096875,
+      "learning_rate": 1.9661481592929293e-05,
+      "loss": 700102.15,
+      "reward": 0.059194572269916534,
+      "reward_std": 0.010659490662510507,
+      "rewards/reward_func_1": 0.059194572269916534,
+      "step": 650
+    },
+    {
+      "completion_length": 441.04375,
+      "epoch": 0.176098938029305,
+      "grad_norm": 3818.766845703125,
+      "kl": 449005.0505859375,
+      "learning_rate": 1.9649267487899507e-05,
+      "loss": 18001.5281,
+      "reward": 0.0049600392580032345,
+      "reward_std": 0.014874692249577492,
+      "rewards/reward_func_1": 0.0049600392580032345,
+      "step": 655
+    },
+    {
+      "completion_length": 24.63125,
+      "epoch": 0.1774432047318188,
+      "grad_norm": 1.1994178295135498,
+      "kl": 7.169140625,
+      "learning_rate": 1.9636840852069284e-05,
+      "loss": 0.2868,
+      "reward": 0.02127237692475319,
+      "reward_std": 0.01846984715666622,
+      "rewards/reward_func_1": 0.02127237692475319,
+      "step": 660
+    },
+    {
+      "completion_length": 4.38125,
+      "epoch": 0.17878747143433257,
+      "grad_norm": 271670.03125,
+      "kl": 6157.2625,
+      "learning_rate": 1.962420195914259e-05,
+      "loss": 245.975,
+      "reward": 0.04686172604560852,
+      "reward_std": 0.013880293245892971,
+      "rewards/reward_func_1": 0.04686172604560852,
+      "step": 665
+    },
+    {
+      "completion_length": 7.53125,
+      "epoch": 0.18013173813684635,
+      "grad_norm": 1.2155910730361938,
+      "kl": 11889.5125,
+      "learning_rate": 1.961135108749849e-05,
+      "loss": 477.0842,
+      "reward": 0.04393459558486938,
+      "reward_std": 0.01759743633447215,
+      "rewards/reward_func_1": 0.04393459558486938,
+      "step": 670
+    },
+    {
+      "completion_length": 12.85625,
+      "epoch": 0.18147600483936013,
+      "grad_norm": 1.1338227987289429,
+      "kl": 10.88984375,
+      "learning_rate": 1.9598288520185e-05,
+      "loss": 0.4355,
+      "reward": 0.029730018973350526,
+      "reward_std": 0.018472507712431252,
+      "rewards/reward_func_1": 0.029730018973350526,
+      "step": 675
+    },
+    {
+      "completion_length": 10.8125,
+      "epoch": 0.1828202715418739,
+      "grad_norm": 1.7272660732269287,
+      "kl": 12.0734375,
+      "learning_rate": 1.958501454491286e-05,
+      "loss": 0.4834,
+      "reward": 0.039747095108032225,
+      "reward_std": 0.020039613964036106,
+      "rewards/reward_func_1": 0.039747095108032225,
+      "step": 680
+    },
+    {
+      "completion_length": 9.821875,
+      "epoch": 0.1841645382443877,
+      "grad_norm": 0.9799548983573914,
+      "kl": 15.834375,
+      "learning_rate": 1.95715294540492e-05,
+      "loss": 0.6336,
+      "reward": 0.04382616728544235,
+      "reward_std": 0.017328777379589155,
+      "rewards/reward_func_1": 0.04382616728544235,
+      "step": 685
+    },
+    {
+      "completion_length": 6.875,
+      "epoch": 0.18550880494690147,
+      "grad_norm": 0.0692245364189148,
+      "kl": 15.3328125,
+      "learning_rate": 1.9557833544611083e-05,
+      "loss": 0.6131,
+      "reward": 0.05723133087158203,
+      "reward_std": 0.012962383369449526,
+      "rewards/reward_func_1": 0.05723133087158203,
+      "step": 690
+    },
+    {
+      "completion_length": 111.0,
+      "epoch": 0.18685307164941525,
+      "grad_norm": 29.649320602416992,
+      "kl": 15.096875,
+      "learning_rate": 1.9543927118258988e-05,
+      "loss": 0.6041,
+      "reward": 0.059267282485961914,
+      "reward_std": 0.016366570102400148,
+      "rewards/reward_func_1": 0.059267282485961914,
+      "step": 695
+    },
+    {
+      "completion_length": 33.9375,
+      "epoch": 0.18819733835192903,
+      "grad_norm": 0.007695461623370647,
+      "kl": 322.8,
+      "learning_rate": 1.9529810481290143e-05,
+      "loss": 12.915,
+      "reward": 0.056771063804626466,
+      "reward_std": 0.012514285945508163,
+      "rewards/reward_func_1": 0.056771063804626466,
+      "step": 700
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.1895416050544428,
+      "grad_norm": 0.023763682693243027,
+      "kl": 18.1828125,
+      "learning_rate": 1.9515483944631793e-05,
+      "loss": 0.7269,
+      "reward": 0.06125969886779785,
+      "reward_std": 0.009932457827380859,
+      "rewards/reward_func_1": 0.06125969886779785,
+      "step": 705
+    },
+    {
+      "completion_length": 5.2,
+      "epoch": 0.1908858717569566,
+      "grad_norm": 0.3108590841293335,
+      "kl": 18.0453125,
+      "learning_rate": 1.9500947823834345e-05,
+      "loss": 0.7218,
+      "reward": 0.0602872371673584,
+      "reward_std": 0.009525550016041962,
+      "rewards/reward_func_1": 0.0602872371673584,
+      "step": 710
+    },
+    {
+      "completion_length": 11.98125,
+      "epoch": 0.19223013845947035,
+      "grad_norm": 0.009819800965487957,
+      "kl": 18.3109375,
+      "learning_rate": 1.9486202439064433e-05,
+      "loss": 0.732,
+      "reward": 0.05926952362060547,
+      "reward_std": 0.010095558775356039,
+      "rewards/reward_func_1": 0.05926952362060547,
+      "step": 715
+    },
+    {
+      "completion_length": 4.425,
+      "epoch": 0.19357440516198413,
+      "grad_norm": 0.007831516675651073,
+      "kl": 17.8375,
+      "learning_rate": 1.9471248115097827e-05,
+      "loss": 0.7131,
+      "reward": 0.06079845428466797,
+      "reward_std": 0.010249754647520603,
+      "rewards/reward_func_1": 0.06079845428466797,
+      "step": 720
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.1949186718644979,
+      "grad_norm": 0.006839285604655743,
+      "kl": 18.021875,
+      "learning_rate": 1.9456085181312333e-05,
+      "loss": 0.7214,
+      "reward": 0.06195640563964844,
+      "reward_std": 0.012506642258085777,
+      "rewards/reward_func_1": 0.06195640563964844,
+      "step": 725
+    },
+    {
+      "completion_length": 2.00625,
+      "epoch": 0.19626293856701169,
+      "grad_norm": 0.0004078986239619553,
+      "kl": 17.8234375,
+      "learning_rate": 1.9440713971680494e-05,
+      "loss": 0.7135,
+      "reward": 0.05450363159179687,
+      "reward_std": 0.010389497011783533,
+      "rewards/reward_func_1": 0.05450363159179687,
+      "step": 730
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.19760720526952547,
+      "grad_norm": 2.693261922104284e-05,
+      "kl": 18.0421875,
+      "learning_rate": 1.9425134824762263e-05,
+      "loss": 0.722,
+      "reward": 0.06317386627197266,
+      "reward_std": 0.01099952881995705,
+      "rewards/reward_func_1": 0.06317386627197266,
+      "step": 735
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.19895147197203925,
+      "grad_norm": 0.0003110544930677861,
+      "kl": 18.16875,
+      "learning_rate": 1.9409348083697516e-05,
+      "loss": 0.7272,
+      "reward": 0.061242103576660156,
+      "reward_std": 0.011685801808926043,
+      "rewards/reward_func_1": 0.061242103576660156,
+      "step": 740
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20029573867455303,
+      "grad_norm": 2.4723798560444266e-05,
+      "kl": 17.915625,
+      "learning_rate": 1.9393354096198535e-05,
+      "loss": 0.7161,
+      "reward": 0.054812145233154294,
+      "reward_std": 0.01056510213547881,
+      "rewards/reward_func_1": 0.054812145233154294,
+      "step": 745
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2016400053770668,
+      "grad_norm": 0.00010621309047564864,
+      "kl": 17.85625,
+      "learning_rate": 1.937715321454232e-05,
+      "loss": 0.7141,
+      "reward": 0.05950497388839722,
+      "reward_std": 0.011216246478579706,
+      "rewards/reward_func_1": 0.05950497388839722,
+      "step": 750
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20298427207958059,
+      "grad_norm": 7.163731061154976e-05,
+      "kl": 17.6765625,
+      "learning_rate": 1.9360745795562813e-05,
+      "loss": 0.7074,
+      "reward": 0.06266632080078124,
+      "reward_std": 0.011260019605106208,
+      "rewards/reward_func_1": 0.06266632080078124,
+      "step": 755
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20432853878209437,
+      "grad_norm": 0.00017179730639327317,
+      "kl": 17.6109375,
+      "learning_rate": 1.9344132200643102e-05,
+      "loss": 0.7048,
+      "reward": 0.0631840705871582,
+      "reward_std": 0.01339399583703198,
+      "rewards/reward_func_1": 0.0631840705871582,
+      "step": 760
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20567280548460815,
+      "grad_norm": 0.00033472245559096336,
+      "kl": 17.9296875,
+      "learning_rate": 1.9327312795707392e-05,
+      "loss": 0.7169,
+      "reward": 0.06261520385742188,
+      "reward_std": 0.011459613528859335,
+      "rewards/reward_func_1": 0.06261520385742188,
+      "step": 765
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20701707218712193,
+      "grad_norm": 0.02129560336470604,
+      "kl": 17.7078125,
+      "learning_rate": 1.931028795121299e-05,
+      "loss": 0.7074,
+      "reward": 0.060090065002441406,
+      "reward_std": 0.010715857451577904,
+      "rewards/reward_func_1": 0.060090065002441406,
+      "step": 770
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2083613388896357,
+      "grad_norm": 0.00020586424216162413,
+      "kl": 17.7390625,
+      "learning_rate": 1.9293058042142117e-05,
+      "loss": 0.7097,
+      "reward": 0.05955848693847656,
+      "reward_std": 0.010681279400523635,
+      "rewards/reward_func_1": 0.05955848693847656,
+      "step": 775
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.20970560559214949,
+      "grad_norm": 0.00016600097296759486,
+      "kl": 17.5921875,
+      "learning_rate": 1.9275623447993678e-05,
+      "loss": 0.7034,
+      "reward": 0.06024360656738281,
+      "reward_std": 0.010565872873849002,
+      "rewards/reward_func_1": 0.06024360656738281,
+      "step": 780
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21104987229466327,
+      "grad_norm": 0.0002437598304823041,
+      "kl": 17.684375,
+      "learning_rate": 1.9257984552774874e-05,
+      "loss": 0.7073,
+      "reward": 0.06276130676269531,
+      "reward_std": 0.013013198171756812,
+      "rewards/reward_func_1": 0.06276130676269531,
+      "step": 785
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21239413899717705,
+      "grad_norm": 0.00045892002526670694,
+      "kl": 17.6234375,
+      "learning_rate": 1.9240141744992763e-05,
+      "loss": 0.7051,
+      "reward": 0.06035938262939453,
+      "reward_std": 0.013352590511203744,
+      "rewards/reward_func_1": 0.06035938262939453,
+      "step": 790
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21373840569969083,
+      "grad_norm": 0.00022575826733373106,
+      "kl": 17.875,
+      "learning_rate": 1.9222095417645695e-05,
+      "loss": 0.7155,
+      "reward": 0.058776569366455075,
+      "reward_std": 0.011941832641605287,
+      "rewards/reward_func_1": 0.058776569366455075,
+      "step": 795
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2150826724022046,
+      "grad_norm": 0.0002709394320845604,
+      "kl": 17.8890625,
+      "learning_rate": 1.920384596821467e-05,
+      "loss": 0.7157,
+      "reward": 0.05806446075439453,
+      "reward_std": 0.00929926319167862,
+      "rewards/reward_func_1": 0.05806446075439453,
+      "step": 800
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21642693910471839,
+      "grad_norm": 0.0005522365099750459,
+      "kl": 18.215625,
+      "learning_rate": 1.9185393798654547e-05,
+      "loss": 0.7285,
+      "reward": 0.060375118255615236,
+      "reward_std": 0.012819936085725204,
+      "rewards/reward_func_1": 0.060375118255615236,
+      "step": 805
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21777120580723217,
+      "grad_norm": 0.00012125197827117518,
+      "kl": 18.078125,
+      "learning_rate": 1.9166739315385244e-05,
+      "loss": 0.7234,
+      "reward": 0.06392664909362793,
+      "reward_std": 0.009824539528926835,
+      "rewards/reward_func_1": 0.06392664909362793,
+      "step": 810
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.21911547250974595,
+      "grad_norm": 0.0004721728328149766,
+      "kl": 17.84375,
+      "learning_rate": 1.9147882929282734e-05,
+      "loss": 0.7138,
+      "reward": 0.061508560180664064,
+      "reward_std": 0.011408517364179716,
+      "rewards/reward_func_1": 0.061508560180664064,
+      "step": 815
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.22045973921225973,
+      "grad_norm": 0.0005403547547757626,
+      "kl": 17.646875,
+      "learning_rate": 1.9128825055670035e-05,
+      "loss": 0.7059,
+      "reward": 0.059009552001953125,
+      "reward_std": 0.009685787269700086,
+      "rewards/reward_func_1": 0.059009552001953125,
+      "step": 820
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.22180400591477348,
+      "grad_norm": 0.0022164226975291967,
+      "kl": 17.9734375,
+      "learning_rate": 1.9109566114308036e-05,
+      "loss": 0.7187,
+      "reward": 0.05564627647399902,
+      "reward_std": 0.010326084749249276,
+      "rewards/reward_func_1": 0.05564627647399902,
+      "step": 825
+    },
+    {
+      "completion_length": 2.35,
+      "epoch": 0.22314827261728726,
+      "grad_norm": 0.0012156780576333404,
+      "kl": 17.66875,
+      "learning_rate": 1.9090106529386263e-05,
+      "loss": 0.7067,
+      "reward": 0.0656036376953125,
+      "reward_std": 0.015077763356384822,
+      "rewards/reward_func_1": 0.0656036376953125,
+      "step": 830
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.22449253931980104,
+      "grad_norm": 0.0010059759952127934,
+      "kl": 18.1734375,
+      "learning_rate": 1.907044672951354e-05,
+      "loss": 0.7272,
+      "reward": 0.057573127746582034,
+      "reward_std": 0.010585914782132022,
+      "rewards/reward_func_1": 0.057573127746582034,
+      "step": 835
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.22583680602231482,
+      "grad_norm": 0.0004464346857275814,
+      "kl": 17.9625,
+      "learning_rate": 1.9050587147708544e-05,
+      "loss": 0.7182,
+      "reward": 0.06241474151611328,
+      "reward_std": 0.009492194746417226,
+      "rewards/reward_func_1": 0.06241474151611328,
+      "step": 840
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2271810727248286,
+      "grad_norm": 0.0005409275181591511,
+      "kl": 18.2171875,
+      "learning_rate": 1.9030528221390255e-05,
+      "loss": 0.7287,
+      "reward": 0.06225318908691406,
+      "reward_std": 0.011348171227291459,
+      "rewards/reward_func_1": 0.06225318908691406,
+      "step": 845
+    },
+    {
+      "completion_length": 6.090625,
+      "epoch": 0.22852533942734238,
+      "grad_norm": 13.99404525756836,
+      "kl": 17.190625,
+      "learning_rate": 1.9010270392368343e-05,
+      "loss": 0.6867,
+      "reward": 0.0607336699962616,
+      "reward_std": 0.014851068891584874,
+      "rewards/reward_func_1": 0.0607336699962616,
+      "step": 850
+    },
+    {
+      "completion_length": 3.1,
+      "epoch": 0.22986960612985616,
+      "grad_norm": 0.0005020995158702135,
+      "kl": 17.60625,
+      "learning_rate": 1.898981410683343e-05,
+      "loss": 0.7042,
+      "reward": 0.06041567623615265,
+      "reward_std": 0.012269638044381281,
+      "rewards/reward_func_1": 0.06041567623615265,
+      "step": 855
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.23121387283236994,
+      "grad_norm": 0.0013556176563724875,
+      "kl": 17.64375,
+      "learning_rate": 1.8969159815347253e-05,
+      "loss": 0.7065,
+      "reward": 0.06363449096679688,
+      "reward_std": 0.010352238497580402,
+      "rewards/reward_func_1": 0.06363449096679688,
+      "step": 860
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.23255813953488372,
+      "grad_norm": 0.0008380432846024632,
+      "kl": 17.7609375,
+      "learning_rate": 1.8948307972832744e-05,
+      "loss": 0.7101,
+      "reward": 0.06133832931518555,
+      "reward_std": 0.012271754596440587,
+      "rewards/reward_func_1": 0.06133832931518555,
+      "step": 865
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2339024062373975,
+      "grad_norm": 0.0016801492311060429,
+      "kl": 17.803125,
+      "learning_rate": 1.8927259038564023e-05,
+      "loss": 0.7121,
+      "reward": 0.06001472473144531,
+      "reward_std": 0.011952074009786883,
+      "rewards/reward_func_1": 0.06001472473144531,
+      "step": 870
+    },
+    {
+      "completion_length": 2.459375,
+      "epoch": 0.23524667293991128,
+      "grad_norm": 0.07210814952850342,
+      "kl": 31147.065625,
+      "learning_rate": 1.8906013476156265e-05,
+      "loss": 1248.7868,
+      "reward": 0.05841388702392578,
+      "reward_std": 0.011889992751093814,
+      "rewards/reward_func_1": 0.05841388702392578,
+      "step": 875
+    },
+    {
+      "completion_length": 2.028125,
+      "epoch": 0.23659093964242506,
+      "grad_norm": 0.04944615811109543,
+      "kl": 17.9125,
+      "learning_rate": 1.8884571753555495e-05,
+      "loss": 0.7165,
+      "reward": 0.059661483764648436,
+      "reward_std": 0.01035475345343002,
+      "rewards/reward_func_1": 0.059661483764648436,
+      "step": 880
+    },
+    {
+      "completion_length": 4.909375,
+      "epoch": 0.23793520634493884,
+      "grad_norm": 0.5363011360168457,
+      "kl": 17.625,
+      "learning_rate": 1.8862934343028288e-05,
+      "loss": 0.7049,
+      "reward": 0.06338434219360352,
+      "reward_std": 0.012126463351160055,
+      "rewards/reward_func_1": 0.06338434219360352,
+      "step": 885
+    },
+    {
+      "completion_length": 62.634375,
+      "epoch": 0.23927947304745262,
+      "grad_norm": 0.1920609325170517,
+      "kl": 16.6390625,
+      "learning_rate": 1.884110172115135e-05,
+      "loss": 0.6654,
+      "reward": 0.05351438522338867,
+      "reward_std": 0.02005903590179514,
+      "rewards/reward_func_1": 0.05351438522338867,
+      "step": 890
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2406237397499664,
+      "grad_norm": 0.0013905576197430491,
+      "kl": 18.1953125,
+      "learning_rate": 1.8819074368801045e-05,
+      "loss": 0.7282,
+      "reward": 0.06563434600830079,
+      "reward_std": 0.009951398673729272,
+      "rewards/reward_func_1": 0.06563434600830079,
+      "step": 895
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.24196800645248018,
+      "grad_norm": 0.0008331938879564404,
+      "kl": 18.003125,
+      "learning_rate": 1.8796852771142778e-05,
+      "loss": 0.7201,
+      "reward": 0.061870574951171875,
+      "reward_std": 0.012352473140344955,
+      "rewards/reward_func_1": 0.061870574951171875,
+      "step": 900
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.24331227315499396,
+      "grad_norm": 0.00020705144561361521,
+      "kl": 18.053125,
+      "learning_rate": 1.8774437417620334e-05,
+      "loss": 0.7223,
+      "reward": 0.06816902160644531,
+      "reward_std": 0.012587691200315021,
+      "rewards/reward_func_1": 0.06816902160644531,
+      "step": 905
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.24465653985750774,
+      "grad_norm": 0.0003319174575153738,
+      "kl": 17.8921875,
+      "learning_rate": 1.8751828801945074e-05,
+      "loss": 0.7151,
+      "reward": 0.058438873291015624,
+      "reward_std": 0.012193899090743799,
+      "rewards/reward_func_1": 0.058438873291015624,
+      "step": 910
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.24600080656002152,
+      "grad_norm": 0.000264251691987738,
+      "kl": 18.0390625,
+      "learning_rate": 1.872902742208508e-05,
+      "loss": 0.7217,
+      "reward": 0.060257339477539064,
+      "reward_std": 0.010663219789967116,
+      "rewards/reward_func_1": 0.060257339477539064,
+      "step": 915
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2473450732625353,
+      "grad_norm": 0.0001942398666869849,
+      "kl": 17.9234375,
+      "learning_rate": 1.8706033780254168e-05,
+      "loss": 0.7168,
+      "reward": 0.05674247741699219,
+      "reward_std": 0.009646143747158931,
+      "rewards/reward_func_1": 0.05674247741699219,
+      "step": 920
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.24868933996504908,
+      "grad_norm": 0.000582867010962218,
+      "kl": 17.5609375,
+      "learning_rate": 1.8682848382900852e-05,
+      "loss": 0.7027,
+      "reward": 0.06358718872070312,
+      "reward_std": 0.01264539449075528,
+      "rewards/reward_func_1": 0.06358718872070312,
+      "step": 925
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.25003360666756286,
+      "grad_norm": 0.0004526897973846644,
+      "kl": 17.9296875,
+      "learning_rate": 1.865947174069716e-05,
+      "loss": 0.7172,
+      "reward": 0.059136390686035156,
+      "reward_std": 0.010592962511873338,
+      "rewards/reward_func_1": 0.059136390686035156,
+      "step": 930
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2513778733700766,
+      "grad_norm": 0.0003386743483133614,
+      "kl": 17.75625,
+      "learning_rate": 1.8635904368527406e-05,
+      "loss": 0.7107,
+      "reward": 0.06310138702392579,
+      "reward_std": 0.011821250266802964,
+      "rewards/reward_func_1": 0.06310138702392579,
+      "step": 935
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2527221400725904,
+      "grad_norm": 0.00037925346987321973,
+      "kl": 17.703125,
+      "learning_rate": 1.861214678547685e-05,
+      "loss": 0.7079,
+      "reward": 0.06231670379638672,
+      "reward_std": 0.010538342622749042,
+      "rewards/reward_func_1": 0.06231670379638672,
+      "step": 940
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.25406640677510417,
+      "grad_norm": 0.0006748105515725911,
+      "kl": 17.96875,
+      "learning_rate": 1.858819951482026e-05,
+      "loss": 0.7188,
+      "reward": 0.05954875946044922,
+      "reward_std": 0.011791958093454014,
+      "rewards/reward_func_1": 0.05954875946044922,
+      "step": 945
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.255410673477618,
+      "grad_norm": 0.000709658779669553,
+      "kl": 17.6859375,
+      "learning_rate": 1.856406308401036e-05,
+      "loss": 0.7072,
+      "reward": 0.0561366081237793,
+      "reward_std": 0.009839185555756557,
+      "rewards/reward_func_1": 0.0561366081237793,
+      "step": 950
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.25675494018013173,
+      "grad_norm": 0.0004975300398655236,
+      "kl": 17.975,
+      "learning_rate": 1.853973802466627e-05,
+      "loss": 0.7186,
+      "reward": 0.0569252610206604,
+      "reward_std": 0.009655545311397873,
+      "rewards/reward_func_1": 0.0569252610206604,
+      "step": 955
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.25809920688264554,
+      "grad_norm": 0.0006206005346029997,
+      "kl": 17.8875,
+      "learning_rate": 1.8515224872561745e-05,
+      "loss": 0.7151,
+      "reward": 0.06045455932617187,
+      "reward_std": 0.011623913834773703,
+      "rewards/reward_func_1": 0.06045455932617187,
+      "step": 960
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2594434735851593,
+      "grad_norm": 0.0007395711145363748,
+      "kl": 18.040625,
+      "learning_rate": 1.8490524167613405e-05,
+      "loss": 0.7214,
+      "reward": 0.057852745056152344,
+      "reward_std": 0.011105244704231155,
+      "rewards/reward_func_1": 0.057852745056152344,
+      "step": 965
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2607877402876731,
+      "grad_norm": 0.0008309457916766405,
+      "kl": 17.6515625,
+      "learning_rate": 1.8465636453868825e-05,
+      "loss": 0.7064,
+      "reward": 0.06783523559570312,
+      "reward_std": 0.009911755218854523,
+      "rewards/reward_func_1": 0.06783523559570312,
+      "step": 970
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.26213200699018685,
+      "grad_norm": 0.000725765130482614,
+      "kl": 17.36875,
+      "learning_rate": 1.8440562279494557e-05,
+      "loss": 0.695,
+      "reward": 0.05620386600494385,
+      "reward_std": 0.009591523706330918,
+      "rewards/reward_func_1": 0.05620386600494385,
+      "step": 975
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.26347627369270066,
+      "grad_norm": 0.0008614324615336955,
+      "kl": 17.940625,
+      "learning_rate": 1.8415302196764068e-05,
+      "loss": 0.7172,
+      "reward": 0.06371011734008789,
+      "reward_std": 0.012515782276750542,
+      "rewards/reward_func_1": 0.06371011734008789,
+      "step": 980
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2648205403952144,
+      "grad_norm": 0.0006585062947124243,
+      "kl": 18.028125,
+      "learning_rate": 1.8389856762045556e-05,
+      "loss": 0.7213,
+      "reward": 0.05774202346801758,
+      "reward_std": 0.01049440445349319,
+      "rewards/reward_func_1": 0.05774202346801758,
+      "step": 985
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.26616480709772816,
+      "grad_norm": 0.0015991459367796779,
+      "kl": 17.9125,
+      "learning_rate": 1.836422653578971e-05,
+      "loss": 0.716,
+      "reward": 0.061675214767456056,
+      "reward_std": 0.010034650065063034,
+      "rewards/reward_func_1": 0.061675214767456056,
+      "step": 990
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.26750907380024197,
+      "grad_norm": 0.0015729337465018034,
+      "kl": 17.6875,
+      "learning_rate": 1.8338412082517357e-05,
+      "loss": 0.7081,
+      "reward": 0.057560133934021,
+      "reward_std": 0.010157572498792433,
+      "rewards/reward_func_1": 0.057560133934021,
+      "step": 995
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2688533405027557,
+      "grad_norm": 0.0010110485600307584,
+      "kl": 18.175,
+      "learning_rate": 1.8312413970807043e-05,
+      "loss": 0.7263,
+      "reward": 0.058531570434570315,
+      "reward_std": 0.009644822326663416,
+      "rewards/reward_func_1": 0.058531570434570315,
+      "step": 1000
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.27019760720526953,
+      "grad_norm": 0.0011084715370088816,
+      "kl": 17.73125,
+      "learning_rate": 1.8286232773282492e-05,
+      "loss": 0.7093,
+      "reward": 0.05668430328369141,
+      "reward_std": 0.009714638943114551,
+      "rewards/reward_func_1": 0.05668430328369141,
+      "step": 1005
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2715418739077833,
+      "grad_norm": 0.0011657410068437457,
+      "kl": 17.453125,
+      "learning_rate": 1.8259869066600005e-05,
+      "loss": 0.6981,
+      "reward": 0.060795021057128903,
+      "reward_std": 0.008335485706629698,
+      "rewards/reward_func_1": 0.060795021057128903,
+      "step": 1010
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2728861406102971,
+      "grad_norm": 0.0010427763918414712,
+      "kl": 18.075,
+      "learning_rate": 1.8233323431435744e-05,
+      "loss": 0.723,
+      "reward": 0.06016595363616943,
+      "reward_std": 0.011402350757271052,
+      "rewards/reward_func_1": 0.06016595363616943,
+      "step": 1015
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.27423040731281084,
+      "grad_norm": 0.00200888910330832,
+      "kl": 17.9109375,
+      "learning_rate": 1.820659645247296e-05,
+      "loss": 0.717,
+      "reward": 0.056774234771728514,
+      "reward_std": 0.011336608513374813,
+      "rewards/reward_func_1": 0.056774234771728514,
+      "step": 1020
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.27557467401532465,
+      "grad_norm": 0.0016117419581860304,
+      "kl": 17.765625,
+      "learning_rate": 1.8179688718389116e-05,
+      "loss": 0.7105,
+      "reward": 0.06235724687576294,
+      "reward_std": 0.010060734899889212,
+      "rewards/reward_func_1": 0.06235724687576294,
+      "step": 1025
+    },
+    {
+      "completion_length": 2.45,
+      "epoch": 0.2769189407178384,
+      "grad_norm": 0.0025889407843351364,
+      "kl": 17.771875,
+      "learning_rate": 1.8152600821842902e-05,
+      "loss": 0.711,
+      "reward": 0.058268165588378905,
+      "reward_std": 0.01004049998264236,
+      "rewards/reward_func_1": 0.058268165588378905,
+      "step": 1030
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.2782632074203522,
+      "grad_norm": 0.0021764549892395735,
+      "kl": 17.765625,
+      "learning_rate": 1.8125333359461194e-05,
+      "loss": 0.7108,
+      "reward": 0.05997686386108399,
+      "reward_std": 0.010472600482171402,
+      "rewards/reward_func_1": 0.05997686386108399,
+      "step": 1035
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.27960747412286596,
+      "grad_norm": 0.0027070462238043547,
+      "kl": 18.00625,
+      "learning_rate": 1.8097886931825916e-05,
+      "loss": 0.72,
+      "reward": 0.057472729682922365,
+      "reward_std": 0.010958646313520148,
+      "rewards/reward_func_1": 0.057472729682922365,
+      "step": 1040
+    },
+    {
+      "completion_length": 2.00625,
+      "epoch": 0.28095174082537977,
+      "grad_norm": 0.02638576552271843,
+      "kl": 17.8140625,
+      "learning_rate": 1.8070262143460803e-05,
+      "loss": 0.7121,
+      "reward": 0.06007643789052963,
+      "reward_std": 0.01233230889774859,
+      "rewards/reward_func_1": 0.06007643789052963,
+      "step": 1045
+    },
+    {
+      "completion_length": 2.4625,
+      "epoch": 0.2822960075278935,
+      "grad_norm": 0.38510629534721375,
+      "kl": 17.2859375,
+      "learning_rate": 1.8042459602818092e-05,
+      "loss": 0.6911,
+      "reward": 0.060521507263183595,
+      "reward_std": 0.011037798667530296,
+      "rewards/reward_func_1": 0.060521507263183595,
+      "step": 1050
+    },
+    {
+      "completion_length": 4.6375,
+      "epoch": 0.28364027423040733,
+      "grad_norm": 1.0965192317962646,
+      "kl": 15.5390625,
+      "learning_rate": 1.8014479922265117e-05,
+      "loss": 0.6215,
+      "reward": 0.05944366455078125,
+      "reward_std": 0.01202023433870636,
+      "rewards/reward_func_1": 0.05944366455078125,
+      "step": 1055
+    },
+    {
+      "completion_length": 68.578125,
+      "epoch": 0.2849845409329211,
+      "grad_norm": 136523.703125,
+      "kl": 128.46640625,
+      "learning_rate": 1.7986323718070826e-05,
+      "loss": 5.144,
+      "reward": 0.056297135353088376,
+      "reward_std": 0.014432728511746973,
+      "rewards/reward_func_1": 0.056297135353088376,
+      "step": 1060
+    },
+    {
+      "completion_length": 151.353125,
+      "epoch": 0.2863288076354349,
+      "grad_norm": 1.3205143213272095,
+      "kl": 6.2421875,
+      "learning_rate": 1.79579916103922e-05,
+      "loss": 0.2498,
+      "reward": 0.022794413566589355,
+      "reward_std": 0.025071121371001936,
+      "rewards/reward_func_1": 0.022794413566589355,
+      "step": 1065
+    },
+    {
+      "completion_length": 60.24375,
+      "epoch": 0.28767307433794864,
+      "grad_norm": 0.668519139289856,
+      "kl": 13.2421875,
+      "learning_rate": 1.79294842232606e-05,
+      "loss": 0.5298,
+      "reward": 0.02910344898700714,
+      "reward_std": 0.021979624161031098,
+      "rewards/reward_func_1": 0.02910344898700714,
+      "step": 1070
+    },
+    {
+      "completion_length": 69.1875,
+      "epoch": 0.28901734104046245,
+      "grad_norm": 0.38422349095344543,
+      "kl": 15.446875,
+      "learning_rate": 1.7900802184568024e-05,
+      "loss": 0.6174,
+      "reward": 0.032975000143051145,
+      "reward_std": 0.019280125828663584,
+      "rewards/reward_func_1": 0.032975000143051145,
+      "step": 1075
+    },
+    {
+      "completion_length": 1.93125,
+      "epoch": 0.2903616077429762,
+      "grad_norm": 0.4854346513748169,
+      "kl": 18.4921875,
+      "learning_rate": 1.7871946126053265e-05,
+      "loss": 0.7396,
+      "reward": 0.0445002555847168,
+      "reward_std": 0.009190794143796666,
+      "rewards/reward_func_1": 0.0445002555847168,
+      "step": 1080
+    },
+    {
+      "completion_length": 13.75625,
+      "epoch": 0.29170587444549,
+      "grad_norm": 0.23374588787555695,
+      "kl": 16.453125,
+      "learning_rate": 1.784291668328801e-05,
+      "loss": 0.658,
+      "reward": 0.04094771146774292,
+      "reward_std": 0.010110658951089136,
+      "rewards/reward_func_1": 0.04094771146774292,
+      "step": 1085
+    },
+    {
+      "completion_length": 55.621875,
+      "epoch": 0.29305014114800376,
+      "grad_norm": 0.29550668597221375,
+      "kl": 14.3140625,
+      "learning_rate": 1.781371449566284e-05,
+      "loss": 0.5726,
+      "reward": 0.039327383041381836,
+      "reward_std": 0.014201272143691313,
+      "rewards/reward_func_1": 0.039327383041381836,
+      "step": 1090
+    },
+    {
+      "completion_length": 2.878125,
+      "epoch": 0.29439440785051757,
+      "grad_norm": 0.15203788876533508,
+      "kl": 16.3359375,
+      "learning_rate": 1.7784340206373135e-05,
+      "loss": 0.6532,
+      "reward": 0.04413075447082519,
+      "reward_std": 0.007815522653254447,
+      "rewards/reward_func_1": 0.04413075447082519,
+      "step": 1095
+    },
+    {
+      "completion_length": 2.740625,
+      "epoch": 0.2957386745530313,
+      "grad_norm": 0.1516532450914383,
+      "kl": 17.3421875,
+      "learning_rate": 1.7754794462404924e-05,
+      "loss": 0.6937,
+      "reward": 0.045378980413079265,
+      "reward_std": 0.008754154351481701,
+      "rewards/reward_func_1": 0.045378980413079265,
+      "step": 1100
+    },
+    {
+      "completion_length": 2.86875,
+      "epoch": 0.2970829412555451,
+      "grad_norm": 0.3617139160633087,
+      "kl": 17.16875,
+      "learning_rate": 1.772507791452062e-05,
+      "loss": 0.687,
+      "reward": 0.03869695663452148,
+      "reward_std": 0.008698664297844516,
+      "rewards/reward_func_1": 0.03869695663452148,
+      "step": 1105
+    },
+    {
+      "completion_length": 6.446875,
+      "epoch": 0.2984272079580589,
+      "grad_norm": 0.12706607580184937,
+      "kl": 16.409375,
+      "learning_rate": 1.7695191217244694e-05,
+      "loss": 0.6564,
+      "reward": 0.04749107360839844,
+      "reward_std": 0.008462011188385077,
+      "rewards/reward_func_1": 0.04749107360839844,
+      "step": 1110
+    },
+    {
+      "completion_length": 72.753125,
+      "epoch": 0.29977147466057263,
+      "grad_norm": 0.321855753660202,
+      "kl": 13.3921875,
+      "learning_rate": 1.766513502884926e-05,
+      "loss": 0.5358,
+      "reward": 0.04237784147262573,
+      "reward_std": 0.0151958847156493,
+      "rewards/reward_func_1": 0.04237784147262573,
+      "step": 1115
+    },
+    {
+      "completion_length": 131.153125,
+      "epoch": 0.30111574136308644,
+      "grad_norm": 0.1894012689590454,
+      "kl": 12.8296875,
+      "learning_rate": 1.7634910011339576e-05,
+      "loss": 0.5134,
+      "reward": 0.03622118234634399,
+      "reward_std": 0.016812963741540444,
+      "rewards/reward_func_1": 0.03622118234634399,
+      "step": 1120
+    },
+    {
+      "completion_length": 225.79375,
+      "epoch": 0.3024600080656002,
+      "grad_norm": 0.6886034607887268,
+      "kl": 119.9203125,
+      "learning_rate": 1.7604516830439447e-05,
+      "loss": 4.8239,
+      "reward": 0.028684809803962708,
+      "reward_std": 0.020281461635022424,
+      "rewards/reward_func_1": 0.028684809803962708,
+      "step": 1125
+    },
+    {
+      "completion_length": 733.759375,
+      "epoch": 0.303804274768114,
+      "grad_norm": 0.5576857328414917,
+      "kl": 7.11328125,
+      "learning_rate": 1.7573956155576596e-05,
+      "loss": 0.2844,
+      "reward": 0.007758472859859466,
+      "reward_std": 0.014932763832621276,
+      "rewards/reward_func_1": 0.007758472859859466,
+      "step": 1130
+    },
+    {
+      "completion_length": 251.840625,
+      "epoch": 0.30514854147062775,
+      "grad_norm": 1.2845573425292969,
+      "kl": 421385.557421875,
+      "learning_rate": 1.7543228659867887e-05,
+      "loss": 16848.1047,
+      "reward": 0.007775214128196239,
+      "reward_std": 0.017382631546934136,
+      "rewards/reward_func_1": 0.007775214128196239,
+      "step": 1135
+    },
+    {
+      "completion_length": 2.09375,
+      "epoch": 0.30649280817314156,
+      "grad_norm": 1.5478886365890503,
+      "kl": 18.3546875,
+      "learning_rate": 1.7512335020104507e-05,
+      "loss": 0.7346,
+      "reward": 0.042022180557250974,
+      "reward_std": 0.011799084773520008,
+      "rewards/reward_func_1": 0.042022180557250974,
+      "step": 1140
+    },
+    {
+      "completion_length": 2.44375,
+      "epoch": 0.3078370748756553,
+      "grad_norm": 0.4710218012332916,
+      "kl": 18.0078125,
+      "learning_rate": 1.7481275916737077e-05,
+      "loss": 0.7209,
+      "reward": 0.043611574172973636,
+      "reward_std": 0.016114455633214675,
+      "rewards/reward_func_1": 0.043611574172973636,
+      "step": 1145
+    },
+    {
+      "completion_length": 3.021875,
+      "epoch": 0.3091813415781691,
+      "grad_norm": 1.128142237663269,
+      "kl": 17.0234375,
+      "learning_rate": 1.7450052033860643e-05,
+      "loss": 0.681,
+      "reward": 0.04341961294412613,
+      "reward_std": 0.01588066411204636,
+      "rewards/reward_func_1": 0.04341961294412613,
+      "step": 1150
+    },
+    {
+      "completion_length": 2.2375,
+      "epoch": 0.3105256082806829,
+      "grad_norm": 1.5875157117843628,
+      "kl": 19.2234375,
+      "learning_rate": 1.7418664059199615e-05,
+      "loss": 0.7687,
+      "reward": 0.05236520916223526,
+      "reward_std": 0.013312915465940022,
+      "rewards/reward_func_1": 0.05236520916223526,
+      "step": 1155
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3118698749831967,
+      "grad_norm": 0.0017928759334608912,
+      "kl": 20.63125,
+      "learning_rate": 1.738711268409263e-05,
+      "loss": 0.8251,
+      "reward": 0.05562934875488281,
+      "reward_std": 0.00945893834286835,
+      "rewards/reward_func_1": 0.05562934875488281,
+      "step": 1160
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.31321414168571043,
+      "grad_norm": 0.003861919976770878,
+      "kl": 21.05625,
+      "learning_rate": 1.73553986034773e-05,
+      "loss": 0.8425,
+      "reward": 0.05521247386932373,
+      "reward_std": 0.0097390030954557,
+      "rewards/reward_func_1": 0.05521247386932373,
+      "step": 1165
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.31455840838822424,
+      "grad_norm": 0.003637129906564951,
+      "kl": 20.484375,
+      "learning_rate": 1.7323522515874945e-05,
+      "loss": 0.8202,
+      "reward": 0.05392255783081055,
+      "reward_std": 0.009917261235386832,
+      "rewards/reward_func_1": 0.05392255783081055,
+      "step": 1170
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.315902675090738,
+      "grad_norm": 0.006247695069760084,
+      "kl": 20.565625,
+      "learning_rate": 1.7291485123375164e-05,
+      "loss": 0.8229,
+      "reward": 0.056228256225585936,
+      "reward_std": 0.009558047083555721,
+      "rewards/reward_func_1": 0.056228256225585936,
+      "step": 1175
+    },
+    {
+      "completion_length": 1.99375,
+      "epoch": 0.3172469417932518,
+      "grad_norm": 0.004366494249552488,
+      "kl": 20.55,
+      "learning_rate": 1.72592871316204e-05,
+      "loss": 0.8223,
+      "reward": 0.062088823318481444,
+      "reward_std": 0.009840463204818661,
+      "rewards/reward_func_1": 0.062088823318481444,
+      "step": 1180
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.31859120849576555,
+      "grad_norm": 0.002056930446997285,
+      "kl": 20.60625,
+      "learning_rate": 1.722692924979039e-05,
+      "loss": 0.8245,
+      "reward": 0.05960988998413086,
+      "reward_std": 0.008964826199371601,
+      "rewards/reward_func_1": 0.05960988998413086,
+      "step": 1185
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.31993547519827936,
+      "grad_norm": 0.015865160152316093,
+      "kl": 21.034375,
+      "learning_rate": 1.719441219058654e-05,
+      "loss": 0.8416,
+      "reward": 0.05759906768798828,
+      "reward_std": 0.009656935631937813,
+      "rewards/reward_func_1": 0.05759906768798828,
+      "step": 1190
+    },
+    {
+      "completion_length": 2.003125,
+      "epoch": 0.3212797419007931,
+      "grad_norm": 0.0372232086956501,
+      "kl": 20.875,
+      "learning_rate": 1.7161736670216233e-05,
+      "loss": 0.8354,
+      "reward": 0.05712289810180664,
+      "reward_std": 0.007381719051045366,
+      "rewards/reward_func_1": 0.05712289810180664,
+      "step": 1195
+    },
+    {
+      "completion_length": 2.003125,
+      "epoch": 0.3226240086033069,
+      "grad_norm": 0.8008459210395813,
+      "kl": 20.2625,
+      "learning_rate": 1.7128903408377053e-05,
+      "loss": 0.8105,
+      "reward": 0.05510530471801758,
+      "reward_std": 0.011877764340533758,
+      "rewards/reward_func_1": 0.05510530471801758,
+      "step": 1200
+    },
+    {
+      "completion_length": 2.13125,
+      "epoch": 0.3239682753058207,
+      "grad_norm": 0.0008978499681688845,
+      "kl": 20.628125,
+      "learning_rate": 1.7095913128240936e-05,
+      "loss": 0.8251,
+      "reward": 0.05747789740562439,
+      "reward_std": 0.011763529140444007,
+      "rewards/reward_func_1": 0.05747789740562439,
+      "step": 1205
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3253125420083344,
+      "grad_norm": 0.0013102364027872682,
+      "kl": 20.58125,
+      "learning_rate": 1.7062766556438233e-05,
+      "loss": 0.8234,
+      "reward": 0.05572299957275391,
+      "reward_std": 0.008765837910323171,
+      "rewards/reward_func_1": 0.05572299957275391,
+      "step": 1210
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.32665680871084823,
+      "grad_norm": 0.0024437177926301956,
+      "kl": 20.46875,
+      "learning_rate": 1.7029464423041713e-05,
+      "loss": 0.8187,
+      "reward": 0.05510997772216797,
+      "reward_std": 0.008241662751242985,
+      "rewards/reward_func_1": 0.05510997772216797,
+      "step": 1215
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.328001075413362,
+      "grad_norm": 0.0028229840099811554,
+      "kl": 20.66875,
+      "learning_rate": 1.6996007461550483e-05,
+      "loss": 0.8269,
+      "reward": 0.053923177719116214,
+      "reward_std": 0.010693888347304892,
+      "rewards/reward_func_1": 0.053923177719116214,
+      "step": 1220
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3293453421158758,
+      "grad_norm": 0.0020496586803346872,
+      "kl": 20.678125,
+      "learning_rate": 1.6962396408873826e-05,
+      "loss": 0.8276,
+      "reward": 0.056081295013427734,
+      "reward_std": 0.00998476523818681,
+      "rewards/reward_func_1": 0.056081295013427734,
+      "step": 1225
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.33068960881838955,
+      "grad_norm": 0.002698215888813138,
+      "kl": 20.396875,
+      "learning_rate": 1.6928632005314983e-05,
+      "loss": 0.8162,
+      "reward": 0.05358821749687195,
+      "reward_std": 0.010563099296268775,
+      "rewards/reward_func_1": 0.05358821749687195,
+      "step": 1230
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.33203387552090335,
+      "grad_norm": 0.004789648577570915,
+      "kl": 20.478125,
+      "learning_rate": 1.689471499455482e-05,
+      "loss": 0.8191,
+      "reward": 0.05402927398681641,
+      "reward_std": 0.009172404053242645,
+      "rewards/reward_func_1": 0.05402927398681641,
+      "step": 1235
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3333781422234171,
+      "grad_norm": 0.010540174320340157,
+      "kl": 20.75625,
+      "learning_rate": 1.6860646123635482e-05,
+      "loss": 0.8302,
+      "reward": 0.05363121032714844,
+      "reward_std": 0.010847341820772271,
+      "rewards/reward_func_1": 0.05363121032714844,
+      "step": 1240
+    },
+    {
+      "completion_length": 2.01875,
+      "epoch": 0.3347224089259309,
+      "grad_norm": 0.8453480005264282,
+      "kl": 20.55625,
+      "learning_rate": 1.6826426142943925e-05,
+      "loss": 0.8223,
+      "reward": 0.05549154281616211,
+      "reward_std": 0.010959918500157073,
+      "rewards/reward_func_1": 0.05549154281616211,
+      "step": 1245
+    },
+    {
+      "completion_length": 2.65625,
+      "epoch": 0.33606667562844467,
+      "grad_norm": 0.7238678932189941,
+      "kl": 16.04375,
+      "learning_rate": 1.679205580619538e-05,
+      "loss": 0.6421,
+      "reward": 0.060180139541625974,
+      "reward_std": 0.012135649514675606,
+      "rewards/reward_func_1": 0.060180139541625974,
+      "step": 1250
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3374109423309585,
+      "grad_norm": 0.06061722710728645,
+      "kl": 15.15,
+      "learning_rate": 1.6757535870416755e-05,
+      "loss": 0.6056,
+      "reward": 0.06041898727416992,
+      "reward_std": 0.010352135712309973,
+      "rewards/reward_func_1": 0.06041898727416992,
+      "step": 1255
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3387552090334722,
+      "grad_norm": 0.6215311884880066,
+      "kl": 14.9609375,
+      "learning_rate": 1.6722867095929976e-05,
+      "loss": 0.5983,
+      "reward": 0.05601742267608643,
+      "reward_std": 0.010467067039280664,
+      "rewards/reward_func_1": 0.05601742267608643,
+      "step": 1260
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.34009947573598603,
+      "grad_norm": 0.005102403461933136,
+      "kl": 13.65625,
+      "learning_rate": 1.6688050246335216e-05,
+      "loss": 0.5462,
+      "reward": 0.05668201446533203,
+      "reward_std": 0.011318438543821686,
+      "rewards/reward_func_1": 0.05668201446533203,
+      "step": 1265
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3414437424384998,
+      "grad_norm": 0.05287908762693405,
+      "kl": 13.490625,
+      "learning_rate": 1.6653086088494106e-05,
+      "loss": 0.5396,
+      "reward": 0.05806665420532227,
+      "reward_std": 0.011723793356213718,
+      "rewards/reward_func_1": 0.05806665420532227,
+      "step": 1270
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3427880091410136,
+      "grad_norm": 0.00010848957026610151,
+      "kl": 13.9109375,
+      "learning_rate": 1.6617975392512812e-05,
+      "loss": 0.5563,
+      "reward": 0.06332006454467773,
+      "reward_std": 0.012813021524925717,
+      "rewards/reward_func_1": 0.06332006454467773,
+      "step": 1275
+    },
+    {
+      "completion_length": 1.9875,
+      "epoch": 0.34413227584352735,
+      "grad_norm": 0.0001873042929219082,
+      "kl": 13.890625,
+      "learning_rate": 1.6582718931725094e-05,
+      "loss": 0.5556,
+      "reward": 0.05860910415649414,
+      "reward_std": 0.012335632972826716,
+      "rewards/reward_func_1": 0.05860910415649414,
+      "step": 1280
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.34547654254604115,
+      "grad_norm": 6.566791398654459e-06,
+      "kl": 13.903125,
+      "learning_rate": 1.6547317482675277e-05,
+      "loss": 0.5563,
+      "reward": 0.05752272605895996,
+      "reward_std": 0.010450741471140645,
+      "rewards/reward_func_1": 0.05752272605895996,
+      "step": 1285
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3468208092485549,
+      "grad_norm": 1.5863972748775268e-06,
+      "kl": 13.7296875,
+      "learning_rate": 1.651177182510112e-05,
+      "loss": 0.5491,
+      "reward": 0.054108810424804685,
+      "reward_std": 0.010263701246731215,
+      "rewards/reward_func_1": 0.054108810424804685,
+      "step": 1290
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3481650759510687,
+      "grad_norm": 3.900764113495825e-06,
+      "kl": 13.80625,
+      "learning_rate": 1.6476082741916677e-05,
+      "loss": 0.5522,
+      "reward": 0.06382217407226562,
+      "reward_std": 0.01203576557818451,
+      "rewards/reward_func_1": 0.06382217407226562,
+      "step": 1295
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.34950934265358247,
+      "grad_norm": 2.0937131921527907e-06,
+      "kl": 13.7046875,
+      "learning_rate": 1.644025101919503e-05,
+      "loss": 0.5484,
+      "reward": 0.05921125411987305,
+      "reward_std": 0.011942052780796075,
+      "rewards/reward_func_1": 0.05921125411987305,
+      "step": 1300
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3508536093560963,
+      "grad_norm": 1.797810909920372e-06,
+      "kl": 13.8578125,
+      "learning_rate": 1.6404277446150968e-05,
+      "loss": 0.5542,
+      "reward": 0.061875534057617185,
+      "reward_std": 0.010323166584566935,
+      "rewards/reward_func_1": 0.061875534057617185,
+      "step": 1305
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.35219787605861,
+      "grad_norm": 4.8423517000628635e-06,
+      "kl": 13.696875,
+      "learning_rate": 1.6368162815123637e-05,
+      "loss": 0.5476,
+      "reward": 0.05844669342041016,
+      "reward_std": 0.01120743685751222,
+      "rewards/reward_func_1": 0.05844669342041016,
+      "step": 1310
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.35354214276112383,
+      "grad_norm": 1.9106237232335843e-05,
+      "kl": 13.6734375,
+      "learning_rate": 1.633190792155906e-05,
+      "loss": 0.5468,
+      "reward": 0.05963554382324219,
+      "reward_std": 0.011439791695011081,
+      "rewards/reward_func_1": 0.05963554382324219,
+      "step": 1315
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3548864094636376,
+      "grad_norm": 3.11785652229446e-06,
+      "kl": 13.5453125,
+      "learning_rate": 1.629551356399262e-05,
+      "loss": 0.5419,
+      "reward": 0.06005144119262695,
+      "reward_std": 0.00977201181158307,
+      "rewards/reward_func_1": 0.06005144119262695,
+      "step": 1320
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.35623067616615134,
+      "grad_norm": 3.1484196370001882e-06,
+      "kl": 13.89375,
+      "learning_rate": 1.625898054403148e-05,
+      "loss": 0.5557,
+      "reward": 0.06197786331176758,
+      "reward_std": 0.010603644404909573,
+      "rewards/reward_func_1": 0.06197786331176758,
+      "step": 1325
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.35757494286866515,
+      "grad_norm": 2.623250566102797e-06,
+      "kl": 14.0703125,
+      "learning_rate": 1.6222309666336933e-05,
+      "loss": 0.5626,
+      "reward": 0.06794366836547852,
+      "reward_std": 0.01082740986457793,
+      "rewards/reward_func_1": 0.06794366836547852,
+      "step": 1330
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3589192095711789,
+      "grad_norm": 1.4281185940490104e-06,
+      "kl": 13.784375,
+      "learning_rate": 1.6185501738606654e-05,
+      "loss": 0.5515,
+      "reward": 0.05785388946533203,
+      "reward_std": 0.009857135304082476,
+      "rewards/reward_func_1": 0.05785388946533203,
+      "step": 1335
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3602634762736927,
+      "grad_norm": 1.23358740893309e-05,
+      "kl": 13.928125,
+      "learning_rate": 1.614855757155693e-05,
+      "loss": 0.5574,
+      "reward": 0.061734676361083984,
+      "reward_std": 0.012116704345680773,
+      "rewards/reward_func_1": 0.061734676361083984,
+      "step": 1340
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.36160774297620646,
+      "grad_norm": 3.1937454423314193e-06,
+      "kl": 13.4515625,
+      "learning_rate": 1.6111477978904813e-05,
+      "loss": 0.5378,
+      "reward": 0.05473334789276123,
+      "reward_std": 0.010365012554575514,
+      "rewards/reward_func_1": 0.05473334789276123,
+      "step": 1345
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.36295200967872027,
+      "grad_norm": 2.7216408398089698e-06,
+      "kl": 13.628125,
+      "learning_rate": 1.6074263777350167e-05,
+      "loss": 0.5452,
+      "reward": 0.0586578369140625,
+      "reward_std": 0.010139925488329028,
+      "rewards/reward_func_1": 0.0586578369140625,
+      "step": 1350
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.364296276381234,
+      "grad_norm": 1.4683068911836017e-06,
+      "kl": 14.2203125,
+      "learning_rate": 1.6036915786557705e-05,
+      "loss": 0.569,
+      "reward": 0.057494735717773436,
+      "reward_std": 0.00982674182887422,
+      "rewards/reward_func_1": 0.057494735717773436,
+      "step": 1355
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3656405430837478,
+      "grad_norm": 2.2047534002922475e-06,
+      "kl": 13.690625,
+      "learning_rate": 1.5999434829138923e-05,
+      "loss": 0.5477,
+      "reward": 0.058840179443359376,
+      "reward_std": 0.009708692382264416,
+      "rewards/reward_func_1": 0.058840179443359376,
+      "step": 1360
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3669848097862616,
+      "grad_norm": 8.66782011144096e-06,
+      "kl": 13.884375,
+      "learning_rate": 1.5961821730633986e-05,
+      "loss": 0.5552,
+      "reward": 0.06289253234863282,
+      "reward_std": 0.013519753767468501,
+      "rewards/reward_func_1": 0.06289253234863282,
+      "step": 1365
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3683290764887754,
+      "grad_norm": 2.6345257992943516e-06,
+      "kl": 13.71875,
+      "learning_rate": 1.5924077319493546e-05,
+      "loss": 0.5486,
+      "reward": 0.05802221298217773,
+      "reward_std": 0.010079689413032611,
+      "rewards/reward_func_1": 0.05802221298217773,
+      "step": 1370
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.36967334319128914,
+      "grad_norm": 1.2998112651985139e-05,
+      "kl": 13.8421875,
+      "learning_rate": 1.5886202427060493e-05,
+      "loss": 0.5539,
+      "reward": 0.06998028755187988,
+      "reward_std": 0.011192185156687628,
+      "rewards/reward_func_1": 0.06998028755187988,
+      "step": 1375
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.37101760989380295,
+      "grad_norm": 5.262471859168727e-06,
+      "kl": 13.78125,
+      "learning_rate": 1.5848197887551643e-05,
+      "loss": 0.5507,
+      "reward": 0.05722208023071289,
+      "reward_std": 0.010396837347070687,
+      "rewards/reward_func_1": 0.05722208023071289,
+      "step": 1380
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3723618765963167,
+      "grad_norm": 3.908398866769858e-06,
+      "kl": 13.89375,
+      "learning_rate": 1.5810064538039368e-05,
+      "loss": 0.556,
+      "reward": 0.059538209438323976,
+      "reward_std": 0.010347902441571933,
+      "rewards/reward_func_1": 0.059538209438323976,
+      "step": 1385
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3737061432988305,
+      "grad_norm": 8.753636393521447e-06,
+      "kl": 14.0859375,
+      "learning_rate": 1.577180321843315e-05,
+      "loss": 0.5638,
+      "reward": 0.06107792854309082,
+      "reward_std": 0.010959995364828501,
+      "rewards/reward_func_1": 0.06107792854309082,
+      "step": 1390
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.37505041000134426,
+      "grad_norm": 1.4902374232406146e-06,
+      "kl": 13.696875,
+      "learning_rate": 1.5733414771461094e-05,
+      "loss": 0.5476,
+      "reward": 0.06554374694824219,
+      "reward_std": 0.012540119105688063,
+      "rewards/reward_func_1": 0.06554374694824219,
+      "step": 1395
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.37639467670385807,
+      "grad_norm": 3.1805816433916334e-06,
+      "kl": 13.6890625,
+      "learning_rate": 1.569490004265136e-05,
+      "loss": 0.5474,
+      "reward": 0.06188135147094727,
+      "reward_std": 0.008616514109598938,
+      "rewards/reward_func_1": 0.06188135147094727,
+      "step": 1400
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3777389434063718,
+      "grad_norm": 1.402103134751087e-05,
+      "kl": 13.7421875,
+      "learning_rate": 1.5656259880313528e-05,
+      "loss": 0.5496,
+      "reward": 0.06100940704345703,
+      "reward_std": 0.010816287656780332,
+      "rewards/reward_func_1": 0.06100940704345703,
+      "step": 1405
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3790832101088856,
+      "grad_norm": 2.776348765110015e-06,
+      "kl": 13.56875,
+      "learning_rate": 1.5617495135519946e-05,
+      "loss": 0.5429,
+      "reward": 0.05631539821624756,
+      "reward_std": 0.012434210258652456,
+      "rewards/reward_func_1": 0.05631539821624756,
+      "step": 1410
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3804274768113994,
+      "grad_norm": 2.385505240454222e-06,
+      "kl": 14.0,
+      "learning_rate": 1.557860666208695e-05,
+      "loss": 0.56,
+      "reward": 0.05535392761230469,
+      "reward_std": 0.01153669813356828,
+      "rewards/reward_func_1": 0.05535392761230469,
+      "step": 1415
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3817717435139132,
+      "grad_norm": 6.945092536625452e-06,
+      "kl": 13.7421875,
+      "learning_rate": 1.553959531655607e-05,
+      "loss": 0.5495,
+      "reward": 0.061875534057617185,
+      "reward_std": 0.011464458813861711,
+      "rewards/reward_func_1": 0.061875534057617185,
+      "step": 1420
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.38311601021642694,
+      "grad_norm": 1.5258659004757646e-05,
+      "kl": 13.609375,
+      "learning_rate": 1.5500461958175174e-05,
+      "loss": 0.5442,
+      "reward": 0.05548095703125,
+      "reward_std": 0.0085141017458227,
+      "rewards/reward_func_1": 0.05548095703125,
+      "step": 1425
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3844602769189407,
+      "grad_norm": 3.450870644883253e-05,
+      "kl": 13.884375,
+      "learning_rate": 1.546120744887954e-05,
+      "loss": 0.5551,
+      "reward": 0.05991678237915039,
+      "reward_std": 0.011126938453890034,
+      "rewards/reward_func_1": 0.05991678237915039,
+      "step": 1430
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3858045436214545,
+      "grad_norm": 4.157363036938477e-06,
+      "kl": 13.8671875,
+      "learning_rate": 1.5421832653272845e-05,
+      "loss": 0.5547,
+      "reward": 0.05983428955078125,
+      "reward_std": 0.009211386787137598,
+      "rewards/reward_func_1": 0.05983428955078125,
+      "step": 1435
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.38714881032396825,
+      "grad_norm": 3.8689913708367385e-06,
+      "kl": 13.85625,
+      "learning_rate": 1.5382338438608165e-05,
+      "loss": 0.5545,
+      "reward": 0.06216297149658203,
+      "reward_std": 0.009720365148677957,
+      "rewards/reward_func_1": 0.06216297149658203,
+      "step": 1440
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.38849307702648206,
+      "grad_norm": 3.0080229862505803e-06,
+      "kl": 13.74375,
+      "learning_rate": 1.5342725674768844e-05,
+      "loss": 0.5499,
+      "reward": 0.06219477653503418,
+      "reward_std": 0.010860501191928051,
+      "rewards/reward_func_1": 0.06219477653503418,
+      "step": 1445
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3898373437289958,
+      "grad_norm": 2.156152959287283e-06,
+      "kl": 13.659375,
+      "learning_rate": 1.5302995234249335e-05,
+      "loss": 0.5464,
+      "reward": 0.05769138336181641,
+      "reward_std": 0.011336278253293131,
+      "rewards/reward_func_1": 0.05769138336181641,
+      "step": 1450
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3911816104315096,
+      "grad_norm": 3.143540379824117e-05,
+      "kl": 13.565625,
+      "learning_rate": 1.5263147992135998e-05,
+      "loss": 0.5427,
+      "reward": 0.057453060150146486,
+      "reward_std": 0.011146099481265992,
+      "rewards/reward_func_1": 0.057453060150146486,
+      "step": 1455
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.39252587713402337,
+      "grad_norm": 2.1904502318648156e-06,
+      "kl": 13.6375,
+      "learning_rate": 1.5223184826087811e-05,
+      "loss": 0.5455,
+      "reward": 0.060272598266601564,
+      "reward_std": 0.012035325131728314,
+      "rewards/reward_func_1": 0.060272598266601564,
+      "step": 1460
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3938701438365372,
+      "grad_norm": 3.4340512229391607e-06,
+      "kl": 13.9953125,
+      "learning_rate": 1.5183106616317048e-05,
+      "loss": 0.5596,
+      "reward": 0.06144716739654541,
+      "reward_std": 0.013058099864429096,
+      "rewards/reward_func_1": 0.06144716739654541,
+      "step": 1465
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.39521441053905093,
+      "grad_norm": 2.2754304609406972e-06,
+      "kl": 14.1265625,
+      "learning_rate": 1.5142914245569885e-05,
+      "loss": 0.5651,
+      "reward": 0.057547581195831296,
+      "reward_std": 0.009754268628603313,
+      "rewards/reward_func_1": 0.057547581195831296,
+      "step": 1470
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.39655867724156474,
+      "grad_norm": 2.4745954760874156e-06,
+      "kl": 13.73125,
+      "learning_rate": 1.5102608599106966e-05,
+      "loss": 0.5491,
+      "reward": 0.061440467834472656,
+      "reward_std": 0.010538783113224781,
+      "rewards/reward_func_1": 0.061440467834472656,
+      "step": 1475
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3979029439440785,
+      "grad_norm": 6.128909262770321e-06,
+      "kl": 13.5640625,
+      "learning_rate": 1.5062190564683893e-05,
+      "loss": 0.5427,
+      "reward": 0.057086181640625,
+      "reward_std": 0.01106494044579449,
+      "rewards/reward_func_1": 0.057086181640625,
+      "step": 1480
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.3992472106465923,
+      "grad_norm": 1.8235305105918087e-06,
+      "kl": 13.7984375,
+      "learning_rate": 1.5021661032531692e-05,
+      "loss": 0.552,
+      "reward": 0.058293724060058595,
+      "reward_std": 0.010370958992280067,
+      "rewards/reward_func_1": 0.058293724060058595,
+      "step": 1485
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.40059147734910605,
+      "grad_norm": 2.1967098291497678e-06,
+      "kl": 13.8296875,
+      "learning_rate": 1.4981020895337175e-05,
+      "loss": 0.5532,
+      "reward": 0.05586849227547645,
+      "reward_std": 0.011479038602556103,
+      "rewards/reward_func_1": 0.05586849227547645,
+      "step": 1490
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.40193574405161986,
+      "grad_norm": 1.2756601108776522e-06,
+      "kl": 13.4796875,
+      "learning_rate": 1.4940271048223307e-05,
+      "loss": 0.5394,
+      "reward": 0.05812692642211914,
+      "reward_std": 0.012107894703513011,
+      "rewards/reward_func_1": 0.05812692642211914,
+      "step": 1495
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4032800107541336,
+      "grad_norm": 1.9987196537840646e-06,
+      "kl": 13.553125,
+      "learning_rate": 1.4899412388729472e-05,
+      "loss": 0.5421,
+      "reward": 0.051792049407958986,
+      "reward_std": 0.01174548725830391,
+      "rewards/reward_func_1": 0.051792049407958986,
+      "step": 1500
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4046242774566474,
+      "grad_norm": 1.8275987940796767e-06,
+      "kl": 13.9421875,
+      "learning_rate": 1.4858445816791718e-05,
+      "loss": 0.5575,
+      "reward": 0.05959300994873047,
+      "reward_std": 0.009343751921551301,
+      "rewards/reward_func_1": 0.05959300994873047,
+      "step": 1505
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.40596854415916117,
+      "grad_norm": 5.036072707298445e-06,
+      "kl": 13.7765625,
+      "learning_rate": 1.4817372234722918e-05,
+      "loss": 0.551,
+      "reward": 0.06010627746582031,
+      "reward_std": 0.01253571416818886,
+      "rewards/reward_func_1": 0.06010627746582031,
+      "step": 1510
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.407312810861675,
+      "grad_norm": 2.4326691345777363e-06,
+      "kl": 14.025,
+      "learning_rate": 1.4776192547192915e-05,
+      "loss": 0.5612,
+      "reward": 0.06703472137451172,
+      "reward_std": 0.010542747608269565,
+      "rewards/reward_func_1": 0.06703472137451172,
+      "step": 1515
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.40865707756418873,
+      "grad_norm": 2.35791821978637e-06,
+      "kl": 13.6375,
+      "learning_rate": 1.4734907661208587e-05,
+      "loss": 0.5454,
+      "reward": 0.05951080322265625,
+      "reward_std": 0.009829605106278904,
+      "rewards/reward_func_1": 0.05951080322265625,
+      "step": 1520
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.41000134426670254,
+      "grad_norm": 4.144026206631679e-06,
+      "kl": 13.9828125,
+      "learning_rate": 1.469351848609386e-05,
+      "loss": 0.5595,
+      "reward": 0.05874214172363281,
+      "reward_std": 0.008459481771024003,
+      "rewards/reward_func_1": 0.05874214172363281,
+      "step": 1525
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4113456109692163,
+      "grad_norm": 4.286873263481539e-06,
+      "kl": 13.88125,
+      "learning_rate": 1.4652025933469705e-05,
+      "loss": 0.5551,
+      "reward": 0.06733989715576172,
+      "reward_std": 0.01017450345098041,
+      "rewards/reward_func_1": 0.06733989715576172,
+      "step": 1530
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4126898776717301,
+      "grad_norm": 6.012166977598099e-06,
+      "kl": 13.8875,
+      "learning_rate": 1.461043091723403e-05,
+      "loss": 0.5554,
+      "reward": 0.06552686691284179,
+      "reward_std": 0.010358074885152746,
+      "rewards/reward_func_1": 0.06552686691284179,
+      "step": 1535
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.41403414437424385,
+      "grad_norm": 2.3474919998989208e-06,
+      "kl": 13.9390625,
+      "learning_rate": 1.4568734353541572e-05,
+      "loss": 0.5574,
+      "reward": 0.058895301818847653,
+      "reward_std": 0.012217024579877033,
+      "rewards/reward_func_1": 0.058895301818847653,
+      "step": 1540
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4153784110767576,
+      "grad_norm": 3.684157127281651e-05,
+      "kl": 13.8734375,
+      "learning_rate": 1.4526937160783707e-05,
+      "loss": 0.555,
+      "reward": 0.05571174621582031,
+      "reward_std": 0.009185398211957362,
+      "rewards/reward_func_1": 0.05571174621582031,
+      "step": 1545
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4167226777792714,
+      "grad_norm": 1.638373532841797e-06,
+      "kl": 14.0515625,
+      "learning_rate": 1.4485040259568228e-05,
+      "loss": 0.5622,
+      "reward": 0.06209487915039062,
+      "reward_std": 0.010831043922371464,
+      "rewards/reward_func_1": 0.06209487915039062,
+      "step": 1550
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.41806694448178516,
+      "grad_norm": 1.416817667632131e-06,
+      "kl": 13.9046875,
+      "learning_rate": 1.4443044572699058e-05,
+      "loss": 0.556,
+      "reward": 0.06337127685546876,
+      "reward_std": 0.00729374265865772,
+      "rewards/reward_func_1": 0.06337127685546876,
+      "step": 1555
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.41941121118429897,
+      "grad_norm": 4.485429144551745e-06,
+      "kl": 13.6140625,
+      "learning_rate": 1.440095102515595e-05,
+      "loss": 0.5445,
+      "reward": 0.060857629776000975,
+      "reward_std": 0.011825820308149559,
+      "rewards/reward_func_1": 0.060857629776000975,
+      "step": 1560
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4207554778868127,
+      "grad_norm": 2.9214272672106745e-06,
+      "kl": 14.034375,
+      "learning_rate": 1.4358760544074074e-05,
+      "loss": 0.5612,
+      "reward": 0.06148242950439453,
+      "reward_std": 0.012079593736416427,
+      "rewards/reward_func_1": 0.06148242950439453,
+      "step": 1565
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.42209974458932653,
+      "grad_norm": 2.7933485853282036e-06,
+      "kl": 13.725,
+      "learning_rate": 1.4316474058723635e-05,
+      "loss": 0.549,
+      "reward": 0.06508445739746094,
+      "reward_std": 0.009462902668019524,
+      "rewards/reward_func_1": 0.06508445739746094,
+      "step": 1570
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4234440112918403,
+      "grad_norm": 1.132559646066511e-05,
+      "kl": 13.465625,
+      "learning_rate": 1.4274092500489376e-05,
+      "loss": 0.5386,
+      "reward": 0.06119532585144043,
+      "reward_std": 0.010296352157456567,
+      "rewards/reward_func_1": 0.06119532585144043,
+      "step": 1575
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4247882779943541,
+      "grad_norm": 3.3802455163822742e-06,
+      "kl": 13.753125,
+      "learning_rate": 1.423161680285009e-05,
+      "loss": 0.55,
+      "reward": 0.05784816741943359,
+      "reward_std": 0.011395523198734736,
+      "rewards/reward_func_1": 0.05784816741943359,
+      "step": 1580
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.42613254469686784,
+      "grad_norm": 3.657454954009154e-06,
+      "kl": 13.796875,
+      "learning_rate": 1.4189047901358033e-05,
+      "loss": 0.5516,
+      "reward": 0.0637430191040039,
+      "reward_std": 0.012986108286713715,
+      "rewards/reward_func_1": 0.0637430191040039,
+      "step": 1585
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.42747681139938165,
+      "grad_norm": 2.5834062853391515e-06,
+      "kl": 13.596875,
+      "learning_rate": 1.4146386733618338e-05,
+      "loss": 0.5439,
+      "reward": 0.059173583984375,
+      "reward_std": 0.011515995301306248,
+      "rewards/reward_func_1": 0.059173583984375,
+      "step": 1590
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4288210781018954,
+      "grad_norm": 4.3796212594315875e-06,
+      "kl": 13.8921875,
+      "learning_rate": 1.4103634239268355e-05,
+      "loss": 0.5556,
+      "reward": 0.06446866989135742,
+      "reward_std": 0.009868808073224499,
+      "rewards/reward_func_1": 0.06446866989135742,
+      "step": 1595
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4301653448044092,
+      "grad_norm": 8.73978751769755e-06,
+      "kl": 14.025,
+      "learning_rate": 1.4060791359956956e-05,
+      "loss": 0.5611,
+      "reward": 0.0633920669555664,
+      "reward_std": 0.01284162982410635,
+      "rewards/reward_func_1": 0.0633920669555664,
+      "step": 1600
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.43150961150692296,
+      "grad_norm": 3.975842446379829e-06,
+      "kl": 13.9109375,
+      "learning_rate": 1.401785903932379e-05,
+      "loss": 0.5564,
+      "reward": 0.06275310516357421,
+      "reward_std": 0.011346189048344968,
+      "rewards/reward_func_1": 0.06275310516357421,
+      "step": 1605
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.43285387820943677,
+      "grad_norm": 2.2908070604898967e-06,
+      "kl": 13.628125,
+      "learning_rate": 1.3974838222978517e-05,
+      "loss": 0.5454,
+      "reward": 0.06408562660217285,
+      "reward_std": 0.011092856073810253,
+      "rewards/reward_func_1": 0.06408562660217285,
+      "step": 1610
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4341981449119505,
+      "grad_norm": 4.72618330604746e-06,
+      "kl": 14.3875,
+      "learning_rate": 1.3931729858479954e-05,
+      "loss": 0.5759,
+      "reward": 0.06461887359619141,
+      "reward_std": 0.009859557841264178,
+      "rewards/reward_func_1": 0.06461887359619141,
+      "step": 1615
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.43554241161446433,
+      "grad_norm": 3.2984760309773264e-06,
+      "kl": 14.025,
+      "learning_rate": 1.3888534895315222e-05,
+      "loss": 0.561,
+      "reward": 0.06250219345092774,
+      "reward_std": 0.01138484149123542,
+      "rewards/reward_func_1": 0.06250219345092774,
+      "step": 1620
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4368866783169781,
+      "grad_norm": 1.7083860939237638e-06,
+      "kl": 13.5921875,
+      "learning_rate": 1.384525428487883e-05,
+      "loss": 0.5439,
+      "reward": 0.057589149475097655,
+      "reward_std": 0.012263275221630465,
+      "rewards/reward_func_1": 0.057589149475097655,
+      "step": 1625
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4382309450194919,
+      "grad_norm": 1.8176706362282857e-05,
+      "kl": 13.703125,
+      "learning_rate": 1.380188898045172e-05,
+      "loss": 0.5484,
+      "reward": 0.05926389694213867,
+      "reward_std": 0.011890075955307111,
+      "rewards/reward_func_1": 0.05926389694213867,
+      "step": 1630
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.43957521172200564,
+      "grad_norm": 1.6534449969185516e-06,
+      "kl": 13.7234375,
+      "learning_rate": 1.3758439937180269e-05,
+      "loss": 0.5489,
+      "reward": 0.06111717224121094,
+      "reward_std": 0.009865944929333636,
+      "rewards/reward_func_1": 0.06111717224121094,
+      "step": 1635
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.44091947842451945,
+      "grad_norm": 5.1437118600006215e-06,
+      "kl": 13.85625,
+      "learning_rate": 1.371490811205524e-05,
+      "loss": 0.554,
+      "reward": 0.06157407760620117,
+      "reward_std": 0.012079483611159958,
+      "rewards/reward_func_1": 0.06157407760620117,
+      "step": 1640
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4422637451270332,
+      "grad_norm": 5.265788786346093e-06,
+      "kl": 13.75,
+      "learning_rate": 1.3671294463890734e-05,
+      "loss": 0.5499,
+      "reward": 0.057445335388183597,
+      "reward_std": 0.01464254588354379,
+      "rewards/reward_func_1": 0.057445335388183597,
+      "step": 1645
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.44360801182954696,
+      "grad_norm": 1.6486019376316108e-05,
+      "kl": 14.0984375,
+      "learning_rate": 1.3627599953303036e-05,
+      "loss": 0.5636,
+      "reward": 0.062333667278289796,
+      "reward_std": 0.00997068356446107,
+      "rewards/reward_func_1": 0.062333667278289796,
+      "step": 1650
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.44495227853206076,
+      "grad_norm": 1.7718589333526324e-06,
+      "kl": 13.64375,
+      "learning_rate": 1.3583825542689486e-05,
+      "loss": 0.5456,
+      "reward": 0.05308668613433838,
+      "reward_std": 0.01278767061594408,
+      "rewards/reward_func_1": 0.05308668613433838,
+      "step": 1655
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4462965452345745,
+      "grad_norm": 5.0791340981959365e-06,
+      "kl": 14.1375,
+      "learning_rate": 1.353997219620726e-05,
+      "loss": 0.5657,
+      "reward": 0.06480164527893066,
+      "reward_std": 0.010595990939327749,
+      "rewards/reward_func_1": 0.06480164527893066,
+      "step": 1660
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4476408119370883,
+      "grad_norm": 1.975413169930107e-06,
+      "kl": 13.9046875,
+      "learning_rate": 1.3496040879752146e-05,
+      "loss": 0.5562,
+      "reward": 0.058099555969238284,
+      "reward_std": 0.012097873717721086,
+      "rewards/reward_func_1": 0.058099555969238284,
+      "step": 1665
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4489850786396021,
+      "grad_norm": 3.103926246694755e-06,
+      "kl": 14.0609375,
+      "learning_rate": 1.3452032560937271e-05,
+      "loss": 0.5626,
+      "reward": 0.06475410461425782,
+      "reward_std": 0.01139398144750885,
+      "rewards/reward_func_1": 0.06475410461425782,
+      "step": 1670
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4503293453421159,
+      "grad_norm": 1.6186576203836012e-06,
+      "kl": 14.034375,
+      "learning_rate": 1.3407948209071779e-05,
+      "loss": 0.5614,
+      "reward": 0.06743978261947632,
+      "reward_std": 0.013266765065782237,
+      "rewards/reward_func_1": 0.06743978261947632,
+      "step": 1675
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.45167361204462964,
+      "grad_norm": 1.8007198377745226e-05,
+      "kl": 14.275,
+      "learning_rate": 1.3363788795139487e-05,
+      "loss": 0.571,
+      "reward": 0.06518707275390626,
+      "reward_std": 0.011862215257133357,
+      "rewards/reward_func_1": 0.06518707275390626,
+      "step": 1680
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.45301787874714344,
+      "grad_norm": 5.033749403082766e-06,
+      "kl": 13.915625,
+      "learning_rate": 1.3319555291777501e-05,
+      "loss": 0.5568,
+      "reward": 0.06435184478759766,
+      "reward_std": 0.011379225243217661,
+      "rewards/reward_func_1": 0.06435184478759766,
+      "step": 1685
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4543621454496572,
+      "grad_norm": 6.873391612316482e-06,
+      "kl": 13.596875,
+      "learning_rate": 1.3275248673254788e-05,
+      "loss": 0.544,
+      "reward": 0.06184234619140625,
+      "reward_std": 0.010739423423365224,
+      "rewards/reward_func_1": 0.06184234619140625,
+      "step": 1690
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.455706412152171,
+      "grad_norm": 4.609265033650445e-06,
+      "kl": 13.6984375,
+      "learning_rate": 1.3230869915450722e-05,
+      "loss": 0.5481,
+      "reward": 0.05451488494873047,
+      "reward_std": 0.010388137760855898,
+      "rewards/reward_func_1": 0.05451488494873047,
+      "step": 1695
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.45705067885468476,
+      "grad_norm": 2.1986843421473168e-06,
+      "kl": 13.5859375,
+      "learning_rate": 1.3186419995833582e-05,
+      "loss": 0.5436,
+      "reward": 0.05490055084228516,
+      "reward_std": 0.011986211253679357,
+      "rewards/reward_func_1": 0.05490055084228516,
+      "step": 1700
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.45839494555719856,
+      "grad_norm": 3.9593110159330536e-06,
+      "kl": 13.828125,
+      "learning_rate": 1.3141899893439032e-05,
+      "loss": 0.5533,
+      "reward": 0.061890792846679685,
+      "reward_std": 0.01013331833673874,
+      "rewards/reward_func_1": 0.061890792846679685,
+      "step": 1705
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4597392122597123,
+      "grad_norm": 2.6946911475533852e-06,
+      "kl": 14.0953125,
+      "learning_rate": 1.3097310588848555e-05,
+      "loss": 0.5641,
+      "reward": 0.06313896179199219,
+      "reward_std": 0.013185867536230944,
+      "rewards/reward_func_1": 0.06313896179199219,
+      "step": 1710
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4610834789622261,
+      "grad_norm": 3.4700431115197716e-06,
+      "kl": 13.9703125,
+      "learning_rate": 1.3052653064167848e-05,
+      "loss": 0.5591,
+      "reward": 0.057857322692871097,
+      "reward_std": 0.01133649832190713,
+      "rewards/reward_func_1": 0.057857322692871097,
+      "step": 1715
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4624277456647399,
+      "grad_norm": 1.3458391549647786e-05,
+      "kl": 13.5859375,
+      "learning_rate": 1.3007928303005201e-05,
+      "loss": 0.5436,
+      "reward": 0.05681304931640625,
+      "reward_std": 0.010392763031995855,
+      "rewards/reward_func_1": 0.05681304931640625,
+      "step": 1720
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4637720123672537,
+      "grad_norm": 4.882398570771329e-06,
+      "kl": 13.8015625,
+      "learning_rate": 1.2963137290449823e-05,
+      "loss": 0.552,
+      "reward": 0.062281131744384766,
+      "reward_std": 0.012929836504918057,
+      "rewards/reward_func_1": 0.062281131744384766,
+      "step": 1725
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.46511627906976744,
+      "grad_norm": 6.2865415202395525e-06,
+      "kl": 13.8671875,
+      "learning_rate": 1.291828101305015e-05,
+      "loss": 0.5546,
+      "reward": 0.061757802963256836,
+      "reward_std": 0.0114550436315767,
+      "rewards/reward_func_1": 0.061757802963256836,
+      "step": 1730
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.46646054577228124,
+      "grad_norm": 2.4083929019980133e-06,
+      "kl": 14.290625,
+      "learning_rate": 1.2873360458792114e-05,
+      "loss": 0.5719,
+      "reward": 0.06473960876464843,
+      "reward_std": 0.012107564476900734,
+      "rewards/reward_func_1": 0.06473960876464843,
+      "step": 1735
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.467804812474795,
+      "grad_norm": 2.2838785298517905e-05,
+      "kl": 13.7671875,
+      "learning_rate": 1.2828376617077385e-05,
+      "loss": 0.5504,
+      "reward": 0.059980010986328124,
+      "reward_std": 0.00752433567395201,
+      "rewards/reward_func_1": 0.059980010986328124,
+      "step": 1740
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4691490791773088,
+      "grad_norm": 2.683865886865533e-06,
+      "kl": 13.9546875,
+      "learning_rate": 1.2783330478701572e-05,
+      "loss": 0.558,
+      "reward": 0.05912628173828125,
+      "reward_std": 0.010197188393794932,
+      "rewards/reward_func_1": 0.05912628173828125,
+      "step": 1745
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.47049334587982256,
+      "grad_norm": 2.190610530306003e-06,
+      "kl": 14.090625,
+      "learning_rate": 1.2738223035832412e-05,
+      "loss": 0.5638,
+      "reward": 0.06425952911376953,
+      "reward_std": 0.008773326113077929,
+      "rewards/reward_func_1": 0.06425952911376953,
+      "step": 1750
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.47183761258233636,
+      "grad_norm": 3.1278939331969013e-06,
+      "kl": 14.303125,
+      "learning_rate": 1.2693055281987903e-05,
+      "loss": 0.5719,
+      "reward": 0.06397314071655273,
+      "reward_std": 0.01130247107357718,
+      "rewards/reward_func_1": 0.06397314071655273,
+      "step": 1755
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4731818792848501,
+      "grad_norm": 1.2678321581915952e-05,
+      "kl": 13.93125,
+      "learning_rate": 1.264782821201443e-05,
+      "loss": 0.5571,
+      "reward": 0.057790946960449216,
+      "reward_std": 0.010541425982955844,
+      "rewards/reward_func_1": 0.057790946960449216,
+      "step": 1760
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.47452614598736387,
+      "grad_norm": 2.6648983748600585e-06,
+      "kl": 13.5921875,
+      "learning_rate": 1.2602542822064852e-05,
+      "loss": 0.5438,
+      "reward": 0.06369266510009766,
+      "reward_std": 0.010999528719548835,
+      "rewards/reward_func_1": 0.06369266510009766,
+      "step": 1765
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4758704126898777,
+      "grad_norm": 9.32496550376527e-06,
+      "kl": 14.321875,
+      "learning_rate": 1.2557200109576557e-05,
+      "loss": 0.5729,
+      "reward": 0.061204147338867185,
+      "reward_std": 0.013532968414074276,
+      "rewards/reward_func_1": 0.061204147338867185,
+      "step": 1770
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4772146793923914,
+      "grad_norm": 1.453070012757962e-06,
+      "kl": 13.70625,
+      "learning_rate": 1.2511801073249499e-05,
+      "loss": 0.5482,
+      "reward": 0.060839509963989256,
+      "reward_std": 0.01080455974151846,
+      "rewards/reward_func_1": 0.060839509963989256,
+      "step": 1775
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.47855894609490524,
+      "grad_norm": 2.337250180062256e-06,
+      "kl": 13.696875,
+      "learning_rate": 1.2466346713024194e-05,
+      "loss": 0.5479,
+      "reward": 0.05852031707763672,
+      "reward_std": 0.009971881102683256,
+      "rewards/reward_func_1": 0.05852031707763672,
+      "step": 1780
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.479903212797419,
+      "grad_norm": 2.7332425815984607e-06,
+      "kl": 13.7046875,
+      "learning_rate": 1.2420838030059704e-05,
+      "loss": 0.5481,
+      "reward": 0.059021949768066406,
+      "reward_std": 0.01221724480674311,
+      "rewards/reward_func_1": 0.059021949768066406,
+      "step": 1785
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4812474794999328,
+      "grad_norm": 2.8585989184648497e-06,
+      "kl": 13.725,
+      "learning_rate": 1.2375276026711576e-05,
+      "loss": 0.5493,
+      "reward": 0.0618377685546875,
+      "reward_std": 0.011973217026388738,
+      "rewards/reward_func_1": 0.0618377685546875,
+      "step": 1790
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.48259174620244655,
+      "grad_norm": 2.026320362347178e-05,
+      "kl": 14.0484375,
+      "learning_rate": 1.232966170650977e-05,
+      "loss": 0.5619,
+      "reward": 0.062344479560852054,
+      "reward_std": 0.009559368583722971,
+      "rewards/reward_func_1": 0.062344479560852054,
+      "step": 1795
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.48393601290496036,
+      "grad_norm": 2.701003268157365e-06,
+      "kl": 13.63125,
+      "learning_rate": 1.2283996074136566e-05,
+      "loss": 0.5452,
+      "reward": 0.06439933776855469,
+      "reward_std": 0.011508286974640214,
+      "rewards/reward_func_1": 0.06439933776855469,
+      "step": 1800
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4852802796074741,
+      "grad_norm": 3.2230218494078144e-06,
+      "kl": 13.9390625,
+      "learning_rate": 1.2238280135404411e-05,
+      "loss": 0.5573,
+      "reward": 0.06914815902709961,
+      "reward_std": 0.010676544258603825,
+      "rewards/reward_func_1": 0.06914815902709961,
+      "step": 1805
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4866245463099879,
+      "grad_norm": 4.649204583984101e-06,
+      "kl": 13.778125,
+      "learning_rate": 1.2192514897233789e-05,
+      "loss": 0.5511,
+      "reward": 0.0602226972579956,
+      "reward_std": 0.011667439006851054,
+      "rewards/reward_func_1": 0.0602226972579956,
+      "step": 1810
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.48796881301250167,
+      "grad_norm": 1.511799382569734e-06,
+      "kl": 13.9296875,
+      "learning_rate": 1.2146701367631027e-05,
+      "loss": 0.5574,
+      "reward": 0.06371002197265625,
+      "reward_std": 0.010727530277472396,
+      "rewards/reward_func_1": 0.06371002197265625,
+      "step": 1815
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4893130797150155,
+      "grad_norm": 4.017825631308369e-06,
+      "kl": 13.875,
+      "learning_rate": 1.2100840555666101e-05,
+      "loss": 0.5552,
+      "reward": 0.06100995540618896,
+      "reward_std": 0.01184462348173838,
+      "rewards/reward_func_1": 0.06100995540618896,
+      "step": 1820
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4906573464175292,
+      "grad_norm": 2.1121263671375345e-06,
+      "kl": 13.44375,
+      "learning_rate": 1.205493347145041e-05,
+      "loss": 0.5377,
+      "reward": 0.056847544759511946,
+      "reward_std": 0.013147425842907979,
+      "rewards/reward_func_1": 0.056847544759511946,
+      "step": 1825
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.49200161312004304,
+      "grad_norm": 3.4769893773045624e-06,
+      "kl": 13.3515625,
+      "learning_rate": 1.2008981126114523e-05,
+      "loss": 0.5341,
+      "reward": 0.0553741455078125,
+      "reward_std": 0.010613445060516823,
+      "rewards/reward_func_1": 0.0553741455078125,
+      "step": 1830
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4933458798225568,
+      "grad_norm": 1.7378441043547355e-05,
+      "kl": 13.7125,
+      "learning_rate": 1.1962984531785922e-05,
+      "loss": 0.5482,
+      "reward": 0.05479507446289063,
+      "reward_std": 0.009559368582631577,
+      "rewards/reward_func_1": 0.05479507446289063,
+      "step": 1835
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4946901465250706,
+      "grad_norm": 7.995906344149262e-06,
+      "kl": 13.79375,
+      "learning_rate": 1.1916944701566688e-05,
+      "loss": 0.5518,
+      "reward": 0.062036323547363284,
+      "reward_std": 0.010829502148044411,
+      "rewards/reward_func_1": 0.062036323547363284,
+      "step": 1840
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.49603441322758435,
+      "grad_norm": 5.335733931133291e-06,
+      "kl": 14.0375,
+      "learning_rate": 1.1870862649511201e-05,
+      "loss": 0.5616,
+      "reward": 0.0632176399230957,
+      "reward_std": 0.011605303342366823,
+      "rewards/reward_func_1": 0.0632176399230957,
+      "step": 1845
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.49737867993009816,
+      "grad_norm": 3.3654257549642352e-06,
+      "kl": 13.9046875,
+      "learning_rate": 1.1824739390603801e-05,
+      "loss": 0.5563,
+      "reward": 0.06469783782958985,
+      "reward_std": 0.008109517797129229,
+      "rewards/reward_func_1": 0.06469783782958985,
+      "step": 1850
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.4987229466326119,
+      "grad_norm": 3.5574796584114665e-06,
+      "kl": 13.8203125,
+      "learning_rate": 1.1778575940736439e-05,
+      "loss": 0.5526,
+      "reward": 0.05752217769622803,
+      "reward_std": 0.011466909085720544,
+      "rewards/reward_func_1": 0.05752217769622803,
+      "step": 1855
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5000672133351257,
+      "grad_norm": 4.107642325834604e-06,
+      "kl": 13.8015625,
+      "learning_rate": 1.1732373316686292e-05,
+      "loss": 0.5522,
+      "reward": 0.06678012609481812,
+      "reward_std": 0.011566609619330847,
+      "rewards/reward_func_1": 0.06678012609481812,
+      "step": 1860
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5014114800376395,
+      "grad_norm": 8.976133358373772e-06,
+      "kl": 13.6671875,
+      "learning_rate": 1.1686132536093367e-05,
+      "loss": 0.5469,
+      "reward": 0.05934562683105469,
+      "reward_std": 0.012758818920701742,
+      "rewards/reward_func_1": 0.05934562683105469,
+      "step": 1865
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5027557467401532,
+      "grad_norm": 6.34239995633834e-06,
+      "kl": 13.81875,
+      "learning_rate": 1.1639854617438098e-05,
+      "loss": 0.5528,
+      "reward": 0.05746040344238281,
+      "reward_std": 0.010571158733364427,
+      "rewards/reward_func_1": 0.05746040344238281,
+      "step": 1870
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.504100013442667,
+      "grad_norm": 5.687683824362466e-06,
+      "kl": 13.778125,
+      "learning_rate": 1.1593540580018904e-05,
+      "loss": 0.5512,
+      "reward": 0.05971870422363281,
+      "reward_std": 0.010947331442730501,
+      "rewards/reward_func_1": 0.05971870422363281,
+      "step": 1875
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5054442801451808,
+      "grad_norm": 4.094200448889751e-06,
+      "kl": 13.8390625,
+      "learning_rate": 1.1547191443929738e-05,
+      "loss": 0.5535,
+      "reward": 0.059337806701660153,
+      "reward_std": 0.010353339680295903,
+      "rewards/reward_func_1": 0.059337806701660153,
+      "step": 1880
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5067885468476946,
+      "grad_norm": 3.398595481485245e-06,
+      "kl": 13.9453125,
+      "learning_rate": 1.1500808230037628e-05,
+      "loss": 0.5578,
+      "reward": 0.05960531234741211,
+      "reward_std": 0.0105580543531687,
+      "rewards/reward_func_1": 0.05960531234741211,
+      "step": 1885
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5081328135502083,
+      "grad_norm": 6.3564093579771e-06,
+      "kl": 13.6453125,
+      "learning_rate": 1.145439195996018e-05,
+      "loss": 0.5457,
+      "reward": 0.05985393524169922,
+      "reward_std": 0.011576782021438702,
+      "rewards/reward_func_1": 0.05985393524169922,
+      "step": 1890
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5094770802527221,
+      "grad_norm": 4.291194272809662e-06,
+      "kl": 13.728125,
+      "learning_rate": 1.1407943656043088e-05,
+      "loss": 0.5492,
+      "reward": 0.062256813049316406,
+      "reward_std": 0.01353142662846949,
+      "rewards/reward_func_1": 0.062256813049316406,
+      "step": 1895
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.510821346955236,
+      "grad_norm": 1.4876853811074398e-06,
+      "kl": 13.75625,
+      "learning_rate": 1.1361464341337604e-05,
+      "loss": 0.5501,
+      "reward": 0.05925731658935547,
+      "reward_std": 0.009176148010010366,
+      "rewards/reward_func_1": 0.05925731658935547,
+      "step": 1900
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5121656136577497,
+      "grad_norm": 5.476561454997864e-06,
+      "kl": 14.0125,
+      "learning_rate": 1.1314955039578017e-05,
+      "loss": 0.5605,
+      "reward": 0.06184120178222656,
+      "reward_std": 0.010909009404713288,
+      "rewards/reward_func_1": 0.06184120178222656,
+      "step": 1905
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5135098803602635,
+      "grad_norm": 6.360010047501419e-06,
+      "kl": 13.7359375,
+      "learning_rate": 1.126841677515909e-05,
+      "loss": 0.5494,
+      "reward": 0.06381258964538575,
+      "reward_std": 0.009671746863750741,
+      "rewards/reward_func_1": 0.06381258964538575,
+      "step": 1910
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5148541470627772,
+      "grad_norm": 3.01595628116047e-06,
+      "kl": 13.8765625,
+      "learning_rate": 1.1221850573113515e-05,
+      "loss": 0.5552,
+      "reward": 0.062322235107421874,
+      "reward_std": 0.011111631775202113,
+      "rewards/reward_func_1": 0.062322235107421874,
+      "step": 1915
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5161984137652911,
+      "grad_norm": 4.450800588529091e-06,
+      "kl": 13.7265625,
+      "learning_rate": 1.117525745908932e-05,
+      "loss": 0.5491,
+      "reward": 0.06123924255371094,
+      "reward_std": 0.011204353353969054,
+      "rewards/reward_func_1": 0.06123924255371094,
+      "step": 1920
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5175426804678048,
+      "grad_norm": 6.122299964772537e-06,
+      "kl": 13.78125,
+      "learning_rate": 1.1128638459327288e-05,
+      "loss": 0.5513,
+      "reward": 0.06520743370056152,
+      "reward_std": 0.010965666610718471,
+      "rewards/reward_func_1": 0.06520743370056152,
+      "step": 1925
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5188869471703186,
+      "grad_norm": 2.6760865239339182e-06,
+      "kl": 13.5875,
+      "learning_rate": 1.1081994600638353e-05,
+      "loss": 0.5434,
+      "reward": 0.056897735595703124,
+      "reward_std": 0.009793705747870262,
+      "rewards/reward_func_1": 0.056897735595703124,
+      "step": 1930
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5202312138728323,
+      "grad_norm": 3.4807439988071565e-06,
+      "kl": 13.7859375,
+      "learning_rate": 1.1035326910380973e-05,
+      "loss": 0.5516,
+      "reward": 0.05627828128635883,
+      "reward_std": 0.011038010333140846,
+      "rewards/reward_func_1": 0.05627828128635883,
+      "step": 1935
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5215754805753462,
+      "grad_norm": 3.09952747556963e-06,
+      "kl": 13.984375,
+      "learning_rate": 1.0988636416438521e-05,
+      "loss": 0.5592,
+      "reward": 0.057964515686035153,
+      "reward_std": 0.010637010936625302,
+      "rewards/reward_func_1": 0.057964515686035153,
+      "step": 1940
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.52291974727786,
+      "grad_norm": 2.1990246750647202e-05,
+      "kl": 13.753125,
+      "learning_rate": 1.094192414719663e-05,
+      "loss": 0.5502,
+      "reward": 0.061472320556640626,
+      "reward_std": 0.010977944992919219,
+      "rewards/reward_func_1": 0.061472320556640626,
+      "step": 1945
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5242640139803737,
+      "grad_norm": 9.643837984185666e-06,
+      "kl": 13.534375,
+      "learning_rate": 1.0895191131520541e-05,
+      "loss": 0.5414,
+      "reward": 0.05615215301513672,
+      "reward_std": 0.010869586077751592,
+      "rewards/reward_func_1": 0.05615215301513672,
+      "step": 1950
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5256082806828875,
+      "grad_norm": 2.5311317131127e-06,
+      "kl": 14.0046875,
+      "learning_rate": 1.0848438398732462e-05,
+      "loss": 0.5601,
+      "reward": 0.06194038391113281,
+      "reward_std": 0.012956816235237057,
+      "rewards/reward_func_1": 0.06194038391113281,
+      "step": 1955
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5269525473854013,
+      "grad_norm": 5.817757482873276e-06,
+      "kl": 13.809375,
+      "learning_rate": 1.0801666978588865e-05,
+      "loss": 0.5522,
+      "reward": 0.06431331634521484,
+      "reward_std": 0.011656289092206862,
+      "rewards/reward_func_1": 0.06431331634521484,
+      "step": 1960
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5282968140879151,
+      "grad_norm": 2.050035391221172e-06,
+      "kl": 13.7453125,
+      "learning_rate": 1.0754877901257831e-05,
+      "loss": 0.5499,
+      "reward": 0.059846115112304685,
+      "reward_std": 0.01306429406904499,
+      "rewards/reward_func_1": 0.059846115112304685,
+      "step": 1965
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5296410807904288,
+      "grad_norm": 1.0048594958789181e-05,
+      "kl": 13.7875,
+      "learning_rate": 1.0708072197296356e-05,
+      "loss": 0.5518,
+      "reward": 0.06069736480712891,
+      "reward_std": 0.011032124502526131,
+      "rewards/reward_func_1": 0.06069736480712891,
+      "step": 1970
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5309853474929426,
+      "grad_norm": 2.601335108920466e-05,
+      "kl": 13.584375,
+      "learning_rate": 1.0661250897627634e-05,
+      "loss": 0.5436,
+      "reward": 0.053227472305297854,
+      "reward_std": 0.010092408429773058,
+      "rewards/reward_func_1": 0.053227472305297854,
+      "step": 1975
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5323296141954563,
+      "grad_norm": 1.8432465367368422e-06,
+      "kl": 13.54375,
+      "learning_rate": 1.061441503351837e-05,
+      "loss": 0.5418,
+      "reward": 0.05695056915283203,
+      "reward_std": 0.009967696487728972,
+      "rewards/reward_func_1": 0.05695056915283203,
+      "step": 1980
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5336738808979702,
+      "grad_norm": 7.412290869979188e-06,
+      "kl": 13.8828125,
+      "learning_rate": 1.056756563655607e-05,
+      "loss": 0.5549,
+      "reward": 0.0633920669555664,
+      "reward_std": 0.01274736642735661,
+      "rewards/reward_func_1": 0.0633920669555664,
+      "step": 1985
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5350181476004839,
+      "grad_norm": 4.0991371861309744e-06,
+      "kl": 13.6109375,
+      "learning_rate": 1.052070373862629e-05,
+      "loss": 0.5444,
+      "reward": 0.05980701446533203,
+      "reward_std": 0.01131843865441624,
+      "rewards/reward_func_1": 0.05980701446533203,
+      "step": 1990
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5363624143029977,
+      "grad_norm": 1.9139324649586342e-06,
+      "kl": 13.7296875,
+      "learning_rate": 1.047383037188994e-05,
+      "loss": 0.5491,
+      "reward": 0.060098457336425784,
+      "reward_std": 0.011104363739286782,
+      "rewards/reward_func_1": 0.060098457336425784,
+      "step": 1995
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5377066810055114,
+      "grad_norm": 2.1989344531903043e-05,
+      "kl": 13.8578125,
+      "learning_rate": 1.0426946568760534e-05,
+      "loss": 0.5541,
+      "reward": 0.06157665252685547,
+      "reward_std": 0.010787656143656931,
+      "rewards/reward_func_1": 0.06157665252685547,
+      "step": 2000
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5390509477080253,
+      "grad_norm": 5.320031505107181e-06,
+      "kl": 13.80625,
+      "learning_rate": 1.0380053361881454e-05,
+      "loss": 0.5523,
+      "reward": 0.06187152862548828,
+      "reward_std": 0.010924646601051791,
+      "rewards/reward_func_1": 0.06187152862548828,
+      "step": 2005
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5403952144105391,
+      "grad_norm": 2.400984612904722e-06,
+      "kl": 13.659375,
+      "learning_rate": 1.0333151784103204e-05,
+      "loss": 0.5463,
+      "reward": 0.06603701114654541,
+      "reward_std": 0.012158522802928928,
+      "rewards/reward_func_1": 0.06603701114654541,
+      "step": 2010
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5417394811130528,
+      "grad_norm": 3.1185063562588766e-06,
+      "kl": 13.85,
+      "learning_rate": 1.0286242868460658e-05,
+      "loss": 0.5541,
+      "reward": 0.06205949783325195,
+      "reward_std": 0.011392109425651142,
+      "rewards/reward_func_1": 0.06205949783325195,
+      "step": 2015
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5430837478155666,
+      "grad_norm": 1.7467871202825336e-06,
+      "kl": 14.109375,
+      "learning_rate": 1.0239327648150324e-05,
+      "loss": 0.5644,
+      "reward": 0.05861544609069824,
+      "reward_std": 0.012278527018861497,
+      "rewards/reward_func_1": 0.05861544609069824,
+      "step": 2020
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5444280145180804,
+      "grad_norm": 2.397207026660908e-06,
+      "kl": 13.9265625,
+      "learning_rate": 1.0192407156507555e-05,
+      "loss": 0.557,
+      "reward": 0.06040668487548828,
+      "reward_std": 0.010451347306661774,
+      "rewards/reward_func_1": 0.06040668487548828,
+      "step": 2025
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5457722812205942,
+      "grad_norm": 2.443831590426271e-06,
+      "kl": 13.7421875,
+      "learning_rate": 1.0145482426983829e-05,
+      "loss": 0.5496,
+      "reward": 0.05943064689636231,
+      "reward_std": 0.010262710415554465,
+      "rewards/reward_func_1": 0.05943064689636231,
+      "step": 2030
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5471165479231079,
+      "grad_norm": 1.7376810319547076e-06,
+      "kl": 14.0484375,
+      "learning_rate": 1.0098554493123946e-05,
+      "loss": 0.5619,
+      "reward": 0.06599822044372558,
+      "reward_std": 0.010502938941499451,
+      "rewards/reward_func_1": 0.06599822044372558,
+      "step": 2035
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5484608146256217,
+      "grad_norm": 1.9027496591661475e-06,
+      "kl": 13.709375,
+      "learning_rate": 1.0051624388543303e-05,
+      "loss": 0.5482,
+      "reward": 0.06099987030029297,
+      "reward_std": 0.011544406516259187,
+      "rewards/reward_func_1": 0.06099987030029297,
+      "step": 2040
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5498050813281355,
+      "grad_norm": 3.951816779590445e-06,
+      "kl": 13.634375,
+      "learning_rate": 1.0004693146905086e-05,
+      "loss": 0.5452,
+      "reward": 0.06201457977294922,
+      "reward_std": 0.00889511961795506,
+      "rewards/reward_func_1": 0.06201457977294922,
+      "step": 2045
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5511493480306493,
+      "grad_norm": 4.0156542127078865e-06,
+      "kl": 13.6765625,
+      "learning_rate": 9.957761801897546e-06,
+      "loss": 0.547,
+      "reward": 0.05980491638183594,
+      "reward_std": 0.011892608562629903,
+      "rewards/reward_func_1": 0.05980491638183594,
+      "step": 2050
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.552493614733163,
+      "grad_norm": 1.6064385590652819e-06,
+      "kl": 13.7890625,
+      "learning_rate": 9.910831387211203e-06,
+      "loss": 0.552,
+      "reward": 0.05945572853088379,
+      "reward_std": 0.009774930006824434,
+      "rewards/reward_func_1": 0.05945572853088379,
+      "step": 2055
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5538378814356768,
+      "grad_norm": 5.076154593552928e-06,
+      "kl": 13.6640625,
+      "learning_rate": 9.863902936516079e-06,
+      "loss": 0.5466,
+      "reward": 0.0619448184967041,
+      "reward_std": 0.011996782931964845,
+      "rewards/reward_func_1": 0.0619448184967041,
+      "step": 2060
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5551821481381907,
+      "grad_norm": 1.1968987564614508e-05,
+      "kl": 13.6625,
+      "learning_rate": 9.81697748343895e-06,
+      "loss": 0.5466,
+      "reward": 0.06266765594482422,
+      "reward_std": 0.011780065088532864,
+      "rewards/reward_func_1": 0.06266765594482422,
+      "step": 2065
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5565264148407044,
+      "grad_norm": 2.355890319449827e-05,
+      "kl": 13.9328125,
+      "learning_rate": 9.77005606154056e-06,
+      "loss": 0.5576,
+      "reward": 0.05969257354736328,
+      "reward_std": 0.009937083004115266,
+      "rewards/reward_func_1": 0.05969257354736328,
+      "step": 2070
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5578706815432182,
+      "grad_norm": 8.669927410664968e-06,
+      "kl": 13.525,
+      "learning_rate": 9.723139704292866e-06,
+      "loss": 0.5408,
+      "reward": 0.06073760986328125,
+      "reward_std": 0.01386090821470134,
+      "rewards/reward_func_1": 0.06073760986328125,
+      "step": 2075
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5592149482457319,
+      "grad_norm": 3.4235183647979284e-06,
+      "kl": 13.79375,
+      "learning_rate": 9.676229445056269e-06,
+      "loss": 0.552,
+      "reward": 0.06341695785522461,
+      "reward_std": 0.010479317836870904,
+      "rewards/reward_func_1": 0.06341695785522461,
+      "step": 2080
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5605592149482457,
+      "grad_norm": 2.277251951454673e-06,
+      "kl": 14.06875,
+      "learning_rate": 9.629326317056872e-06,
+      "loss": 0.5628,
+      "reward": 0.06306524276733398,
+      "reward_std": 0.011487474158639089,
+      "rewards/reward_func_1": 0.06306524276733398,
+      "step": 2085
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5619034816507595,
+      "grad_norm": 4.600749434757745e-06,
+      "kl": 13.9359375,
+      "learning_rate": 9.582431353363687e-06,
+      "loss": 0.5572,
+      "reward": 0.0586451530456543,
+      "reward_std": 0.009657706473444706,
+      "rewards/reward_func_1": 0.0586451530456543,
+      "step": 2090
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5632477483532733,
+      "grad_norm": 7.010680747043807e-06,
+      "kl": 13.775,
+      "learning_rate": 9.535545586865922e-06,
+      "loss": 0.5508,
+      "reward": 0.06332626342773437,
+      "reward_std": 0.01151423337869346,
+      "rewards/reward_func_1": 0.06332626342773437,
+      "step": 2095
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.564592015055787,
+      "grad_norm": 3.5268849387648515e-06,
+      "kl": 13.725,
+      "learning_rate": 9.488670050250195e-06,
+      "loss": 0.5491,
+      "reward": 0.05642566680908203,
+      "reward_std": 0.01242845638480503,
+      "rewards/reward_func_1": 0.05642566680908203,
+      "step": 2100
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5659362817583008,
+      "grad_norm": 2.5853701117739547e-06,
+      "kl": 13.7984375,
+      "learning_rate": 9.441805775977822e-06,
+      "loss": 0.5522,
+      "reward": 0.05613641738891602,
+      "reward_std": 0.011232214039591782,
+      "rewards/reward_func_1": 0.05613641738891602,
+      "step": 2105
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5672805484608147,
+      "grad_norm": 4.693101436714642e-05,
+      "kl": 13.8296875,
+      "learning_rate": 9.394953796262037e-06,
+      "loss": 0.5533,
+      "reward": 0.06460676193237305,
+      "reward_std": 0.009554302979813656,
+      "rewards/reward_func_1": 0.06460676193237305,
+      "step": 2110
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5686248151633284,
+      "grad_norm": 3.470984211162431e-06,
+      "kl": 13.940625,
+      "learning_rate": 9.348115143045305e-06,
+      "loss": 0.5579,
+      "reward": 0.05569601058959961,
+      "reward_std": 0.00965253066533478,
+      "rewards/reward_func_1": 0.05569601058959961,
+      "step": 2115
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5699690818658422,
+      "grad_norm": 5.797121502837399e-06,
+      "kl": 13.55625,
+      "learning_rate": 9.301290847976545e-06,
+      "loss": 0.5421,
+      "reward": 0.06446545943617821,
+      "reward_std": 0.014136035600677133,
+      "rewards/reward_func_1": 0.06446545943617821,
+      "step": 2120
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5713133485683559,
+      "grad_norm": 2.2634183096670313e-06,
+      "kl": 13.9234375,
+      "learning_rate": 9.254481942388444e-06,
+      "loss": 0.5566,
+      "reward": 0.05872611999511719,
+      "reward_std": 0.010083984247467015,
+      "rewards/reward_func_1": 0.05872611999511719,
+      "step": 2125
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5726576152708698,
+      "grad_norm": 2.7756636882259045e-06,
+      "kl": 13.84375,
+      "learning_rate": 9.207689457274716e-06,
+      "loss": 0.5536,
+      "reward": 0.06077961921691895,
+      "reward_std": 0.01000992787303403,
+      "rewards/reward_func_1": 0.06077961921691895,
+      "step": 2130
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5740018819733835,
+      "grad_norm": 3.0030598736630054e-06,
+      "kl": 14.05625,
+      "learning_rate": 9.160914423267416e-06,
+      "loss": 0.5621,
+      "reward": 0.06290161609649658,
+      "reward_std": 0.010859317294671201,
+      "rewards/reward_func_1": 0.06290161609649658,
+      "step": 2135
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5753461486758973,
+      "grad_norm": 8.018588232516777e-06,
+      "kl": 14.0125,
+      "learning_rate": 9.114157870614213e-06,
+      "loss": 0.5605,
+      "reward": 0.06700577735900878,
+      "reward_std": 0.013860853042569943,
+      "rewards/reward_func_1": 0.06700577735900878,
+      "step": 2140
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.576690415378411,
+      "grad_norm": 3.3272003747697454e-06,
+      "kl": 14.1421875,
+      "learning_rate": 9.067420829155731e-06,
+      "loss": 0.5659,
+      "reward": 0.06595449447631836,
+      "reward_std": 0.010813094197510508,
+      "rewards/reward_func_1": 0.06595449447631836,
+      "step": 2145
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5780346820809249,
+      "grad_norm": 1.5704621546319686e-05,
+      "kl": 13.5234375,
+      "learning_rate": 9.020704328302829e-06,
+      "loss": 0.5408,
+      "reward": 0.06404781341552734,
+      "reward_std": 0.011609598056566028,
+      "rewards/reward_func_1": 0.06404781341552734,
+      "step": 2150
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5793789487834387,
+      "grad_norm": 2.1239829948171973e-05,
+      "kl": 13.6328125,
+      "learning_rate": 8.974009397013965e-06,
+      "loss": 0.5455,
+      "reward": 0.058431386947631836,
+      "reward_std": 0.010231435889363639,
+      "rewards/reward_func_1": 0.058431386947631836,
+      "step": 2155
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5807232154859524,
+      "grad_norm": 3.322323755128309e-05,
+      "kl": 13.49375,
+      "learning_rate": 8.927337063772504e-06,
+      "loss": 0.5398,
+      "reward": 0.059176063537597655,
+      "reward_std": 0.010483282200584653,
+      "rewards/reward_func_1": 0.059176063537597655,
+      "step": 2160
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5820674821884662,
+      "grad_norm": 1.4426668712985702e-06,
+      "kl": 13.8109375,
+      "learning_rate": 8.88068835656408e-06,
+      "loss": 0.5523,
+      "reward": 0.06375694274902344,
+      "reward_std": 0.011770154316764092,
+      "rewards/reward_func_1": 0.06375694274902344,
+      "step": 2165
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.58341174889098,
+      "grad_norm": 2.1057694539194927e-06,
+      "kl": 13.95625,
+      "learning_rate": 8.834064302853944e-06,
+      "loss": 0.5581,
+      "reward": 0.06186666488647461,
+      "reward_std": 0.011866289676254383,
+      "rewards/reward_func_1": 0.06186666488647461,
+      "step": 2170
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5847560155934938,
+      "grad_norm": 2.079190153381205e-06,
+      "kl": 13.7578125,
+      "learning_rate": 8.787465929564352e-06,
+      "loss": 0.5504,
+      "reward": 0.05595951080322266,
+      "reward_std": 0.011447059749116306,
+      "rewards/reward_func_1": 0.05595951080322266,
+      "step": 2175
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5861002822960075,
+      "grad_norm": 3.0134608550724806e-06,
+      "kl": 13.93125,
+      "learning_rate": 8.740894263051913e-06,
+      "loss": 0.557,
+      "reward": 0.06060028076171875,
+      "reward_std": 0.009739526234625373,
+      "rewards/reward_func_1": 0.06060028076171875,
+      "step": 2180
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5874445489985213,
+      "grad_norm": 3.109391400357708e-05,
+      "kl": 13.9,
+      "learning_rate": 8.694350329085028e-06,
+      "loss": 0.5558,
+      "reward": 0.0626680850982666,
+      "reward_std": 0.010819756354612764,
+      "rewards/reward_func_1": 0.0626680850982666,
+      "step": 2185
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5887888157010351,
+      "grad_norm": 1.3205424693296663e-05,
+      "kl": 13.6703125,
+      "learning_rate": 8.647835152821252e-06,
+      "loss": 0.5469,
+      "reward": 0.05972356796264648,
+      "reward_std": 0.010678636631928385,
+      "rewards/reward_func_1": 0.05972356796264648,
+      "step": 2190
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5901330824035489,
+      "grad_norm": 4.43181943410309e-06,
+      "kl": 13.965625,
+      "learning_rate": 8.601349758784744e-06,
+      "loss": 0.5587,
+      "reward": 0.06157550811767578,
+      "reward_std": 0.010384173551574349,
+      "rewards/reward_func_1": 0.06157550811767578,
+      "step": 2195
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5914773491060626,
+      "grad_norm": 2.103875203829375e-06,
+      "kl": 14.0875,
+      "learning_rate": 8.55489517084369e-06,
+      "loss": 0.5634,
+      "reward": 0.06180839538574219,
+      "reward_std": 0.01010028199889348,
+      "rewards/reward_func_1": 0.06180839538574219,
+      "step": 2200
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5928216158085764,
+      "grad_norm": 2.1625994122587144e-05,
+      "kl": 14.025,
+      "learning_rate": 8.508472412187759e-06,
+      "loss": 0.5611,
+      "reward": 0.06270132064819336,
+      "reward_std": 0.012054376184096327,
+      "rewards/reward_func_1": 0.06270132064819336,
+      "step": 2205
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5941658825110901,
+      "grad_norm": 6.9565994635922834e-06,
+      "kl": 13.8640625,
+      "learning_rate": 8.462082505305547e-06,
+      "loss": 0.5548,
+      "reward": 0.06188421249389649,
+      "reward_std": 0.011324935717857443,
+      "rewards/reward_func_1": 0.06188421249389649,
+      "step": 2210
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.595510149213604,
+      "grad_norm": 2.927747118519619e-06,
+      "kl": 13.953125,
+      "learning_rate": 8.415726471962092e-06,
+      "loss": 0.558,
+      "reward": 0.060194778442382815,
+      "reward_std": 0.009112278009342844,
+      "rewards/reward_func_1": 0.060194778442382815,
+      "step": 2215
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5968544159161178,
+      "grad_norm": 1.6029367770897807e-06,
+      "kl": 13.9375,
+      "learning_rate": 8.369405333176322e-06,
+      "loss": 0.5573,
+      "reward": 0.0625925064086914,
+      "reward_std": 0.01229454953354434,
+      "rewards/reward_func_1": 0.0625925064086914,
+      "step": 2220
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5981986826186315,
+      "grad_norm": 3.083619731114595e-06,
+      "kl": 14.1265625,
+      "learning_rate": 8.323120109198616e-06,
+      "loss": 0.5648,
+      "reward": 0.06270506381988525,
+      "reward_std": 0.011008751340705203,
+      "rewards/reward_func_1": 0.06270506381988525,
+      "step": 2225
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.5995429493211453,
+      "grad_norm": 5.694411811418831e-06,
+      "kl": 13.75,
+      "learning_rate": 8.276871819488287e-06,
+      "loss": 0.5501,
+      "reward": 0.06018905639648438,
+      "reward_std": 0.009886647743405775,
+      "rewards/reward_func_1": 0.06018905639648438,
+      "step": 2230
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6008872160236591,
+      "grad_norm": 2.441943252051715e-06,
+      "kl": 13.8171875,
+      "learning_rate": 8.230661482691168e-06,
+      "loss": 0.5526,
+      "reward": 0.0654977798461914,
+      "reward_std": 0.010158205546758836,
+      "rewards/reward_func_1": 0.0654977798461914,
+      "step": 2235
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6022314827261729,
+      "grad_norm": 2.561245310062077e-06,
+      "kl": 13.81875,
+      "learning_rate": 8.18449011661714e-06,
+      "loss": 0.5528,
+      "reward": 0.059857940673828124,
+      "reward_std": 0.009442199986369814,
+      "rewards/reward_func_1": 0.059857940673828124,
+      "step": 2240
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6035757494286866,
+      "grad_norm": 9.140064321400132e-06,
+      "kl": 13.8984375,
+      "learning_rate": 8.138358738217743e-06,
+      "loss": 0.5559,
+      "reward": 0.062485790252685545,
+      "reward_std": 0.009677528292741044,
+      "rewards/reward_func_1": 0.062485790252685545,
+      "step": 2245
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6049200161312004,
+      "grad_norm": 1.274393162020715e-05,
+      "kl": 13.6140625,
+      "learning_rate": 8.09226836356376e-06,
+      "loss": 0.5446,
+      "reward": 0.06158370971679687,
+      "reward_std": 0.012039840093348176,
+      "rewards/reward_func_1": 0.06158370971679687,
+      "step": 2250
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6062642828337143,
+      "grad_norm": 4.513978183240397e-06,
+      "kl": 14.028125,
+      "learning_rate": 8.046220007822845e-06,
+      "loss": 0.5613,
+      "reward": 0.05757331848144531,
+      "reward_std": 0.011246860059327447,
+      "rewards/reward_func_1": 0.05757331848144531,
+      "step": 2255
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.607608549536228,
+      "grad_norm": 4.966521828464465e-06,
+      "kl": 13.853125,
+      "learning_rate": 8.000214685237154e-06,
+      "loss": 0.554,
+      "reward": 0.059112969785928726,
+      "reward_std": 0.013033414728124627,
+      "rewards/reward_func_1": 0.059112969785928726,
+      "step": 2260
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6089528162387418,
+      "grad_norm": 3.217521907572518e-06,
+      "kl": 13.6359375,
+      "learning_rate": 7.954253409101019e-06,
+      "loss": 0.5456,
+      "reward": 0.061025047302246095,
+      "reward_std": 0.012668960404334939,
+      "rewards/reward_func_1": 0.061025047302246095,
+      "step": 2265
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6102970829412555,
+      "grad_norm": 3.596692977225757e-06,
+      "kl": 13.675,
+      "learning_rate": 7.908337191738625e-06,
+      "loss": 0.5469,
+      "reward": 0.05897402763366699,
+      "reward_std": 0.010948267369531094,
+      "rewards/reward_func_1": 0.05897402763366699,
+      "step": 2270
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6116413496437694,
+      "grad_norm": 1.489972146373475e-05,
+      "kl": 13.6046875,
+      "learning_rate": 7.862467044481696e-06,
+      "loss": 0.5443,
+      "reward": 0.06776981353759766,
+      "reward_std": 0.009472813666070579,
+      "rewards/reward_func_1": 0.06776981353759766,
+      "step": 2275
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6129856163462831,
+      "grad_norm": 4.7771932258910965e-06,
+      "kl": 13.8671875,
+      "learning_rate": 7.81664397764726e-06,
+      "loss": 0.5547,
+      "reward": 0.05934486389160156,
+      "reward_std": 0.010944688416202553,
+      "rewards/reward_func_1": 0.05934486389160156,
+      "step": 2280
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6143298830487969,
+      "grad_norm": 1.628923541829863e-06,
+      "kl": 13.64375,
+      "learning_rate": 7.770869000515344e-06,
+      "loss": 0.5459,
+      "reward": 0.059722518920898436,
+      "reward_std": 0.009479641152574913,
+      "rewards/reward_func_1": 0.059722518920898436,
+      "step": 2285
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6156741497513106,
+      "grad_norm": 3.469725243121502e-06,
+      "kl": 13.8546875,
+      "learning_rate": 7.725143121306793e-06,
+      "loss": 0.5542,
+      "reward": 0.05222053527832031,
+      "reward_std": 0.011934454514994286,
+      "rewards/reward_func_1": 0.05222053527832031,
+      "step": 2290
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6170184164538245,
+      "grad_norm": 4.984348834113916e-06,
+      "kl": 13.95625,
+      "learning_rate": 7.679467347161025e-06,
+      "loss": 0.5581,
+      "reward": 0.060247611999511716,
+      "reward_std": 0.01037932816798275,
+      "rewards/reward_func_1": 0.060247611999511716,
+      "step": 2295
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6183626831563382,
+      "grad_norm": 1.597562345523329e-06,
+      "kl": 13.9984375,
+      "learning_rate": 7.633842684113876e-06,
+      "loss": 0.5599,
+      "reward": 0.05987234115600586,
+      "reward_std": 0.009311927141970955,
+      "rewards/reward_func_1": 0.05987234115600586,
+      "step": 2300
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.619706949858852,
+      "grad_norm": 2.6105003598786425e-06,
+      "kl": 13.990625,
+      "learning_rate": 7.588270137075421e-06,
+      "loss": 0.5599,
+      "reward": 0.057819366455078125,
+      "reward_std": 0.012239268912526313,
+      "rewards/reward_func_1": 0.057819366455078125,
+      "step": 2305
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6210512165613657,
+      "grad_norm": 1.7647248569119256e-06,
+      "kl": 13.7203125,
+      "learning_rate": 7.542750709807861e-06,
+      "loss": 0.5489,
+      "reward": 0.05905556678771973,
+      "reward_std": 0.011430266295792534,
+      "rewards/reward_func_1": 0.05905556678771973,
+      "step": 2310
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6223954832638795,
+      "grad_norm": 2.6496518330532126e-06,
+      "kl": 13.6484375,
+      "learning_rate": 7.497285404903387e-06,
+      "loss": 0.5465,
+      "reward": 0.055854058265686034,
+      "reward_std": 0.010062372921674978,
+      "rewards/reward_func_1": 0.055854058265686034,
+      "step": 2315
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6237397499663934,
+      "grad_norm": 1.3658197531185579e-05,
+      "kl": 13.98125,
+      "learning_rate": 7.451875223762129e-06,
+      "loss": 0.5593,
+      "reward": 0.06093788146972656,
+      "reward_std": 0.012115493134479039,
+      "rewards/reward_func_1": 0.06093788146972656,
+      "step": 2320
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6250840166689071,
+      "grad_norm": 2.737979684752645e-06,
+      "kl": 13.5125,
+      "learning_rate": 7.4065211665700685e-06,
+      "loss": 0.5404,
+      "reward": 0.052369880676269534,
+      "reward_std": 0.013848574734220164,
+      "rewards/reward_func_1": 0.052369880676269534,
+      "step": 2325
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6264282833714209,
+      "grad_norm": 4.7086259655770846e-06,
+      "kl": 13.759375,
+      "learning_rate": 7.36122423227704e-06,
+      "loss": 0.5506,
+      "reward": 0.06091470718383789,
+      "reward_std": 0.011245868943660753,
+      "rewards/reward_func_1": 0.06091470718383789,
+      "step": 2330
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6277725500739346,
+      "grad_norm": 6.927496997377602e-06,
+      "kl": 13.696875,
+      "learning_rate": 7.315985418574693e-06,
+      "loss": 0.5479,
+      "reward": 0.05918540954589844,
+      "reward_std": 0.012543642877426464,
+      "rewards/reward_func_1": 0.05918540954589844,
+      "step": 2335
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6291168167764485,
+      "grad_norm": 3.3043399980670074e-06,
+      "kl": 13.603125,
+      "learning_rate": 7.270805721874559e-06,
+      "loss": 0.544,
+      "reward": 0.058438873291015624,
+      "reward_std": 0.010428001565014711,
+      "rewards/reward_func_1": 0.058438873291015624,
+      "step": 2340
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6304610834789622,
+      "grad_norm": 1.634690306673292e-05,
+      "kl": 13.9828125,
+      "learning_rate": 7.225686137286065e-06,
+      "loss": 0.5591,
+      "reward": 0.06279127690941096,
+      "reward_std": 0.0103473931827466,
+      "rewards/reward_func_1": 0.06279127690941096,
+      "step": 2345
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.631805350181476,
+      "grad_norm": 3.8762250369472895e-06,
+      "kl": 14.221875,
+      "learning_rate": 7.180627658594643e-06,
+      "loss": 0.5689,
+      "reward": 0.06403388977050781,
+      "reward_std": 0.01183446466930036,
+      "rewards/reward_func_1": 0.06403388977050781,
+      "step": 2350
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6331496168839897,
+      "grad_norm": 2.116005816787947e-06,
+      "kl": 14.1421875,
+      "learning_rate": 7.135631278239823e-06,
+      "loss": 0.5657,
+      "reward": 0.059031105041503905,
+      "reward_std": 0.010066585054846654,
+      "rewards/reward_func_1": 0.059031105041503905,
+      "step": 2355
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6344938835865036,
+      "grad_norm": 2.5220099360012682e-06,
+      "kl": 13.640625,
+      "learning_rate": 7.090697987293398e-06,
+      "loss": 0.5456,
+      "reward": 0.059407520294189456,
+      "reward_std": 0.010223947776830755,
+      "rewards/reward_func_1": 0.059407520294189456,
+      "step": 2360
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6358381502890174,
+      "grad_norm": 1.0680985269573284e-06,
+      "kl": 13.6078125,
+      "learning_rate": 7.045828775437558e-06,
+      "loss": 0.5443,
+      "reward": 0.06002349853515625,
+      "reward_std": 0.011791737930616364,
+      "rewards/reward_func_1": 0.06002349853515625,
+      "step": 2365
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6371824169915311,
+      "grad_norm": 4.526314114627894e-06,
+      "kl": 13.4546875,
+      "learning_rate": 7.001024630943134e-06,
+      "loss": 0.5382,
+      "reward": 0.05956945419311523,
+      "reward_std": 0.012025964839267544,
+      "rewards/reward_func_1": 0.05956945419311523,
+      "step": 2370
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6385266836940449,
+      "grad_norm": 4.205965524306521e-06,
+      "kl": 14.1140625,
+      "learning_rate": 6.956286540647794e-06,
+      "loss": 0.5649,
+      "reward": 0.060262870788574216,
+      "reward_std": 0.010221635182824684,
+      "rewards/reward_func_1": 0.060262870788574216,
+      "step": 2375
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6398709503965587,
+      "grad_norm": 2.007863940889365e-06,
+      "kl": 13.9953125,
+      "learning_rate": 6.9116154899343356e-06,
+      "loss": 0.5597,
+      "reward": 0.06056399345397949,
+      "reward_std": 0.013314929121406749,
+      "rewards/reward_func_1": 0.06056399345397949,
+      "step": 2380
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6412152170990725,
+      "grad_norm": 3.056561354242149e-06,
+      "kl": 13.8359375,
+      "learning_rate": 6.867012462708963e-06,
+      "loss": 0.5534,
+      "reward": 0.059704828262329104,
+      "reward_std": 0.011303682426660088,
+      "rewards/reward_func_1": 0.059704828262329104,
+      "step": 2385
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6425594838015862,
+      "grad_norm": 2.523838702472858e-06,
+      "kl": 13.78125,
+      "learning_rate": 6.8224784413796244e-06,
+      "loss": 0.5513,
+      "reward": 0.057023143768310545,
+      "reward_std": 0.012784256822487804,
+      "rewards/reward_func_1": 0.057023143768310545,
+      "step": 2390
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6439037505041,
+      "grad_norm": 2.2542815258930204e-06,
+      "kl": 13.7625,
+      "learning_rate": 6.77801440683437e-06,
+      "loss": 0.5508,
+      "reward": 0.057397651672363284,
+      "reward_std": 0.011636027062195353,
+      "rewards/reward_func_1": 0.057397651672363284,
+      "step": 2395
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6452480172066138,
+      "grad_norm": 3.0053756745473947e-06,
+      "kl": 13.6421875,
+      "learning_rate": 6.733621338419763e-06,
+      "loss": 0.5457,
+      "reward": 0.05742425918579101,
+      "reward_std": 0.010394414755865,
+      "rewards/reward_func_1": 0.05742425918579101,
+      "step": 2400
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6465922839091276,
+      "grad_norm": 9.635583410272375e-06,
+      "kl": 13.7828125,
+      "learning_rate": 6.689300213919271e-06,
+      "loss": 0.5511,
+      "reward": 0.061480712890625,
+      "reward_std": 0.010102924931379676,
+      "rewards/reward_func_1": 0.061480712890625,
+      "step": 2405
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6479365506116413,
+      "grad_norm": 1.539800905447919e-06,
+      "kl": 13.753125,
+      "learning_rate": 6.645052009531782e-06,
+      "loss": 0.5501,
+      "reward": 0.06248035430908203,
+      "reward_std": 0.010455972234194633,
+      "rewards/reward_func_1": 0.06248035430908203,
+      "step": 2410
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6492808173141551,
+      "grad_norm": 2.588592906249687e-06,
+      "kl": 13.6578125,
+      "learning_rate": 6.600877699850052e-06,
+      "loss": 0.5464,
+      "reward": 0.05666141510009766,
+      "reward_std": 0.015636276185978203,
+      "rewards/reward_func_1": 0.05666141510009766,
+      "step": 2415
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6506250840166689,
+      "grad_norm": 3.2917205317062326e-06,
+      "kl": 14.1875,
+      "learning_rate": 6.556778257839283e-06,
+      "loss": 0.5674,
+      "reward": 0.061602020263671876,
+      "reward_std": 0.009157647862593876,
+      "rewards/reward_func_1": 0.061602020263671876,
+      "step": 2420
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6519693507191827,
+      "grad_norm": 2.6396586690680124e-05,
+      "kl": 13.7578125,
+      "learning_rate": 6.5127546548156535e-06,
+      "loss": 0.5502,
+      "reward": 0.06312904357910157,
+      "reward_std": 0.011368433445022674,
+      "rewards/reward_func_1": 0.06312904357910157,
+      "step": 2425
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6533136174216965,
+      "grad_norm": 5.049357241659891e-06,
+      "kl": 13.6421875,
+      "learning_rate": 6.46880786042496e-06,
+      "loss": 0.5455,
+      "reward": 0.05724415183067322,
+      "reward_std": 0.012990292893664445,
+      "rewards/reward_func_1": 0.05724415183067322,
+      "step": 2430
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6546578841242102,
+      "grad_norm": 2.6590103061607806e-06,
+      "kl": 13.8875,
+      "learning_rate": 6.424938842621231e-06,
+      "loss": 0.5555,
+      "reward": 0.0595550537109375,
+      "reward_std": 0.011499256859679008,
+      "rewards/reward_func_1": 0.0595550537109375,
+      "step": 2435
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.656002150826724,
+      "grad_norm": 6.467951152444584e-06,
+      "kl": 13.7046875,
+      "learning_rate": 6.38114856764543e-06,
+      "loss": 0.5482,
+      "reward": 0.0562408447265625,
+      "reward_std": 0.011464899309066823,
+      "rewards/reward_func_1": 0.0562408447265625,
+      "step": 2440
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6573464175292378,
+      "grad_norm": 2.2811452708992874e-06,
+      "kl": 14.015625,
+      "learning_rate": 6.337438000004155e-06,
+      "loss": 0.5606,
+      "reward": 0.061225509643554686,
+      "reward_std": 0.009458938350144308,
+      "rewards/reward_func_1": 0.061225509643554686,
+      "step": 2445
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6586906842317516,
+      "grad_norm": 8.717958735360298e-06,
+      "kl": 13.875,
+      "learning_rate": 6.293808102448409e-06,
+      "loss": 0.5548,
+      "reward": 0.056508952379226686,
+      "reward_std": 0.01089983493402542,
+      "rewards/reward_func_1": 0.056508952379226686,
+      "step": 2450
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6600349509342653,
+      "grad_norm": 1.0740451216406655e-05,
+      "kl": 13.8125,
+      "learning_rate": 6.250259835952383e-06,
+      "loss": 0.5524,
+      "reward": 0.06640968322753907,
+      "reward_std": 0.013152831193292514,
+      "rewards/reward_func_1": 0.06640968322753907,
+      "step": 2455
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6613792176367791,
+      "grad_norm": 3.2728050882724347e-06,
+      "kl": 13.753125,
+      "learning_rate": 6.206794159692304e-06,
+      "loss": 0.5502,
+      "reward": 0.05744953155517578,
+      "reward_std": 0.01012560978961119,
+      "rewards/reward_func_1": 0.05744953155517578,
+      "step": 2460
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.662723484339293,
+      "grad_norm": 1.4861791896692012e-05,
+      "kl": 13.9265625,
+      "learning_rate": 6.16341203102529e-06,
+      "loss": 0.5569,
+      "reward": 0.05549445152282715,
+      "reward_std": 0.009857410499535035,
+      "rewards/reward_func_1": 0.05549445152282715,
+      "step": 2465
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6640677510418067,
+      "grad_norm": 5.46200089956983e-06,
+      "kl": 13.8640625,
+      "learning_rate": 6.120114405468285e-06,
+      "loss": 0.5546,
+      "reward": 0.05894393920898437,
+      "reward_std": 0.009983553958227276,
+      "rewards/reward_func_1": 0.05894393920898437,
+      "step": 2470
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6654120177443205,
+      "grad_norm": 1.2916130799567327e-05,
+      "kl": 13.81875,
+      "learning_rate": 6.076902236676994e-06,
+      "loss": 0.553,
+      "reward": 0.055209779739379884,
+      "reward_std": 0.01005386611832364,
+      "rewards/reward_func_1": 0.055209779739379884,
+      "step": 2475
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6667562844468342,
+      "grad_norm": 3.6688261388917454e-06,
+      "kl": 13.7546875,
+      "learning_rate": 6.033776476424888e-06,
+      "loss": 0.5503,
+      "reward": 0.06743335723876953,
+      "reward_std": 0.01147701254230924,
+      "rewards/reward_func_1": 0.06743335723876953,
+      "step": 2480
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6681005511493481,
+      "grad_norm": 5.666680408467073e-06,
+      "kl": 13.990625,
+      "learning_rate": 5.990738074582243e-06,
+      "loss": 0.5592,
+      "reward": 0.06467456817626953,
+      "reward_std": 0.013033680556691251,
+      "rewards/reward_func_1": 0.06467456817626953,
+      "step": 2485
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6694448178518618,
+      "grad_norm": 1.422238983650459e-05,
+      "kl": 13.8578125,
+      "learning_rate": 5.947787979095213e-06,
+      "loss": 0.5543,
+      "reward": 0.06046428680419922,
+      "reward_std": 0.013719953599502333,
+      "rewards/reward_func_1": 0.06046428680419922,
+      "step": 2490
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6707890845543756,
+      "grad_norm": 5.443932877824409e-06,
+      "kl": 13.81875,
+      "learning_rate": 5.9049271359649466e-06,
+      "loss": 0.5526,
+      "reward": 0.056779670715332034,
+      "reward_std": 0.009680721638142131,
+      "rewards/reward_func_1": 0.056779670715332034,
+      "step": 2495
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6721333512568893,
+      "grad_norm": 6.311719971563434e-06,
+      "kl": 13.9265625,
+      "learning_rate": 5.862156489226768e-06,
+      "loss": 0.5572,
+      "reward": 0.056317138671875,
+      "reward_std": 0.012263055084622465,
+      "rewards/reward_func_1": 0.056317138671875,
+      "step": 2500
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6734776179594032,
+      "grad_norm": 3.1811771350476192e-06,
+      "kl": 13.7046875,
+      "learning_rate": 5.819476980929357e-06,
+      "loss": 0.548,
+      "reward": 0.05898451805114746,
+      "reward_std": 0.011924323247512802,
+      "rewards/reward_func_1": 0.05898451805114746,
+      "step": 2505
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.674821884661917,
+      "grad_norm": 2.1738133000326343e-06,
+      "kl": 13.8359375,
+      "learning_rate": 5.776889551114036e-06,
+      "loss": 0.5537,
+      "reward": 0.05574178695678711,
+      "reward_std": 0.009539656856213696,
+      "rewards/reward_func_1": 0.05574178695678711,
+      "step": 2510
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6761661513644307,
+      "grad_norm": 3.079718953813426e-05,
+      "kl": 13.7,
+      "learning_rate": 5.734395137794022e-06,
+      "loss": 0.5483,
+      "reward": 0.058077239990234376,
+      "reward_std": 0.00982013454704429,
+      "rewards/reward_func_1": 0.058077239990234376,
+      "step": 2515
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6775104180669445,
+      "grad_norm": 2.630328253871994e-06,
+      "kl": 13.78125,
+      "learning_rate": 5.691994676933808e-06,
+      "loss": 0.5511,
+      "reward": 0.05584440231323242,
+      "reward_std": 0.009929925179494602,
+      "rewards/reward_func_1": 0.05584440231323242,
+      "step": 2520
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6788546847694582,
+      "grad_norm": 4.1391981540073175e-06,
+      "kl": 13.6796875,
+      "learning_rate": 5.6496891024285215e-06,
+      "loss": 0.5475,
+      "reward": 0.058974266052246094,
+      "reward_std": 0.010749774679425173,
+      "rewards/reward_func_1": 0.058974266052246094,
+      "step": 2525
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6801989514719721,
+      "grad_norm": 2.052042191280634e-06,
+      "kl": 13.6703125,
+      "learning_rate": 5.607479346083355e-06,
+      "loss": 0.5469,
+      "reward": 0.05872535705566406,
+      "reward_std": 0.011635806861886522,
+      "rewards/reward_func_1": 0.05872535705566406,
+      "step": 2530
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6815432181744858,
+      "grad_norm": 2.211010541941505e-05,
+      "kl": 14.2609375,
+      "learning_rate": 5.565366337593066e-06,
+      "loss": 0.5708,
+      "reward": 0.06311745643615722,
+      "reward_std": 0.01183991582802264,
+      "rewards/reward_func_1": 0.06311745643615722,
+      "step": 2535
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6828874848769996,
+      "grad_norm": 4.809753590961918e-06,
+      "kl": 14.0015625,
+      "learning_rate": 5.523351004521462e-06,
+      "loss": 0.5603,
+      "reward": 0.05524139404296875,
+      "reward_std": 0.010296737632233998,
+      "rewards/reward_func_1": 0.05524139404296875,
+      "step": 2540
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6842317515795133,
+      "grad_norm": 8.715678632142954e-06,
+      "kl": 14.078125,
+      "learning_rate": 5.481434272281013e-06,
+      "loss": 0.5629,
+      "reward": 0.06164817810058594,
+      "reward_std": 0.013340477158635622,
+      "rewards/reward_func_1": 0.06164817810058594,
+      "step": 2545
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6855760182820272,
+      "grad_norm": 2.4492214834026527e-06,
+      "kl": 13.778125,
+      "learning_rate": 5.439617064112431e-06,
+      "loss": 0.5511,
+      "reward": 0.05745353698730469,
+      "reward_std": 0.013168468393268995,
+      "rewards/reward_func_1": 0.05745353698730469,
+      "step": 2550
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6869202849845409,
+      "grad_norm": 2.812889078995795e-06,
+      "kl": 13.9015625,
+      "learning_rate": 5.3979003010643675e-06,
+      "loss": 0.5562,
+      "reward": 0.057623672485351565,
+      "reward_std": 0.01229256743681617,
+      "rewards/reward_func_1": 0.057623672485351565,
+      "step": 2555
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6882645516870547,
+      "grad_norm": 3.987305262853624e-06,
+      "kl": 13.965625,
+      "learning_rate": 5.356284901973091e-06,
+      "loss": 0.5588,
+      "reward": 0.059996414184570315,
+      "reward_std": 0.010288478545771796,
+      "rewards/reward_func_1": 0.059996414184570315,
+      "step": 2560
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6896088183895684,
+      "grad_norm": 1.2458726814656984e-05,
+      "kl": 14.121875,
+      "learning_rate": 5.314771783442292e-06,
+      "loss": 0.5647,
+      "reward": 0.05899205207824707,
+      "reward_std": 0.010236831862857797,
+      "rewards/reward_func_1": 0.05899205207824707,
+      "step": 2565
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6909530850920823,
+      "grad_norm": 2.1745931917394046e-06,
+      "kl": 13.653125,
+      "learning_rate": 5.273361859822852e-06,
+      "loss": 0.5463,
+      "reward": 0.06059694290161133,
+      "reward_std": 0.01137404957335093,
+      "rewards/reward_func_1": 0.06059694290161133,
+      "step": 2570
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6922973517945961,
+      "grad_norm": 5.8191294556309e-06,
+      "kl": 14.08125,
+      "learning_rate": 5.232056043192737e-06,
+      "loss": 0.5633,
+      "reward": 0.0685009479522705,
+      "reward_std": 0.012357043109659571,
+      "rewards/reward_func_1": 0.0685009479522705,
+      "step": 2575
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6936416184971098,
+      "grad_norm": 3.856658622680698e-06,
+      "kl": 13.8921875,
+      "learning_rate": 5.190855243336883e-06,
+      "loss": 0.5559,
+      "reward": 0.06555595397949218,
+      "reward_std": 0.011982467219604586,
+      "rewards/reward_func_1": 0.06555595397949218,
+      "step": 2580
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6949858851996236,
+      "grad_norm": 1.3858561032975558e-05,
+      "kl": 14.0171875,
+      "learning_rate": 5.1497603677271855e-06,
+      "loss": 0.5606,
+      "reward": 0.06087760925292969,
+      "reward_std": 0.011154358516796492,
+      "rewards/reward_func_1": 0.06087760925292969,
+      "step": 2585
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6963301519021374,
+      "grad_norm": 4.5027968553768005e-06,
+      "kl": 13.7328125,
+      "learning_rate": 5.108772321502479e-06,
+      "loss": 0.5494,
+      "reward": 0.05637903213500976,
+      "reward_std": 0.012003830538014881,
+      "rewards/reward_func_1": 0.05637903213500976,
+      "step": 2590
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6976744186046512,
+      "grad_norm": 2.7505389880388975e-05,
+      "kl": 13.94375,
+      "learning_rate": 5.0678920074486316e-06,
+      "loss": 0.5578,
+      "reward": 0.06141033172607422,
+      "reward_std": 0.010152479278622195,
+      "rewards/reward_func_1": 0.06141033172607422,
+      "step": 2595
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.6990186853071649,
+      "grad_norm": 2.104391796819982e-06,
+      "kl": 14.35625,
+      "learning_rate": 5.0271203259786395e-06,
+      "loss": 0.5744,
+      "reward": 0.06711845397949219,
+      "reward_std": 0.011301479887515597,
+      "rewards/reward_func_1": 0.06711845397949219,
+      "step": 2600
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7003629520096787,
+      "grad_norm": 1.6334478232238325e-06,
+      "kl": 13.753125,
+      "learning_rate": 4.986458175112807e-06,
+      "loss": 0.5501,
+      "reward": 0.05772566795349121,
+      "reward_std": 0.011294707475099131,
+      "rewards/reward_func_1": 0.05772566795349121,
+      "step": 2605
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7017072187121925,
+      "grad_norm": 4.255656222085236e-06,
+      "kl": 13.778125,
+      "learning_rate": 4.945906450458955e-06,
+      "loss": 0.5511,
+      "reward": 0.058788979053497316,
+      "reward_std": 0.009819350033649244,
+      "rewards/reward_func_1": 0.058788979053497316,
+      "step": 2610
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7030514854147063,
+      "grad_norm": 3.1343079172074795e-06,
+      "kl": 13.5140625,
+      "learning_rate": 4.90546604519271e-06,
+      "loss": 0.5407,
+      "reward": 0.05972156524658203,
+      "reward_std": 0.011394862360612023,
+      "rewards/reward_func_1": 0.05972156524658203,
+      "step": 2615
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.70439575211722,
+      "grad_norm": 4.186888418189483e-06,
+      "kl": 13.715625,
+      "learning_rate": 4.865137850037817e-06,
+      "loss": 0.5488,
+      "reward": 0.057996368408203124,
+      "reward_std": 0.011965288411010988,
+      "rewards/reward_func_1": 0.057996368408203124,
+      "step": 2620
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7057400188197338,
+      "grad_norm": 2.494949512765743e-06,
+      "kl": 13.9375,
+      "learning_rate": 4.824922753246534e-06,
+      "loss": 0.5575,
+      "reward": 0.05783071517944336,
+      "reward_std": 0.011933683512324933,
+      "rewards/reward_func_1": 0.05783071517944336,
+      "step": 2625
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7070842855222477,
+      "grad_norm": 4.90788443130441e-06,
+      "kl": 14.0078125,
+      "learning_rate": 4.784821640580051e-06,
+      "loss": 0.5603,
+      "reward": 0.060264754295349124,
+      "reward_std": 0.011658078715845477,
+      "rewards/reward_func_1": 0.060264754295349124,
+      "step": 2630
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7084285522247614,
+      "grad_norm": 2.91156266030157e-06,
+      "kl": 13.8421875,
+      "learning_rate": 4.744835395289002e-06,
+      "loss": 0.5537,
+      "reward": 0.05923728942871094,
+      "reward_std": 0.012938315909923403,
+      "rewards/reward_func_1": 0.05923728942871094,
+      "step": 2635
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7097728189272752,
+      "grad_norm": 3.442679826548556e-06,
+      "kl": 13.8109375,
+      "learning_rate": 4.704964898093991e-06,
+      "loss": 0.5527,
+      "reward": 0.06276912689208984,
+      "reward_std": 0.011696373121230863,
+      "rewards/reward_func_1": 0.06276912689208984,
+      "step": 2640
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7111170856297889,
+      "grad_norm": 4.503699074120959e-06,
+      "kl": 13.875,
+      "learning_rate": 4.665211027166209e-06,
+      "loss": 0.5547,
+      "reward": 0.059120559692382814,
+      "reward_std": 0.011374600145063595,
+      "rewards/reward_func_1": 0.059120559692382814,
+      "step": 2645
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7124613523323027,
+      "grad_norm": 9.90547505352879e-06,
+      "kl": 13.6984375,
+      "learning_rate": 4.625574658108073e-06,
+      "loss": 0.5478,
+      "reward": 0.057414674758911134,
+      "reward_std": 0.010342052261330536,
+      "rewards/reward_func_1": 0.057414674758911134,
+      "step": 2650
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7138056190348165,
+      "grad_norm": 2.0887078790110536e-05,
+      "kl": 13.509375,
+      "learning_rate": 4.586056663933969e-06,
+      "loss": 0.5406,
+      "reward": 0.05762338638305664,
+      "reward_std": 0.014504123894221265,
+      "rewards/reward_func_1": 0.05762338638305664,
+      "step": 2655
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7151498857373303,
+      "grad_norm": 3.3340979825879913e-06,
+      "kl": 13.7078125,
+      "learning_rate": 4.546657915050988e-06,
+      "loss": 0.5483,
+      "reward": 0.06230294108390808,
+      "reward_std": 0.009969272715534317,
+      "rewards/reward_func_1": 0.06230294108390808,
+      "step": 2660
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.716494152439844,
+      "grad_norm": 3.2463603929500096e-06,
+      "kl": 13.7390625,
+      "learning_rate": 4.507379279239791e-06,
+      "loss": 0.5496,
+      "reward": 0.05877430438995361,
+      "reward_std": 0.011900619864900364,
+      "rewards/reward_func_1": 0.05877430438995361,
+      "step": 2665
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7178384191423578,
+      "grad_norm": 3.1261215553968213e-06,
+      "kl": 13.99375,
+      "learning_rate": 4.468221621635462e-06,
+      "loss": 0.5597,
+      "reward": 0.05568780899047852,
+      "reward_std": 0.008807793819141808,
+      "rewards/reward_func_1": 0.05568780899047852,
+      "step": 2670
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7191826858448717,
+      "grad_norm": 2.6415564207127318e-06,
+      "kl": 13.9015625,
+      "learning_rate": 4.42918580470848e-06,
+      "loss": 0.5562,
+      "reward": 0.060968208312988284,
+      "reward_std": 0.009969678838388063,
+      "rewards/reward_func_1": 0.060968208312988284,
+      "step": 2675
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7205269525473854,
+      "grad_norm": 2.9144048312446102e-06,
+      "kl": 14.0859375,
+      "learning_rate": 4.39027268824571e-06,
+      "loss": 0.5633,
+      "reward": 0.06536164283752441,
+      "reward_std": 0.011233370206900873,
+      "rewards/reward_func_1": 0.06536164283752441,
+      "step": 2680
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7218712192498992,
+      "grad_norm": 3.608389533837908e-06,
+      "kl": 14.0390625,
+      "learning_rate": 4.351483129331458e-06,
+      "loss": 0.5612,
+      "reward": 0.06538281440734864,
+      "reward_std": 0.013752934670628747,
+      "rewards/reward_func_1": 0.06538281440734864,
+      "step": 2685
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7232154859524129,
+      "grad_norm": 6.397221568477107e-06,
+      "kl": 13.865625,
+      "learning_rate": 4.312817982328612e-06,
+      "loss": 0.5546,
+      "reward": 0.06181436069309711,
+      "reward_std": 0.011076993081223918,
+      "rewards/reward_func_1": 0.06181436069309711,
+      "step": 2690
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7245597526549268,
+      "grad_norm": 9.608173968445044e-06,
+      "kl": 13.825,
+      "learning_rate": 4.2742780988598145e-06,
+      "loss": 0.5534,
+      "reward": 0.06040000915527344,
+      "reward_std": 0.012127826601499692,
+      "rewards/reward_func_1": 0.06040000915527344,
+      "step": 2695
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7259040193574405,
+      "grad_norm": 7.110948445188114e-06,
+      "kl": 14.2703125,
+      "learning_rate": 4.235864327788692e-06,
+      "loss": 0.5708,
+      "reward": 0.06447288990020753,
+      "reward_std": 0.00979026438217261,
+      "rewards/reward_func_1": 0.06447288990020753,
+      "step": 2700
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7272482860599543,
+      "grad_norm": 4.501914645516081e-06,
+      "kl": 13.8,
+      "learning_rate": 4.197577515201191e-06,
+      "loss": 0.5523,
+      "reward": 0.0603661984205246,
+      "reward_std": 0.011186352090589935,
+      "rewards/reward_func_1": 0.0603661984205246,
+      "step": 2705
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.728592552762468,
+      "grad_norm": 7.746289156784769e-06,
+      "kl": 13.478125,
+      "learning_rate": 4.159418504386904e-06,
+      "loss": 0.5393,
+      "reward": 0.057269958406686784,
+      "reward_std": 0.01241556809945905,
+      "rewards/reward_func_1": 0.057269958406686784,
+      "step": 2710
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7299368194649819,
+      "grad_norm": 6.022199613653356e-06,
+      "kl": 13.8953125,
+      "learning_rate": 4.1213881358205275e-06,
+      "loss": 0.5558,
+      "reward": 0.0635772705078125,
+      "reward_std": 0.009903606217267224,
+      "rewards/reward_func_1": 0.0635772705078125,
+      "step": 2715
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7312810861674957,
+      "grad_norm": 4.5010624489805195e-06,
+      "kl": 13.7203125,
+      "learning_rate": 4.083487247143326e-06,
+      "loss": 0.5486,
+      "reward": 0.06045243740081787,
+      "reward_std": 0.011131673593808955,
+      "rewards/reward_func_1": 0.06045243740081787,
+      "step": 2720
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7326253528700094,
+      "grad_norm": 4.66905157736619e-06,
+      "kl": 13.7625,
+      "learning_rate": 4.045716673144706e-06,
+      "loss": 0.5505,
+      "reward": 0.061006355285644534,
+      "reward_std": 0.011753415851853789,
+      "rewards/reward_func_1": 0.061006355285644534,
+      "step": 2725
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7339696195725232,
+      "grad_norm": 3.085681328229839e-06,
+      "kl": 13.7109375,
+      "learning_rate": 4.008077245743801e-06,
+      "loss": 0.5486,
+      "reward": 0.06153240203857422,
+      "reward_std": 0.011887542959448183,
+      "rewards/reward_func_1": 0.06153240203857422,
+      "step": 2730
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.735313886275037,
+      "grad_norm": 2.592519194877241e-06,
+      "kl": 14.075,
+      "learning_rate": 3.970569793971178e-06,
+      "loss": 0.5628,
+      "reward": 0.06015148162841797,
+      "reward_std": 0.011894370408845134,
+      "rewards/reward_func_1": 0.06015148162841797,
+      "step": 2735
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7366581529775508,
+      "grad_norm": 1.798785774553835e-06,
+      "kl": 13.7875,
+      "learning_rate": 3.933195143950551e-06,
+      "loss": 0.5514,
+      "reward": 0.06206645965576172,
+      "reward_std": 0.014155591612507124,
+      "rewards/reward_func_1": 0.06206645965576172,
+      "step": 2740
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7380024196800645,
+      "grad_norm": 3.439082775003044e-06,
+      "kl": 13.7390625,
+      "learning_rate": 3.89595411888061e-06,
+      "loss": 0.5497,
+      "reward": 0.06377677917480469,
+      "reward_std": 0.010366774378053379,
+      "rewards/reward_func_1": 0.06377677917480469,
+      "step": 2745
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7393466863825783,
+      "grad_norm": 2.5652859676483786e-06,
+      "kl": 13.728125,
+      "learning_rate": 3.85884753901686e-06,
+      "loss": 0.5493,
+      "reward": 0.06238212585449219,
+      "reward_std": 0.009483605425339192,
+      "rewards/reward_func_1": 0.06238212585449219,
+      "step": 2750
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.740690953085092,
+      "grad_norm": 3.275632843724452e-05,
+      "kl": 13.8734375,
+      "learning_rate": 3.82187622165359e-06,
+      "loss": 0.5549,
+      "reward": 0.05930185317993164,
+      "reward_std": 0.010815516777802259,
+      "rewards/reward_func_1": 0.05930185317993164,
+      "step": 2755
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7420352197876059,
+      "grad_norm": 5.840865014761221e-06,
+      "kl": 13.6109375,
+      "learning_rate": 3.7850409811058343e-06,
+      "loss": 0.5445,
+      "reward": 0.05905466079711914,
+      "reward_std": 0.008731590279785451,
+      "rewards/reward_func_1": 0.05905466079711914,
+      "step": 2760
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7433794864901196,
+      "grad_norm": 1.5586972949677147e-05,
+      "kl": 13.8890625,
+      "learning_rate": 3.7483426286914705e-06,
+      "loss": 0.5557,
+      "reward": 0.0615997314453125,
+      "reward_std": 0.012081135442713276,
+      "rewards/reward_func_1": 0.0615997314453125,
+      "step": 2765
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7447237531926334,
+      "grad_norm": 6.052292064850917e-06,
+      "kl": 13.45,
+      "learning_rate": 3.7117819727133254e-06,
+      "loss": 0.5381,
+      "reward": 0.06048717498779297,
+      "reward_std": 0.008764956895902287,
+      "rewards/reward_func_1": 0.06048717498779297,
+      "step": 2770
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7460680198951471,
+      "grad_norm": 3.043762262677774e-06,
+      "kl": 13.8125,
+      "learning_rate": 3.6753598184413873e-06,
+      "loss": 0.5528,
+      "reward": 0.06238512992858887,
+      "reward_std": 0.012483571946359007,
+      "rewards/reward_func_1": 0.06238512992858887,
+      "step": 2775
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.747412286597661,
+      "grad_norm": 8.626040653325617e-06,
+      "kl": 13.8703125,
+      "learning_rate": 3.6390769680950544e-06,
+      "loss": 0.5549,
+      "reward": 0.06061878204345703,
+      "reward_std": 0.012912878150018514,
+      "rewards/reward_func_1": 0.06061878204345703,
+      "step": 2780
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7487565533001748,
+      "grad_norm": 8.276882908830885e-06,
+      "kl": 13.959375,
+      "learning_rate": 3.6029342208254826e-06,
+      "loss": 0.5585,
+      "reward": 0.06053438186645508,
+      "reward_std": 0.012466668507113355,
+      "rewards/reward_func_1": 0.06053438186645508,
+      "step": 2785
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7501008200026885,
+      "grad_norm": 8.260290996986441e-06,
+      "kl": 13.8265625,
+      "learning_rate": 3.5669323726979655e-06,
+      "loss": 0.5533,
+      "reward": 0.0611328125,
+      "reward_std": 0.011878072742911172,
+      "rewards/reward_func_1": 0.0611328125,
+      "step": 2790
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7514450867052023,
+      "grad_norm": 4.150938366365153e-06,
+      "kl": 13.6796875,
+      "learning_rate": 3.531072216674418e-06,
+      "loss": 0.5471,
+      "reward": 0.06307134628295899,
+      "reward_std": 0.01603546408514376,
+      "rewards/reward_func_1": 0.06307134628295899,
+      "step": 2795
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7527893534077161,
+      "grad_norm": 2.825467163347639e-06,
+      "kl": 13.9296875,
+      "learning_rate": 3.4953545425959047e-06,
+      "loss": 0.557,
+      "reward": 0.0554865837097168,
+      "reward_std": 0.012560931847292522,
+      "rewards/reward_func_1": 0.0554865837097168,
+      "step": 2800
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7541336201102299,
+      "grad_norm": 9.65241724770749e-06,
+      "kl": 14.05,
+      "learning_rate": 3.4597801371652296e-06,
+      "loss": 0.5621,
+      "reward": 0.061970877647399905,
+      "reward_std": 0.010358845694281627,
+      "rewards/reward_func_1": 0.061970877647399905,
+      "step": 2805
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7554778868127436,
+      "grad_norm": 1.7722894654070842e-06,
+      "kl": 13.821875,
+      "learning_rate": 3.424349783929636e-06,
+      "loss": 0.5529,
+      "reward": 0.061666107177734374,
+      "reward_std": 0.011269324702152517,
+      "rewards/reward_func_1": 0.061666107177734374,
+      "step": 2810
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7568221535152574,
+      "grad_norm": 1.5782270565978251e-06,
+      "kl": 13.9140625,
+      "learning_rate": 3.3890642632635153e-06,
+      "loss": 0.5564,
+      "reward": 0.06216366291046142,
+      "reward_std": 0.010467645124299452,
+      "rewards/reward_func_1": 0.06216366291046142,
+      "step": 2815
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7581664202177713,
+      "grad_norm": 9.261582817998715e-06,
+      "kl": 13.984375,
+      "learning_rate": 3.353924352351253e-06,
+      "loss": 0.5595,
+      "reward": 0.05453653335571289,
+      "reward_std": 0.010780278017773526,
+      "rewards/reward_func_1": 0.05453653335571289,
+      "step": 2820
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.759510686920285,
+      "grad_norm": 5.424847131507704e-06,
+      "kl": 13.9875,
+      "learning_rate": 3.3189308251700825e-06,
+      "loss": 0.5595,
+      "reward": 0.057614707946777345,
+      "reward_std": 0.011758481396100251,
+      "rewards/reward_func_1": 0.057614707946777345,
+      "step": 2825
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7608549536227988,
+      "grad_norm": 1.4945719158276916e-05,
+      "kl": 13.875,
+      "learning_rate": 3.2840844524730577e-06,
+      "loss": 0.555,
+      "reward": 0.05519509315490723,
+      "reward_std": 0.009796513656328897,
+      "rewards/reward_func_1": 0.05519509315490723,
+      "step": 2830
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7621992203253125,
+      "grad_norm": 2.619063934616861e-06,
+      "kl": 14.10625,
+      "learning_rate": 3.2493860017720567e-06,
+      "loss": 0.5644,
+      "reward": 0.055352401733398435,
+      "reward_std": 0.010450466238398803,
+      "rewards/reward_func_1": 0.055352401733398435,
+      "step": 2835
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7635434870278264,
+      "grad_norm": 1.0150999514735304e-05,
+      "kl": 13.84375,
+      "learning_rate": 3.214836237320904e-06,
+      "loss": 0.5538,
+      "reward": 0.058098793029785156,
+      "reward_std": 0.011656674755067797,
+      "rewards/reward_func_1": 0.058098793029785156,
+      "step": 2840
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7648877537303401,
+      "grad_norm": 4.336788151704241e-06,
+      "kl": 13.6640625,
+      "learning_rate": 3.1804359200985056e-06,
+      "loss": 0.5466,
+      "reward": 0.05649633407592773,
+      "reward_std": 0.011210850576026133,
+      "rewards/reward_func_1": 0.05649633407592773,
+      "step": 2845
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7662320204328539,
+      "grad_norm": 7.680199814785738e-06,
+      "kl": 13.525,
+      "learning_rate": 3.14618580779212e-06,
+      "loss": 0.5411,
+      "reward": 0.05644134283065796,
+      "reward_std": 0.01211523166639381,
+      "rewards/reward_func_1": 0.05644134283065796,
+      "step": 2850
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7675762871353676,
+      "grad_norm": 3.693724920594832e-06,
+      "kl": 13.6765625,
+      "learning_rate": 3.1120866547806394e-06,
+      "loss": 0.547,
+      "reward": 0.055395317077636716,
+      "reward_std": 0.012751551120891236,
+      "rewards/reward_func_1": 0.055395317077636716,
+      "step": 2855
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7689205538378814,
+      "grad_norm": 6.3646293710917234e-06,
+      "kl": 14.1234375,
+      "learning_rate": 3.0781392121179986e-06,
+      "loss": 0.5649,
+      "reward": 0.05985813140869141,
+      "reward_std": 0.010235290192213142,
+      "rewards/reward_func_1": 0.05985813140869141,
+      "step": 2860
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7702648205403952,
+      "grad_norm": 1.4028549230715726e-05,
+      "kl": 14.3015625,
+      "learning_rate": 3.0443442275166226e-06,
+      "loss": 0.5718,
+      "reward": 0.05446624755859375,
+      "reward_std": 0.009798991409479641,
+      "rewards/reward_func_1": 0.05446624755859375,
+      "step": 2865
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.771609087242909,
+      "grad_norm": 3.3037556477211183e-06,
+      "kl": 13.9109375,
+      "learning_rate": 3.0107024453309486e-06,
+      "loss": 0.5564,
+      "reward": 0.054990959167480466,
+      "reward_std": 0.009734460682375356,
+      "rewards/reward_func_1": 0.054990959167480466,
+      "step": 2870
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7729533539454227,
+      "grad_norm": 3.8873076846357435e-06,
+      "kl": 13.7890625,
+      "learning_rate": 2.9772146065410477e-06,
+      "loss": 0.5516,
+      "reward": 0.057455134391784665,
+      "reward_std": 0.010452806322427932,
+      "rewards/reward_func_1": 0.057455134391784665,
+      "step": 2875
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7742976206479365,
+      "grad_norm": 8.293524842883926e-06,
+      "kl": 14.35,
+      "learning_rate": 2.943881448736301e-06,
+      "loss": 0.5742,
+      "reward": 0.062497615814208984,
+      "reward_std": 0.00955452322596102,
+      "rewards/reward_func_1": 0.062497615814208984,
+      "step": 2880
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7756418873504504,
+      "grad_norm": 2.3304564820136875e-06,
+      "kl": 13.9890625,
+      "learning_rate": 2.910703706099137e-06,
+      "loss": 0.5594,
+      "reward": 0.06021251678466797,
+      "reward_std": 0.012565446839289507,
+      "rewards/reward_func_1": 0.06021251678466797,
+      "step": 2885
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7769861540529641,
+      "grad_norm": 2.393172962911194e-06,
+      "kl": 13.796875,
+      "learning_rate": 2.8776821093888883e-06,
+      "loss": 0.552,
+      "reward": 0.06193351745605469,
+      "reward_std": 0.010560587099462282,
+      "rewards/reward_func_1": 0.06193351745605469,
+      "step": 2890
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7783304207554779,
+      "grad_norm": 8.713544048077893e-06,
+      "kl": 14.021875,
+      "learning_rate": 2.8448173859256665e-06,
+      "loss": 0.5609,
+      "reward": 0.060492420196533205,
+      "reward_std": 0.01069757735276653,
+      "rewards/reward_func_1": 0.060492420196533205,
+      "step": 2895
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7796746874579916,
+      "grad_norm": 3.608857878134586e-05,
+      "kl": 13.4609375,
+      "learning_rate": 2.8121102595743732e-06,
+      "loss": 0.5384,
+      "reward": 0.05852642059326172,
+      "reward_std": 0.010727309926369343,
+      "rewards/reward_func_1": 0.05852642059326172,
+      "step": 2900
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7810189541605055,
+      "grad_norm": 2.3314752979786135e-05,
+      "kl": 13.8453125,
+      "learning_rate": 2.779561450728725e-06,
+      "loss": 0.5537,
+      "reward": 0.06407814025878907,
+      "reward_std": 0.010947771910286975,
+      "rewards/reward_func_1": 0.06407814025878907,
+      "step": 2905
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7823632208630192,
+      "grad_norm": 3.858524905808736e-06,
+      "kl": 13.8734375,
+      "learning_rate": 2.7471716762954183e-06,
+      "loss": 0.5551,
+      "reward": 0.05899543762207031,
+      "reward_std": 0.012197423033649102,
+      "rewards/reward_func_1": 0.05899543762207031,
+      "step": 2910
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.783707487565533,
+      "grad_norm": 9.215535101247951e-06,
+      "kl": 13.7046875,
+      "learning_rate": 2.7149416496783055e-06,
+      "loss": 0.5481,
+      "reward": 0.06349143981933594,
+      "reward_std": 0.010476895228566718,
+      "rewards/reward_func_1": 0.06349143981933594,
+      "step": 2915
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7850517542680467,
+      "grad_norm": 1.8621502704263548e-06,
+      "kl": 13.9234375,
+      "learning_rate": 2.6828720807627173e-06,
+      "loss": 0.5572,
+      "reward": 0.05804300308227539,
+      "reward_std": 0.010599679932784056,
+      "rewards/reward_func_1": 0.05804300308227539,
+      "step": 2920
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7863960209705606,
+      "grad_norm": 2.1705293420382077e-06,
+      "kl": 13.7421875,
+      "learning_rate": 2.6509636758997914e-06,
+      "loss": 0.5496,
+      "reward": 0.06185646057128906,
+      "reward_std": 0.012096992944861995,
+      "rewards/reward_func_1": 0.06185646057128906,
+      "step": 2925
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7877402876730744,
+      "grad_norm": 3.4049091937049525e-06,
+      "kl": 13.90625,
+      "learning_rate": 2.619217137890949e-06,
+      "loss": 0.5562,
+      "reward": 0.06344146728515625,
+      "reward_std": 0.010752197249166784,
+      "rewards/reward_func_1": 0.06344146728515625,
+      "step": 2930
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7890845543755881,
+      "grad_norm": 5.779114417236997e-06,
+      "kl": 13.809375,
+      "learning_rate": 2.587633165972384e-06,
+      "loss": 0.5523,
+      "reward": 0.061241436004638675,
+      "reward_std": 0.011485271743731573,
+      "rewards/reward_func_1": 0.061241436004638675,
+      "step": 2935
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7904288210781019,
+      "grad_norm": 1.907208570628427e-05,
+      "kl": 13.6140625,
+      "learning_rate": 2.556212455799688e-06,
+      "loss": 0.5447,
+      "reward": 0.05872478485107422,
+      "reward_std": 0.01105458896199707,
+      "rewards/reward_func_1": 0.05872478485107422,
+      "step": 2940
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7917730877806157,
+      "grad_norm": 1.2913329555885866e-05,
+      "kl": 13.5984375,
+      "learning_rate": 2.5249556994325063e-06,
+      "loss": 0.5443,
+      "reward": 0.05844389796257019,
+      "reward_std": 0.011415049015340628,
+      "rewards/reward_func_1": 0.05844389796257019,
+      "step": 2945
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7931173544831295,
+      "grad_norm": 7.899559022916947e-06,
+      "kl": 13.740625,
+      "learning_rate": 2.4938635853193127e-06,
+      "loss": 0.5495,
+      "reward": 0.060787391662597653,
+      "reward_std": 0.011730345609248616,
+      "rewards/reward_func_1": 0.060787391662597653,
+      "step": 2950
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7944616211856432,
+      "grad_norm": 2.142528501281049e-06,
+      "kl": 14.0171875,
+      "learning_rate": 2.462936798282236e-06,
+      "loss": 0.5608,
+      "reward": 0.05785312652587891,
+      "reward_std": 0.012231780852016528,
+      "rewards/reward_func_1": 0.05785312652587891,
+      "step": 2955
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.795805887888157,
+      "grad_norm": 2.0210850379953627e-06,
+      "kl": 13.615625,
+      "learning_rate": 2.4321760195019807e-06,
+      "loss": 0.5444,
+      "reward": 0.058881378173828124,
+      "reward_std": 0.009882683275645832,
+      "rewards/reward_func_1": 0.058881378173828124,
+      "step": 2960
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7971501545906707,
+      "grad_norm": 4.331094714871142e-06,
+      "kl": 13.8859375,
+      "learning_rate": 2.401581926502814e-06,
+      "loss": 0.5553,
+      "reward": 0.06248741149902344,
+      "reward_std": 0.01132394474479952,
+      "rewards/reward_func_1": 0.06248741149902344,
+      "step": 2965
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7984944212931846,
+      "grad_norm": 2.8427843972167466e-06,
+      "kl": 14.0859375,
+      "learning_rate": 2.371155193137662e-06,
+      "loss": 0.5634,
+      "reward": 0.06317214965820313,
+      "reward_std": 0.011330111461211346,
+      "rewards/reward_func_1": 0.06317214965820313,
+      "step": 2970
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.7998386879956983,
+      "grad_norm": 3.5108828342345078e-06,
+      "kl": 13.83125,
+      "learning_rate": 2.3408964895732433e-06,
+      "loss": 0.5533,
+      "reward": 0.05587625503540039,
+      "reward_std": 0.009793595474184258,
+      "rewards/reward_func_1": 0.05587625503540039,
+      "step": 2975
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8011829546982121,
+      "grad_norm": 6.495894467661856e-06,
+      "kl": 13.9984375,
+      "learning_rate": 2.310806482275336e-06,
+      "loss": 0.5598,
+      "reward": 0.05911798477172851,
+      "reward_std": 0.010464892169329687,
+      "rewards/reward_func_1": 0.05911798477172851,
+      "step": 2980
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8025272214007259,
+      "grad_norm": 1.9022724018213921e-06,
+      "kl": 14.0625,
+      "learning_rate": 2.2808858339940696e-06,
+      "loss": 0.5627,
+      "reward": 0.06507339477539062,
+      "reward_std": 0.010237712813250255,
+      "rewards/reward_func_1": 0.06507339477539062,
+      "step": 2985
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8038714881032397,
+      "grad_norm": 8.21357753011398e-06,
+      "kl": 14.1390625,
+      "learning_rate": 2.251135203749353e-06,
+      "loss": 0.5655,
+      "reward": 0.054758310317993164,
+      "reward_std": 0.010143174163385994,
+      "rewards/reward_func_1": 0.054758310317993164,
+      "step": 2990
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8052157548057535,
+      "grad_norm": 2.0820609734073514e-06,
+      "kl": 13.9046875,
+      "learning_rate": 2.221555246816335e-06,
+      "loss": 0.5563,
+      "reward": 0.05632228851318359,
+      "reward_std": 0.007249694373967941,
+      "rewards/reward_func_1": 0.05632228851318359,
+      "step": 2995
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8065600215082672,
+      "grad_norm": 3.5432799450063612e-06,
+      "kl": 13.8859375,
+      "learning_rate": 2.1921466147109995e-06,
+      "loss": 0.555,
+      "reward": 0.06329879760742188,
+      "reward_std": 0.011629640086903236,
+      "rewards/reward_func_1": 0.06329879760742188,
+      "step": 3000
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.807904288210781,
+      "grad_norm": 5.990676982037257e-06,
+      "kl": 13.85,
+      "learning_rate": 2.162909955175786e-06,
+      "loss": 0.5541,
+      "reward": 0.059543299674987796,
+      "reward_std": 0.011717213732481468,
+      "rewards/reward_func_1": 0.059543299674987796,
+      "step": 3005
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8092485549132948,
+      "grad_norm": 3.0730845992366085e-06,
+      "kl": 14.1765625,
+      "learning_rate": 2.1338459121653467e-06,
+      "loss": 0.5671,
+      "reward": 0.0583465576171875,
+      "reward_std": 0.010720923148619476,
+      "rewards/reward_func_1": 0.0583465576171875,
+      "step": 3010
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8105928216158086,
+      "grad_norm": 4.65749917566427e-06,
+      "kl": 13.7890625,
+      "learning_rate": 2.1049551258323466e-06,
+      "loss": 0.5514,
+      "reward": 0.05720829963684082,
+      "reward_std": 0.01134266530716559,
+      "rewards/reward_func_1": 0.05720829963684082,
+      "step": 3015
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8119370883183223,
+      "grad_norm": 1.0969602044497151e-05,
+      "kl": 13.865625,
+      "learning_rate": 2.076238232513377e-06,
+      "loss": 0.5545,
+      "reward": 0.05492105484008789,
+      "reward_std": 0.009651319341355703,
+      "rewards/reward_func_1": 0.05492105484008789,
+      "step": 3020
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8132813550208361,
+      "grad_norm": 2.167436605304829e-06,
+      "kl": 14.128125,
+      "learning_rate": 2.0476958647149235e-06,
+      "loss": 0.5653,
+      "reward": 0.062408828735351564,
+      "reward_std": 0.011062738049804465,
+      "rewards/reward_func_1": 0.062408828735351564,
+      "step": 3025
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.81462562172335,
+      "grad_norm": 6.903650046297116e-06,
+      "kl": 13.81875,
+      "learning_rate": 2.019328651099458e-06,
+      "loss": 0.5526,
+      "reward": 0.06055660247802734,
+      "reward_std": 0.01089865797512175,
+      "rewards/reward_func_1": 0.06055660247802734,
+      "step": 3030
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8159698884258637,
+      "grad_norm": 9.66745847108541e-06,
+      "kl": 13.9421875,
+      "learning_rate": 1.9911372164715617e-06,
+      "loss": 0.558,
+      "reward": 0.060862159729003905,
+      "reward_std": 0.010128252705180784,
+      "rewards/reward_func_1": 0.060862159729003905,
+      "step": 3035
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8173141551283775,
+      "grad_norm": 2.5966969587898348e-06,
+      "kl": 13.865625,
+      "learning_rate": 1.963122181764194e-06,
+      "loss": 0.5547,
+      "reward": 0.05717315673828125,
+      "reward_std": 0.010695815431245138,
+      "rewards/reward_func_1": 0.05717315673828125,
+      "step": 3040
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8186584218308912,
+      "grad_norm": 1.2404716471792199e-05,
+      "kl": 13.609375,
+      "learning_rate": 1.935284164024995e-06,
+      "loss": 0.5443,
+      "reward": 0.05703325271606445,
+      "reward_std": 0.00922305959957157,
+      "rewards/reward_func_1": 0.05703325271606445,
+      "step": 3045
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8200026885334051,
+      "grad_norm": 3.450983740549418e-06,
+      "kl": 13.8828125,
+      "learning_rate": 1.9076237764027096e-06,
+      "loss": 0.5555,
+      "reward": 0.060849010944366455,
+      "reward_std": 0.010863185320158664,
+      "rewards/reward_func_1": 0.060849010944366455,
+      "step": 3050
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8213469552359188,
+      "grad_norm": 2.398985998297576e-05,
+      "kl": 14.0265625,
+      "learning_rate": 1.8801416281336593e-06,
+      "loss": 0.5611,
+      "reward": 0.05967788696289063,
+      "reward_std": 0.011869262975233141,
+      "rewards/reward_func_1": 0.05967788696289063,
+      "step": 3055
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8226912219384326,
+      "grad_norm": 2.969979050249094e-06,
+      "kl": 13.9078125,
+      "learning_rate": 1.8528383245283565e-06,
+      "loss": 0.5565,
+      "reward": 0.05824851989746094,
+      "reward_std": 0.008635234561734251,
+      "rewards/reward_func_1": 0.05824851989746094,
+      "step": 3060
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8240354886409463,
+      "grad_norm": 4.690632522397209e-06,
+      "kl": 13.828125,
+      "learning_rate": 1.8257144669581405e-06,
+      "loss": 0.5533,
+      "reward": 0.06130073070526123,
+      "reward_std": 0.011460411777079571,
+      "rewards/reward_func_1": 0.06130073070526123,
+      "step": 3065
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8253797553434602,
+      "grad_norm": 2.5304858354502358e-05,
+      "kl": 13.7609375,
+      "learning_rate": 1.7987706528419547e-06,
+      "loss": 0.5505,
+      "reward": 0.058181381225585936,
+      "reward_std": 0.010406417903141119,
+      "rewards/reward_func_1": 0.058181381225585936,
+      "step": 3070
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.826724022045974,
+      "grad_norm": 3.5739954000746366e-06,
+      "kl": 13.9765625,
+      "learning_rate": 1.7720074756331796e-06,
+      "loss": 0.5591,
+      "reward": 0.06419677734375,
+      "reward_std": 0.010990058263996617,
+      "rewards/reward_func_1": 0.06419677734375,
+      "step": 3075
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8280682887484877,
+      "grad_norm": 2.814767640302307e-06,
+      "kl": 13.90625,
+      "learning_rate": 1.745425524806552e-06,
+      "loss": 0.5562,
+      "reward": 0.06376209259033203,
+      "reward_std": 0.009371502423891797,
+      "rewards/reward_func_1": 0.06376209259033203,
+      "step": 3080
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8294125554510015,
+      "grad_norm": 4.085266937181586e-06,
+      "kl": 14.1734375,
+      "learning_rate": 1.7190253858452032e-06,
+      "loss": 0.5674,
+      "reward": 0.07096824645996094,
+      "reward_std": 0.011460935025388608,
+      "rewards/reward_func_1": 0.07096824645996094,
+      "step": 3085
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8307568221535152,
+      "grad_norm": 2.656307515280787e-06,
+      "kl": 13.6828125,
+      "learning_rate": 1.6928076402277404e-06,
+      "loss": 0.5474,
+      "reward": 0.060984134674072266,
+      "reward_std": 0.011912760638369945,
+      "rewards/reward_func_1": 0.060984134674072266,
+      "step": 3090
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8321010888560291,
+      "grad_norm": 7.214829565782566e-06,
+      "kl": 13.8453125,
+      "learning_rate": 1.666772865415458e-06,
+      "loss": 0.5536,
+      "reward": 0.061499595642089844,
+      "reward_std": 0.010504865943221375,
+      "rewards/reward_func_1": 0.061499595642089844,
+      "step": 3095
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8334453555585428,
+      "grad_norm": 2.9270854611240793e-06,
+      "kl": 13.5703125,
+      "learning_rate": 1.640921634839605e-06,
+      "loss": 0.5428,
+      "reward": 0.05861034393310547,
+      "reward_std": 0.012625352442410077,
+      "rewards/reward_func_1": 0.05861034393310547,
+      "step": 3100
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8347896222610566,
+      "grad_norm": 3.179326768076862e-06,
+      "kl": 14.1953125,
+      "learning_rate": 1.6152545178887657e-06,
+      "loss": 0.568,
+      "reward": 0.055633163452148436,
+      "reward_std": 0.012100516646751203,
+      "rewards/reward_func_1": 0.055633163452148436,
+      "step": 3105
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8361338889635703,
+      "grad_norm": 4.799480848305393e-06,
+      "kl": 13.778125,
+      "learning_rate": 1.5897720798963079e-06,
+      "loss": 0.5512,
+      "reward": 0.06492023468017578,
+      "reward_std": 0.010558164384565315,
+      "rewards/reward_func_1": 0.06492023468017578,
+      "step": 3110
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8374781556660842,
+      "grad_norm": 3.970403668063227e-06,
+      "kl": 13.9,
+      "learning_rate": 1.5644748821279409e-06,
+      "loss": 0.5563,
+      "reward": 0.05761244297027588,
+      "reward_std": 0.01134748296753969,
+      "rewards/reward_func_1": 0.05761244297027588,
+      "step": 3115
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8388224223685979,
+      "grad_norm": 1.801868620532332e-06,
+      "kl": 14.128125,
+      "learning_rate": 1.5393634817693437e-06,
+      "loss": 0.5652,
+      "reward": 0.06230869293212891,
+      "reward_std": 0.012545625171333086,
+      "rewards/reward_func_1": 0.06230869293212891,
+      "step": 3120
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8401666890711117,
+      "grad_norm": 2.9992136205692077e-06,
+      "kl": 13.9578125,
+      "learning_rate": 1.514438431913907e-06,
+      "loss": 0.5582,
+      "reward": 0.06606597900390625,
+      "reward_std": 0.010844698862638325,
+      "rewards/reward_func_1": 0.06606597900390625,
+      "step": 3125
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8415109557736254,
+      "grad_norm": 5.0372464102110825e-06,
+      "kl": 14.0875,
+      "learning_rate": 1.4897002815505314e-06,
+      "loss": 0.5638,
+      "reward": 0.06385841369628906,
+      "reward_std": 0.011049523478141055,
+      "rewards/reward_func_1": 0.06385841369628906,
+      "step": 3130
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8428552224761393,
+      "grad_norm": 7.81911876401864e-06,
+      "kl": 13.8375,
+      "learning_rate": 1.4651495755515522e-06,
+      "loss": 0.5535,
+      "reward": 0.06193408966064453,
+      "reward_std": 0.010152038796877604,
+      "rewards/reward_func_1": 0.06193408966064453,
+      "step": 3135
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8441994891786531,
+      "grad_norm": 4.062687366968021e-05,
+      "kl": 14.0359375,
+      "learning_rate": 1.4407868546607319e-06,
+      "loss": 0.5615,
+      "reward": 0.06457939147949218,
+      "reward_std": 0.012157779483823105,
+      "rewards/reward_func_1": 0.06457939147949218,
+      "step": 3140
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8455437558811668,
+      "grad_norm": 2.0320292151154717e-06,
+      "kl": 13.8375,
+      "learning_rate": 1.4166126554813508e-06,
+      "loss": 0.5534,
+      "reward": 0.055645179748535153,
+      "reward_std": 0.009349037745414535,
+      "rewards/reward_func_1": 0.055645179748535153,
+      "step": 3145
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8468880225836806,
+      "grad_norm": 3.012095476151444e-06,
+      "kl": 13.8390625,
+      "learning_rate": 1.3926275104643816e-06,
+      "loss": 0.5534,
+      "reward": 0.06417160034179688,
+      "reward_std": 0.011896352579060476,
+      "rewards/reward_func_1": 0.06417160034179688,
+      "step": 3150
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8482322892861944,
+      "grad_norm": 8.71294741955353e-06,
+      "kl": 13.71875,
+      "learning_rate": 1.3688319478967772e-06,
+      "loss": 0.5486,
+      "reward": 0.057692861557006835,
+      "reward_std": 0.008569657542102505,
+      "rewards/reward_func_1": 0.057692861557006835,
+      "step": 3155
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8495765559887082,
+      "grad_norm": 2.56103658102802e-06,
+      "kl": 13.7125,
+      "learning_rate": 1.345226491889815e-06,
+      "loss": 0.5482,
+      "reward": 0.05779485702514649,
+      "reward_std": 0.011118789602187462,
+      "rewards/reward_func_1": 0.05779485702514649,
+      "step": 3160
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8509208226912219,
+      "grad_norm": 2.6276434255123604e-06,
+      "kl": 13.7640625,
+      "learning_rate": 1.3218116623675737e-06,
+      "loss": 0.5509,
+      "reward": 0.055435562133789064,
+      "reward_std": 0.010689648687912268,
+      "rewards/reward_func_1": 0.055435562133789064,
+      "step": 3165
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8522650893937357,
+      "grad_norm": 3.175247911713086e-06,
+      "kl": 13.7875,
+      "learning_rate": 1.298587975055462e-06,
+      "loss": 0.5517,
+      "reward": 0.058620452880859375,
+      "reward_std": 0.012298734129581134,
+      "rewards/reward_func_1": 0.058620452880859375,
+      "step": 3170
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8536093560962495,
+      "grad_norm": 6.266296622925438e-06,
+      "kl": 13.953125,
+      "learning_rate": 1.2755559414688766e-06,
+      "loss": 0.5581,
+      "reward": 0.05827016830444336,
+      "reward_std": 0.011117468139855192,
+      "rewards/reward_func_1": 0.05827016830444336,
+      "step": 3175
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8549536227987633,
+      "grad_norm": 1.6521969882887788e-05,
+      "kl": 13.865625,
+      "learning_rate": 1.2527160689019202e-06,
+      "loss": 0.5546,
+      "reward": 0.05954210758209229,
+      "reward_std": 0.009821428551731515,
+      "rewards/reward_func_1": 0.05954210758209229,
+      "step": 3180
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.856297889501277,
+      "grad_norm": 3.3688741041260073e-06,
+      "kl": 13.9859375,
+      "learning_rate": 1.2300688604162458e-06,
+      "loss": 0.5597,
+      "reward": 0.05932321548461914,
+      "reward_std": 0.01184624767920468,
+      "rewards/reward_func_1": 0.05932321548461914,
+      "step": 3185
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8576421562037908,
+      "grad_norm": 2.135618524334859e-05,
+      "kl": 14.153125,
+      "learning_rate": 1.207614814829956e-06,
+      "loss": 0.5663,
+      "reward": 0.060421180725097653,
+      "reward_std": 0.010734908378799446,
+      "rewards/reward_func_1": 0.060421180725097653,
+      "step": 3190
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8589864229063046,
+      "grad_norm": 3.3278847695328295e-06,
+      "kl": 13.675,
+      "learning_rate": 1.1853544267066353e-06,
+      "loss": 0.547,
+      "reward": 0.05666627883911133,
+      "reward_std": 0.009691183110044221,
+      "rewards/reward_func_1": 0.05666627883911133,
+      "step": 3195
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8603306896088184,
+      "grad_norm": 2.5311237550340593e-05,
+      "kl": 13.7171875,
+      "learning_rate": 1.1632881863444412e-06,
+      "loss": 0.5485,
+      "reward": 0.0567962646484375,
+      "reward_std": 0.010639874120533932,
+      "rewards/reward_func_1": 0.0567962646484375,
+      "step": 3200
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8616749563113322,
+      "grad_norm": 3.2120556170411874e-06,
+      "kl": 13.6140625,
+      "learning_rate": 1.141416579765321e-06,
+      "loss": 0.5443,
+      "reward": 0.05926952362060547,
+      "reward_std": 0.011298176337732002,
+      "rewards/reward_func_1": 0.05926952362060547,
+      "step": 3205
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8630192230138459,
+      "grad_norm": 1.2166214219178073e-05,
+      "kl": 13.803125,
+      "learning_rate": 1.1197400887042876e-06,
+      "loss": 0.552,
+      "reward": 0.05761222839355469,
+      "reward_std": 0.01062401667368249,
+      "rewards/reward_func_1": 0.05761222839355469,
+      "step": 3210
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8643634897163597,
+      "grad_norm": 8.964575499703642e-06,
+      "kl": 13.70625,
+      "learning_rate": 1.0982591905988304e-06,
+      "loss": 0.5486,
+      "reward": 0.05393571853637695,
+      "reward_std": 0.01401166350406129,
+      "rewards/reward_func_1": 0.05393571853637695,
+      "step": 3215
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8657077564188735,
+      "grad_norm": 3.3451262879680144e-06,
+      "kl": 13.8015625,
+      "learning_rate": 1.076974358578381e-06,
+      "loss": 0.5522,
+      "reward": 0.05769004821777344,
+      "reward_std": 0.010371179192588897,
+      "rewards/reward_func_1": 0.05769004821777344,
+      "step": 3220
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8670520231213873,
+      "grad_norm": 5.8671166698331945e-06,
+      "kl": 13.8546875,
+      "learning_rate": 1.0558860614539013e-06,
+      "loss": 0.554,
+      "reward": 0.06081085205078125,
+      "reward_std": 0.01084117493883241,
+      "rewards/reward_func_1": 0.06081085205078125,
+      "step": 3225
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.868396289823901,
+      "grad_norm": 9.06308378034737e-06,
+      "kl": 13.728125,
+      "learning_rate": 1.034994763707562e-06,
+      "loss": 0.5495,
+      "reward": 0.058400535583496095,
+      "reward_std": 0.010280990242608822,
+      "rewards/reward_func_1": 0.058400535583496095,
+      "step": 3230
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8697405565264148,
+      "grad_norm": 2.764769988061744e-06,
+      "kl": 13.7484375,
+      "learning_rate": 1.014300925482501e-06,
+      "loss": 0.5501,
+      "reward": 0.06383857727050782,
+      "reward_std": 0.011695932724978774,
+      "rewards/reward_func_1": 0.06383857727050782,
+      "step": 3235
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8710848232289287,
+      "grad_norm": 4.055384579260135e-06,
+      "kl": 14.325,
+      "learning_rate": 9.93805002572692e-07,
+      "loss": 0.5734,
+      "reward": 0.06781425476074218,
+      "reward_std": 0.010846901237528073,
+      "rewards/reward_func_1": 0.06781425476074218,
+      "step": 3240
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8724290899314424,
+      "grad_norm": 3.197312707925448e-06,
+      "kl": 13.928125,
+      "learning_rate": 9.735074464129156e-07,
+      "loss": 0.5572,
+      "reward": 0.05835247039794922,
+      "reward_std": 0.011373498971806839,
+      "rewards/reward_func_1": 0.05835247039794922,
+      "step": 3245
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8737733566339562,
+      "grad_norm": 2.8454533094190992e-06,
+      "kl": 13.928125,
+      "learning_rate": 9.534087040687978e-07,
+      "loss": 0.5575,
+      "reward": 0.058566713333129884,
+      "reward_std": 0.012191201363748405,
+      "rewards/reward_func_1": 0.058566713333129884,
+      "step": 3250
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8751176233364699,
+      "grad_norm": 6.40214238956105e-06,
+      "kl": 13.8328125,
+      "learning_rate": 9.335092182269823e-07,
+      "loss": 0.5531,
+      "reward": 0.05842547416687012,
+      "reward_std": 0.013222923100693151,
+      "rewards/reward_func_1": 0.05842547416687012,
+      "step": 3255
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8764618900389838,
+      "grad_norm": 3.3787041502364445e-06,
+      "kl": 13.7484375,
+      "learning_rate": 9.138094271853626e-07,
+      "loss": 0.5499,
+      "reward": 0.057961654663085935,
+      "reward_std": 0.010742506683163811,
+      "rewards/reward_func_1": 0.057961654663085935,
+      "step": 3260
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8778061567414975,
+      "grad_norm": 3.2537961942580296e-06,
+      "kl": 13.9765625,
+      "learning_rate": 8.943097648434451e-07,
+      "loss": 0.5591,
+      "reward": 0.055088233947753903,
+      "reward_std": 0.011486923421034589,
+      "rewards/reward_func_1": 0.055088233947753903,
+      "step": 3265
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8791504234440113,
+      "grad_norm": 4.171860564383678e-05,
+      "kl": 13.75,
+      "learning_rate": 8.750106606927756e-07,
+      "loss": 0.5501,
+      "reward": 0.06052291393280029,
+      "reward_std": 0.010141026746714488,
+      "rewards/reward_func_1": 0.06052291393280029,
+      "step": 3270
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.880494690146525,
+      "grad_norm": 5.266811058390886e-06,
+      "kl": 13.8078125,
+      "learning_rate": 8.559125398074941e-07,
+      "loss": 0.5522,
+      "reward": 0.06266212463378906,
+      "reward_std": 0.011405433918116614,
+      "rewards/reward_func_1": 0.06266212463378906,
+      "step": 3275
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8818389568490389,
+      "grad_norm": 1.9777207853621803e-06,
+      "kl": 13.625,
+      "learning_rate": 8.370158228349611e-07,
+      "loss": 0.5449,
+      "reward": 0.059973645210266116,
+      "reward_std": 0.01014815697853919,
+      "rewards/reward_func_1": 0.059973645210266116,
+      "step": 3280
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8831832235515527,
+      "grad_norm": 3.1561939977109432e-06,
+      "kl": 14.0640625,
+      "learning_rate": 8.18320925986501e-07,
+      "loss": 0.5626,
+      "reward": 0.061003684997558594,
+      "reward_std": 0.011469524375570472,
+      "rewards/reward_func_1": 0.061003684997558594,
+      "step": 3285
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8845274902540664,
+      "grad_norm": 3.982539965363685e-06,
+      "kl": 13.9203125,
+      "learning_rate": 7.998282610282282e-07,
+      "loss": 0.5569,
+      "reward": 0.05995340347290039,
+      "reward_std": 0.009807140480552335,
+      "rewards/reward_func_1": 0.05995340347290039,
+      "step": 3290
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8858717569565802,
+      "grad_norm": 5.228612280916423e-06,
+      "kl": 13.903125,
+      "learning_rate": 7.815382352719836e-07,
+      "loss": 0.5559,
+      "reward": 0.06256370544433594,
+      "reward_std": 0.010237712755042594,
+      "rewards/reward_func_1": 0.06256370544433594,
+      "step": 3295
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8872160236590939,
+      "grad_norm": 3.3338096727675293e-06,
+      "kl": 13.9515625,
+      "learning_rate": 7.63451251566355e-07,
+      "loss": 0.5578,
+      "reward": 0.059042739868164065,
+      "reward_std": 0.011339801916619763,
+      "rewards/reward_func_1": 0.059042739868164065,
+      "step": 3300
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8885602903616078,
+      "grad_norm": 1.969355253095273e-06,
+      "kl": 13.7625,
+      "learning_rate": 7.455677082878144e-07,
+      "loss": 0.5507,
+      "reward": 0.05534934997558594,
+      "reward_std": 0.011414243758190423,
+      "rewards/reward_func_1": 0.05534934997558594,
+      "step": 3305
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8899045570641215,
+      "grad_norm": 2.7435919491836103e-06,
+      "kl": 13.825,
+      "learning_rate": 7.278879993319399e-07,
+      "loss": 0.5528,
+      "reward": 0.0598332405090332,
+      "reward_std": 0.012269772328363616,
+      "rewards/reward_func_1": 0.0598332405090332,
+      "step": 3310
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8912488237666353,
+      "grad_norm": 1.6360121435354813e-06,
+      "kl": 13.8203125,
+      "learning_rate": 7.104125141047314e-07,
+      "loss": 0.5529,
+      "reward": 0.06058578491210938,
+      "reward_std": 0.0112239549322112,
+      "rewards/reward_func_1": 0.06058578491210938,
+      "step": 3315
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.892593090469149,
+      "grad_norm": 5.12127780893934e-06,
+      "kl": 14.0453125,
+      "learning_rate": 6.931416375140465e-07,
+      "loss": 0.5618,
+      "reward": 0.05910205841064453,
+      "reward_std": 0.010322065434593242,
+      "rewards/reward_func_1": 0.05910205841064453,
+      "step": 3320
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8939373571716629,
+      "grad_norm": 1.6029716789489612e-05,
+      "kl": 13.6296875,
+      "learning_rate": 6.760757499611193e-07,
+      "loss": 0.5452,
+      "reward": 0.05997223854064941,
+      "reward_std": 0.010195591623778455,
+      "rewards/reward_func_1": 0.05997223854064941,
+      "step": 3325
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8952816238741766,
+      "grad_norm": 9.6236435638275e-06,
+      "kl": 13.7984375,
+      "learning_rate": 6.592152273321706e-07,
+      "loss": 0.5521,
+      "reward": 0.051597309112548825,
+      "reward_std": 0.011377132889901987,
+      "rewards/reward_func_1": 0.051597309112548825,
+      "step": 3330
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8966258905766904,
+      "grad_norm": 3.594194595279987e-06,
+      "kl": 13.5984375,
+      "learning_rate": 6.425604409901454e-07,
+      "loss": 0.5443,
+      "reward": 0.06611251831054688,
+      "reward_std": 0.009431628473248566,
+      "rewards/reward_func_1": 0.06611251831054688,
+      "step": 3335
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.8979701572792042,
+      "grad_norm": 5.3717056289315224e-05,
+      "kl": 14.0,
+      "learning_rate": 6.261117577665254e-07,
+      "loss": 0.5599,
+      "reward": 0.05563621520996094,
+      "reward_std": 0.011836006561861723,
+      "rewards/reward_func_1": 0.05563621520996094,
+      "step": 3340
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.899314423981718,
+      "grad_norm": 2.7849052912642946e-06,
+      "kl": 14.0578125,
+      "learning_rate": 6.098695399532451e-07,
+      "loss": 0.5625,
+      "reward": 0.06023540496826172,
+      "reward_std": 0.012114391791692469,
+      "rewards/reward_func_1": 0.06023540496826172,
+      "step": 3345
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9006586906842318,
+      "grad_norm": 2.9332081794564147e-06,
+      "kl": 14.121875,
+      "learning_rate": 5.938341452947227e-07,
+      "loss": 0.5648,
+      "reward": 0.059270381927490234,
+      "reward_std": 0.010218441683537093,
+      "rewards/reward_func_1": 0.059270381927490234,
+      "step": 3350
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9020029573867455,
+      "grad_norm": 2.1531218408199493e-06,
+      "kl": 13.9484375,
+      "learning_rate": 5.780059269799676e-07,
+      "loss": 0.5583,
+      "reward": 0.06291056871414184,
+      "reward_std": 0.013561049330746755,
+      "rewards/reward_func_1": 0.06291056871414184,
+      "step": 3355
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9033472240892593,
+      "grad_norm": 1.5980087482603267e-05,
+      "kl": 13.7625,
+      "learning_rate": 5.623852336348156e-07,
+      "loss": 0.5505,
+      "reward": 0.06374626159667969,
+      "reward_std": 0.011463577805261593,
+      "rewards/reward_func_1": 0.06374626159667969,
+      "step": 3360
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9046914907917731,
+      "grad_norm": 3.6081160033063497e-06,
+      "kl": 13.6828125,
+      "learning_rate": 5.469724093142359e-07,
+      "loss": 0.5474,
+      "reward": 0.058330869674682616,
+      "reward_std": 0.010767394045251422,
+      "rewards/reward_func_1": 0.058330869674682616,
+      "step": 3365
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9060357574942869,
+      "grad_norm": 3.7672652979381382e-06,
+      "kl": 14.1046875,
+      "learning_rate": 5.317677934947652e-07,
+      "loss": 0.5643,
+      "reward": 0.06058921813964844,
+      "reward_std": 0.01233749669700046,
+      "rewards/reward_func_1": 0.06058921813964844,
+      "step": 3370
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9073800241968006,
+      "grad_norm": 6.598625532205915e-06,
+      "kl": 13.7140625,
+      "learning_rate": 5.167717210670232e-07,
+      "loss": 0.5486,
+      "reward": 0.06335010528564453,
+      "reward_std": 0.01017670587534667,
+      "rewards/reward_func_1": 0.06335010528564453,
+      "step": 3375
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9087242908993144,
+      "grad_norm": 6.818109341111267e-06,
+      "kl": 14.1703125,
+      "learning_rate": 5.019845223283393e-07,
+      "loss": 0.5669,
+      "reward": 0.06114330291748047,
+      "reward_std": 0.012632400382426568,
+      "rewards/reward_func_1": 0.06114330291748047,
+      "step": 3380
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9100685576018283,
+      "grad_norm": 4.953264578944072e-06,
+      "kl": 13.8640625,
+      "learning_rate": 4.874065229754743e-07,
+      "loss": 0.5543,
+      "reward": 0.05895808935165405,
+      "reward_std": 0.009361990720572066,
+      "rewards/reward_func_1": 0.05895808935165405,
+      "step": 3385
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.911412824304342,
+      "grad_norm": 3.3497417462058365e-06,
+      "kl": 13.8671875,
+      "learning_rate": 4.730380440974536e-07,
+      "loss": 0.5548,
+      "reward": 0.05743751525878906,
+      "reward_std": 0.011488685388758312,
+      "rewards/reward_func_1": 0.05743751525878906,
+      "step": 3390
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9127570910068558,
+      "grad_norm": 1.907120349642355e-05,
+      "kl": 13.7640625,
+      "learning_rate": 4.588794021684861e-07,
+      "loss": 0.5505,
+      "reward": 0.05307474136352539,
+      "reward_std": 0.011330662002728786,
+      "rewards/reward_func_1": 0.05307474136352539,
+      "step": 3395
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9141013577093695,
+      "grad_norm": 1.0828506674442906e-05,
+      "kl": 13.784375,
+      "learning_rate": 4.4493090904100366e-07,
+      "loss": 0.5516,
+      "reward": 0.0565185546875,
+      "reward_std": 0.009959327296382981,
+      "rewards/reward_func_1": 0.0565185546875,
+      "step": 3400
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9154456244118833,
+      "grad_norm": 6.197794391482603e-06,
+      "kl": 13.771875,
+      "learning_rate": 4.3119287193878035e-07,
+      "loss": 0.5511,
+      "reward": 0.059843674302101135,
+      "reward_std": 0.013095743974554352,
+      "rewards/reward_func_1": 0.059843674302101135,
+      "step": 3405
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9167898911143971,
+      "grad_norm": 1.7243120282728341e-06,
+      "kl": 13.775,
+      "learning_rate": 4.176655934501783e-07,
+      "loss": 0.5513,
+      "reward": 0.060840415954589847,
+      "reward_std": 0.009967916857567616,
+      "rewards/reward_func_1": 0.060840415954589847,
+      "step": 3410
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9181341578169109,
+      "grad_norm": 3.116844027317711e-06,
+      "kl": 13.9265625,
+      "learning_rate": 4.04349371521473e-07,
+      "loss": 0.5572,
+      "reward": 0.06282119750976563,
+      "reward_std": 0.010909009316674202,
+      "rewards/reward_func_1": 0.06282119750976563,
+      "step": 3415
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9194784245194246,
+      "grad_norm": 3.4594206681504147e-06,
+      "kl": 13.4734375,
+      "learning_rate": 3.912444994503006e-07,
+      "loss": 0.539,
+      "reward": 0.055774879455566403,
+      "reward_std": 0.012623810911463806,
+      "rewards/reward_func_1": 0.055774879455566403,
+      "step": 3420
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9208226912219384,
+      "grad_norm": 4.940301550959703e-06,
+      "kl": 13.74375,
+      "learning_rate": 3.783512658791821e-07,
+      "loss": 0.55,
+      "reward": 0.05670597553253174,
+      "reward_std": 0.01125905594062715,
+      "rewards/reward_func_1": 0.05670597553253174,
+      "step": 3425
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9221669579244522,
+      "grad_norm": 5.229050202615326e-06,
+      "kl": 13.6734375,
+      "learning_rate": 3.6566995478918733e-07,
+      "loss": 0.547,
+      "reward": 0.059182238578796384,
+      "reward_std": 0.0114708733453881,
+      "rewards/reward_func_1": 0.059182238578796384,
+      "step": 3430
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.923511224626966,
+      "grad_norm": 4.601683485816466e-06,
+      "kl": 13.7171875,
+      "learning_rate": 3.5320084549365864e-07,
+      "loss": 0.5489,
+      "reward": 0.059515857696533205,
+      "reward_std": 0.01207662059168797,
+      "rewards/reward_func_1": 0.059515857696533205,
+      "step": 3435
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9248554913294798,
+      "grad_norm": 7.413936600642046e-06,
+      "kl": 14.0421875,
+      "learning_rate": 3.409442126320761e-07,
+      "loss": 0.5622,
+      "reward": 0.0631723403930664,
+      "reward_std": 0.011003713330137544,
+      "rewards/reward_func_1": 0.0631723403930664,
+      "step": 3440
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9261997580319935,
+      "grad_norm": 7.578887107229093e-06,
+      "kl": 14.0140625,
+      "learning_rate": 3.289003261639978e-07,
+      "loss": 0.5607,
+      "reward": 0.06122303009033203,
+      "reward_std": 0.008386581853119423,
+      "rewards/reward_func_1": 0.06122303009033203,
+      "step": 3445
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9275440247345074,
+      "grad_norm": 2.735734824454994e-06,
+      "kl": 13.5484375,
+      "learning_rate": 3.170694513631178e-07,
+      "loss": 0.5421,
+      "reward": 0.05534172058105469,
+      "reward_std": 0.01105921419657534,
+      "rewards/reward_func_1": 0.05534172058105469,
+      "step": 3450
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9288882914370211,
+      "grad_norm": 6.3263983065553475e-06,
+      "kl": 13.9046875,
+      "learning_rate": 3.054518488114211e-07,
+      "loss": 0.5563,
+      "reward": 0.054802989959716795,
+      "reward_std": 0.011252586312184575,
+      "rewards/reward_func_1": 0.054802989959716795,
+      "step": 3455
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9302325581395349,
+      "grad_norm": 1.8264050595462322e-05,
+      "kl": 13.9765625,
+      "learning_rate": 2.9404777439345e-07,
+      "loss": 0.5592,
+      "reward": 0.06266040802001953,
+      "reward_std": 0.011020671827282058,
+      "rewards/reward_func_1": 0.06266040802001953,
+      "step": 3460
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9315768248420486,
+      "grad_norm": 4.062319021613803e-06,
+      "kl": 13.3328125,
+      "learning_rate": 2.828574792906602e-07,
+      "loss": 0.5334,
+      "reward": 0.05159635543823242,
+      "reward_std": 0.012718734997179126,
+      "rewards/reward_func_1": 0.05159635543823242,
+      "step": 3465
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9329210915445625,
+      "grad_norm": 3.980569545092294e-06,
+      "kl": 14.04375,
+      "learning_rate": 2.718812099758927e-07,
+      "loss": 0.5621,
+      "reward": 0.059010887145996095,
+      "reward_std": 0.011960663207719335,
+      "rewards/reward_func_1": 0.059010887145996095,
+      "step": 3470
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9342653582470762,
+      "grad_norm": 7.070525043673115e-06,
+      "kl": 13.7671875,
+      "learning_rate": 2.61119208207945e-07,
+      "loss": 0.5507,
+      "reward": 0.06236776113510132,
+      "reward_std": 0.012535562692573877,
+      "rewards/reward_func_1": 0.06236776113510132,
+      "step": 3475
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.93560962494959,
+      "grad_norm": 1.0806640602822881e-05,
+      "kl": 14.009375,
+      "learning_rate": 2.5057171102624623e-07,
+      "loss": 0.5605,
+      "reward": 0.06075210571289062,
+      "reward_std": 0.010957903016242198,
+      "rewards/reward_func_1": 0.06075210571289062,
+      "step": 3480
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9369538916521037,
+      "grad_norm": 7.847236702218652e-06,
+      "kl": 13.5546875,
+      "learning_rate": 2.4023895074563266e-07,
+      "loss": 0.5421,
+      "reward": 0.05129318237304688,
+      "reward_std": 0.011580966626206645,
+      "rewards/reward_func_1": 0.05129318237304688,
+      "step": 3485
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9382981583546176,
+      "grad_norm": 1.2028275705233682e-05,
+      "kl": 13.9296875,
+      "learning_rate": 2.3012115495123944e-07,
+      "loss": 0.5573,
+      "reward": 0.060091400146484376,
+      "reward_std": 0.012295650782471057,
+      "rewards/reward_func_1": 0.060091400146484376,
+      "step": 3490
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9396424250571314,
+      "grad_norm": 4.723935944639379e-06,
+      "kl": 13.775,
+      "learning_rate": 2.2021854649347696e-07,
+      "loss": 0.551,
+      "reward": 0.06290969848632813,
+      "reward_std": 0.01066894597352075,
+      "rewards/reward_func_1": 0.06290969848632813,
+      "step": 3495
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9409866917596451,
+      "grad_norm": 2.703375912460615e-06,
+      "kl": 14.0421875,
+      "learning_rate": 2.105313434831302e-07,
+      "loss": 0.5619,
+      "reward": 0.061441230773925784,
+      "reward_std": 0.01024222782725701,
+      "rewards/reward_func_1": 0.061441230773925784,
+      "step": 3500
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9423309584621589,
+      "grad_norm": 5.110204710945254e-06,
+      "kl": 14.0078125,
+      "learning_rate": 2.0105975928655154e-07,
+      "loss": 0.5603,
+      "reward": 0.05853328704833984,
+      "reward_std": 0.011592859703523573,
+      "rewards/reward_func_1": 0.05853328704833984,
+      "step": 3505
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9436752251646727,
+      "grad_norm": 2.7313865302858176e-06,
+      "kl": 13.821875,
+      "learning_rate": 1.9180400252096332e-07,
+      "loss": 0.5529,
+      "reward": 0.05967512130737305,
+      "reward_std": 0.010000402305377066,
+      "rewards/reward_func_1": 0.05967512130737305,
+      "step": 3510
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9450194918671865,
+      "grad_norm": 5.0758940233208705e-06,
+      "kl": 14.1890625,
+      "learning_rate": 1.8276427704985944e-07,
+      "loss": 0.5674,
+      "reward": 0.06556577682495117,
+      "reward_std": 0.01243010827965918,
+      "rewards/reward_func_1": 0.06556577682495117,
+      "step": 3515
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9463637585697002,
+      "grad_norm": 4.3392617953941226e-05,
+      "kl": 13.828125,
+      "learning_rate": 1.7394078197851883e-07,
+      "loss": 0.5531,
+      "reward": 0.06168599128723144,
+      "reward_std": 0.011275436536379857,
+      "rewards/reward_func_1": 0.06168599128723144,
+      "step": 3520
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.947708025272214,
+      "grad_norm": 3.3501562484161695e-06,
+      "kl": 13.7171875,
+      "learning_rate": 1.6533371164961675e-07,
+      "loss": 0.5485,
+      "reward": 0.05614547729492188,
+      "reward_std": 0.012279793498601066,
+      "rewards/reward_func_1": 0.05614547729492188,
+      "step": 3525
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9490522919747277,
+      "grad_norm": 1.6229182620008942e-06,
+      "kl": 13.9203125,
+      "learning_rate": 1.569432556389494e-07,
+      "loss": 0.5568,
+      "reward": 0.059717750549316405,
+      "reward_std": 0.010331755940569565,
+      "rewards/reward_func_1": 0.059717750549316405,
+      "step": 3530
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9503965586772416,
+      "grad_norm": 4.219915808789665e-06,
+      "kl": 14.1,
+      "learning_rate": 1.4876959875125163e-07,
+      "loss": 0.5642,
+      "reward": 0.06116485595703125,
+      "reward_std": 0.010313475892326096,
+      "rewards/reward_func_1": 0.06116485595703125,
+      "step": 3535
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9517408253797554,
+      "grad_norm": 2.8452825517888414e-06,
+      "kl": 13.8171875,
+      "learning_rate": 1.4081292101613241e-07,
+      "loss": 0.5527,
+      "reward": 0.05729732513427734,
+      "reward_std": 0.010209962361841463,
+      "rewards/reward_func_1": 0.05729732513427734,
+      "step": 3540
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9530850920822691,
+      "grad_norm": 7.0745595621701796e-06,
+      "kl": 13.7015625,
+      "learning_rate": 1.3307339768410365e-07,
+      "loss": 0.5482,
+      "reward": 0.05454435348510742,
+      "reward_std": 0.011191799660446122,
+      "rewards/reward_func_1": 0.05454435348510742,
+      "step": 3545
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9544293587847829,
+      "grad_norm": 5.309683274390409e-06,
+      "kl": 14.0046875,
+      "learning_rate": 1.2555119922272762e-07,
+      "loss": 0.56,
+      "reward": 0.060787296295166014,
+      "reward_std": 0.011707495429436676,
+      "rewards/reward_func_1": 0.060787296295166014,
+      "step": 3550
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9557736254872967,
+      "grad_norm": 2.6409459223941667e-06,
+      "kl": 13.8515625,
+      "learning_rate": 1.182464913128556e-07,
+      "loss": 0.5541,
+      "reward": 0.05850715637207031,
+      "reward_std": 0.011938859339716145,
+      "rewards/reward_func_1": 0.05850715637207031,
+      "step": 3555
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9571178921898105,
+      "grad_norm": 2.5191229724441655e-05,
+      "kl": 13.9328125,
+      "learning_rate": 1.1115943484498292e-07,
+      "loss": 0.5573,
+      "reward": 0.060850906372070315,
+      "reward_std": 0.008408385679285858,
+      "rewards/reward_func_1": 0.060850906372070315,
+      "step": 3560
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9584621588923242,
+      "grad_norm": 2.427203526167432e-06,
+      "kl": 13.684375,
+      "learning_rate": 1.0429018591570195e-07,
+      "loss": 0.5472,
+      "reward": 0.062432861328125,
+      "reward_std": 0.011829619569471105,
+      "rewards/reward_func_1": 0.062432861328125,
+      "step": 3565
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.959806425594838,
+      "grad_norm": 6.474336259998381e-06,
+      "kl": 13.7390625,
+      "learning_rate": 9.7638895824268e-08,
+      "loss": 0.5494,
+      "reward": 0.06272506713867188,
+      "reward_std": 0.010429763507272583,
+      "rewards/reward_func_1": 0.06272506713867188,
+      "step": 3570
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9611506922973518,
+      "grad_norm": 3.807508164754836e-06,
+      "kl": 13.89375,
+      "learning_rate": 9.120571106926212e-08,
+      "loss": 0.5563,
+      "reward": 0.05861101150512695,
+      "reward_std": 0.012294219210161828,
+      "rewards/reward_func_1": 0.05861101150512695,
+      "step": 3575
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9624949589998656,
+      "grad_norm": 2.37896620092215e-06,
+      "kl": 14.1109375,
+      "learning_rate": 8.499077334536921e-08,
+      "loss": 0.5646,
+      "reward": 0.0572235107421875,
+      "reward_std": 0.01089623533844133,
+      "rewards/reward_func_1": 0.0572235107421875,
+      "step": 3580
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9638392257023793,
+      "grad_norm": 3.802741503022844e-06,
+      "kl": 13.753125,
+      "learning_rate": 7.899421954025266e-08,
+      "loss": 0.5501,
+      "reward": 0.0643655776977539,
+      "reward_std": 0.010973099654074758,
+      "rewards/reward_func_1": 0.0643655776977539,
+      "step": 3585
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9651834924048931,
+      "grad_norm": 5.4865031415829435e-06,
+      "kl": 13.7015625,
+      "learning_rate": 7.321618173154466e-08,
+      "loss": 0.5481,
+      "reward": 0.05659542083740234,
+      "reward_std": 0.011230782363782055,
+      "rewards/reward_func_1": 0.05659542083740234,
+      "step": 3590
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.966527759107407,
+      "grad_norm": 1.9297044673294295e-06,
+      "kl": 14.0859375,
+      "learning_rate": 6.765678718392843e-08,
+      "loss": 0.5633,
+      "reward": 0.0672616958618164,
+      "reward_std": 0.013510283493087628,
+      "rewards/reward_func_1": 0.0672616958618164,
+      "step": 3595
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9678720258099207,
+      "grad_norm": 3.233315283068805e-06,
+      "kl": 13.9375,
+      "learning_rate": 6.231615834634497e-08,
+      "loss": 0.5572,
+      "reward": 0.06126976013183594,
+      "reward_std": 0.010508609988391981,
+      "rewards/reward_func_1": 0.06126976013183594,
+      "step": 3600
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9692162925124345,
+      "grad_norm": 3.4369802506262204e-06,
+      "kl": 13.734375,
+      "learning_rate": 5.719441284929073e-08,
+      "loss": 0.5495,
+      "reward": 0.06183929443359375,
+      "reward_std": 0.009256316086975858,
+      "rewards/reward_func_1": 0.06183929443359375,
+      "step": 3605
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9705605592149482,
+      "grad_norm": 2.0105403564230073e-06,
+      "kl": 13.815625,
+      "learning_rate": 5.229166350222747e-08,
+      "loss": 0.5525,
+      "reward": 0.06485710144042969,
+      "reward_std": 0.010634588305765646,
+      "rewards/reward_func_1": 0.06485710144042969,
+      "step": 3610
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9719048259174621,
+      "grad_norm": 1.3547062735597137e-05,
+      "kl": 13.8625,
+      "learning_rate": 4.760801829109763e-08,
+      "loss": 0.5546,
+      "reward": 0.0634115219116211,
+      "reward_std": 0.010813204231817508,
+      "rewards/reward_func_1": 0.0634115219116211,
+      "step": 3615
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9732490926199758,
+      "grad_norm": 1.8916140334113152e-06,
+      "kl": 13.721875,
+      "learning_rate": 4.3143580375945016e-08,
+      "loss": 0.549,
+      "reward": 0.05869293212890625,
+      "reward_std": 0.010853508513537235,
+      "rewards/reward_func_1": 0.05869293212890625,
+      "step": 3620
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9745933593224896,
+      "grad_norm": 4.1851594687614124e-06,
+      "kl": 13.4625,
+      "learning_rate": 3.889844808864451e-08,
+      "loss": 0.5387,
+      "reward": 0.059955787658691403,
+      "reward_std": 0.010277136050717672,
+      "rewards/reward_func_1": 0.059955787658691403,
+      "step": 3625
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9759376260250033,
+      "grad_norm": 7.070749688864453e-06,
+      "kl": 13.9234375,
+      "learning_rate": 3.487271493073596e-08,
+      "loss": 0.5572,
+      "reward": 0.0633173942565918,
+      "reward_std": 0.008260493339184905,
+      "rewards/reward_func_1": 0.0633173942565918,
+      "step": 3630
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9772818927275171,
+      "grad_norm": 2.236297405033838e-06,
+      "kl": 13.834375,
+      "learning_rate": 3.106646957136472e-08,
+      "loss": 0.5532,
+      "reward": 0.06124534606933594,
+      "reward_std": 0.011660033430234761,
+      "rewards/reward_func_1": 0.06124534606933594,
+      "step": 3635
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.978626159430031,
+      "grad_norm": 2.2983462258707732e-05,
+      "kl": 13.9046875,
+      "learning_rate": 2.7479795845324342e-08,
+      "loss": 0.5563,
+      "reward": 0.05750617980957031,
+      "reward_std": 0.01097970688406349,
+      "rewards/reward_func_1": 0.05750617980957031,
+      "step": 3640
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9799704261325447,
+      "grad_norm": 3.4399413380015176e-06,
+      "kl": 13.965625,
+      "learning_rate": 2.411277275121915e-08,
+      "loss": 0.5586,
+      "reward": 0.062087726593017575,
+      "reward_std": 0.016095589974429458,
+      "rewards/reward_func_1": 0.062087726593017575,
+      "step": 3645
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9813146928350585,
+      "grad_norm": 5.6823751037882175e-06,
+      "kl": 13.8625,
+      "learning_rate": 2.096547444971453e-08,
+      "loss": 0.5547,
+      "reward": 0.06416492462158203,
+      "reward_std": 0.012935893231770024,
+      "rewards/reward_func_1": 0.06416492462158203,
+      "step": 3650
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9826589595375722,
+      "grad_norm": 3.1565050448989496e-06,
+      "kl": 13.8734375,
+      "learning_rate": 1.8037970261909343e-08,
+      "loss": 0.5549,
+      "reward": 0.060312080383300784,
+      "reward_std": 0.01092442618610221,
+      "rewards/reward_func_1": 0.060312080383300784,
+      "step": 3655
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9840032262400861,
+      "grad_norm": 3.1137699352257187e-06,
+      "kl": 13.7796875,
+      "learning_rate": 1.533032466780826e-08,
+      "loss": 0.5516,
+      "reward": 0.06079998016357422,
+      "reward_std": 0.011588454757293221,
+      "rewards/reward_func_1": 0.06079998016357422,
+      "step": 3660
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9853474929425998,
+      "grad_norm": 1.9204537238692865e-05,
+      "kl": 13.83125,
+      "learning_rate": 1.2842597304901783e-08,
+      "loss": 0.5532,
+      "reward": 0.05525150299072266,
+      "reward_std": 0.008975948392617283,
+      "rewards/reward_func_1": 0.05525150299072266,
+      "step": 3665
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9866917596451136,
+      "grad_norm": 3.472564230833086e-06,
+      "kl": 13.6734375,
+      "learning_rate": 1.057484296684841e-08,
+      "loss": 0.5469,
+      "reward": 0.06324386596679688,
+      "reward_std": 0.013345763047982472,
+      "rewards/reward_func_1": 0.06324386596679688,
+      "step": 3670
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9880360263476273,
+      "grad_norm": 2.0865973056061193e-06,
+      "kl": 13.6015625,
+      "learning_rate": 8.527111602273375e-09,
+      "loss": 0.5439,
+      "reward": 0.057076644897460935,
+      "reward_std": 0.01272732454162906,
+      "rewards/reward_func_1": 0.057076644897460935,
+      "step": 3675
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9893802930501412,
+      "grad_norm": 5.3152875807427336e-06,
+      "kl": 13.7734375,
+      "learning_rate": 6.699448313668422e-09,
+      "loss": 0.5508,
+      "reward": 0.06054000854492188,
+      "reward_std": 0.010864520556788193,
+      "rewards/reward_func_1": 0.06054000854492188,
+      "step": 3680
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9907245597526549,
+      "grad_norm": 1.824535843297781e-06,
+      "kl": 14.0265625,
+      "learning_rate": 5.0918933563914866e-09,
+      "loss": 0.561,
+      "reward": 0.05926017761230469,
+      "reward_std": 0.011259193480873364,
+      "rewards/reward_func_1": 0.05926017761230469,
+      "step": 3685
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9920688264551687,
+      "grad_norm": 3.396847887415788e-06,
+      "kl": 13.934375,
+      "learning_rate": 3.7044821377896225e-09,
+      "loss": 0.5574,
+      "reward": 0.058501815795898436,
+      "reward_std": 0.010197188252641353,
+      "rewards/reward_func_1": 0.058501815795898436,
+      "step": 3690
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9934130931576824,
+      "grad_norm": 2.0717141069326317e-06,
+      "kl": 13.6671875,
+      "learning_rate": 2.537245216410744e-09,
+      "loss": 0.5469,
+      "reward": 0.06255474090576171,
+      "reward_std": 0.010947992081491975,
+      "rewards/reward_func_1": 0.06255474090576171,
+      "step": 3695
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9947573598601963,
+      "grad_norm": 4.1668949961604085e-06,
+      "kl": 13.8796875,
+      "learning_rate": 1.590208301335272e-09,
+      "loss": 0.5552,
+      "reward": 0.06215476989746094,
+      "reward_std": 0.011439351307490141,
+      "rewards/reward_func_1": 0.06215476989746094,
+      "step": 3700
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9961016265627101,
+      "grad_norm": 4.461783646547701e-06,
+      "kl": 14.01875,
+      "learning_rate": 8.633922516110283e-10,
+      "loss": 0.5609,
+      "reward": 0.05896968841552734,
+      "reward_std": 0.01078853727231035,
+      "rewards/reward_func_1": 0.05896968841552734,
+      "step": 3705
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9974458932652238,
+      "grad_norm": 3.0964040433900664e-06,
+      "kl": 13.6203125,
+      "learning_rate": 3.568130757880539e-10,
+      "loss": 0.545,
+      "reward": 0.056448173522949216,
+      "reward_std": 0.009608041982573923,
+      "rewards/reward_func_1": 0.056448173522949216,
+      "step": 3710
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9987901599677376,
+      "grad_norm": 6.054201548977289e-06,
+      "kl": 13.7796875,
+      "learning_rate": 7.048193157221939e-11,
+      "loss": 0.5512,
+      "reward": 0.06598529815673829,
+      "reward_std": 0.011622372209239984,
+      "rewards/reward_func_1": 0.06598529815673829,
+      "step": 3715
+    },
+    {
+      "completion_length": 2.0,
+      "epoch": 0.9998655733297486,
+      "kl": 13.673828125,
+      "reward": 0.06168794631958008,
+      "reward_std": 0.012013631283480208,
+      "rewards/reward_func_1": 0.06168794631958008,
+      "step": 3719,
+      "total_flos": 0.0,
+      "train_loss": 604316576171.3622,
+      "train_runtime": 47153.6794,
+      "train_samples_per_second": 1.262,
+      "train_steps_per_second": 0.079
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3719,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}