diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6543 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5714285714285714,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 2253.854206085205,
+      "epoch": 0.001142857142857143,
+      "grad_norm": 0.028186120092868805,
+      "kl": 0.0,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "reward": 0.945640604943037,
+      "reward_std": 0.7231155578047037,
+      "rewards/cosine_scaled_reward": 0.16032031644135714,
+      "rewards/format_reward": 0.6250000037252903,
+      "step": 1
+    },
+    {
+      "completion_length": 2566.395854949951,
+      "epoch": 0.002285714285714286,
+      "grad_norm": 0.02438599243760109,
+      "kl": 0.0,
+      "learning_rate": 2e-08,
+      "loss": 0.0,
+      "reward": 0.7753396667540073,
+      "reward_std": 0.8076020702719688,
+      "rewards/cosine_scaled_reward": 0.12725313939154148,
+      "rewards/format_reward": 0.5208333414047956,
+      "step": 2
+    },
+    {
+      "completion_length": 2850.8333587646484,
+      "epoch": 0.0034285714285714284,
+      "grad_norm": 0.026358311995863914,
+      "kl": 0.00019162893295288086,
+      "learning_rate": 4e-08,
+      "loss": 0.0,
+      "reward": 0.25994101725518703,
+      "reward_std": 0.7457061782479286,
+      "rewards/cosine_scaled_reward": -0.07836283510550857,
+      "rewards/format_reward": 0.4166666753590107,
+      "step": 3
+    },
+    {
+      "completion_length": 1333.3750228881836,
+      "epoch": 0.004571428571428572,
+      "grad_norm": 0.03962448611855507,
+      "kl": 8.179247379302979e-05,
+      "learning_rate": 6e-08,
+      "loss": 0.0,
+      "reward": 1.1095947362482548,
+      "reward_std": 0.7611480094492435,
+      "rewards/cosine_scaled_reward": 0.10688067661249079,
+      "rewards/format_reward": 0.895833333954215,
+      "step": 4
+    },
+    {
+      "completion_length": 3136.312530517578,
+      "epoch": 0.005714285714285714,
+      "grad_norm": 0.02204746939241886,
+      "kl": 0.00017178058624267578,
+      "learning_rate": 8e-08,
+      "loss": 0.0,
+      "reward": 0.3529331162571907,
+      "reward_std": 0.9226996600627899,
+      "rewards/cosine_scaled_reward": -0.07353345490992069,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 5
+    },
+    {
+      "completion_length": 2268.208351135254,
+      "epoch": 0.006857142857142857,
+      "grad_norm": 0.02553640492260456,
+      "kl": 0.00011495500802993774,
+      "learning_rate": 1e-07,
+      "loss": 0.0,
+      "reward": 0.47384920963668264,
+      "reward_std": 0.9380720984190702,
+      "rewards/cosine_scaled_reward": -0.08599207969382405,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 6
+    },
+    {
+      "completion_length": 2404.916702270508,
+      "epoch": 0.008,
+      "grad_norm": 0.02169734612107277,
+      "kl": 0.0001201927661895752,
+      "learning_rate": 1.2e-07,
+      "loss": 0.0,
+      "reward": 0.7995812203735113,
+      "reward_std": 0.7245029136538506,
+      "rewards/cosine_scaled_reward": 0.00395727576687932,
+      "rewards/format_reward": 0.7916666716337204,
+      "step": 7
+    },
+    {
+      "completion_length": 1825.270881652832,
+      "epoch": 0.009142857142857144,
+      "grad_norm": 0.03927920013666153,
+      "kl": 6.420910358428955e-05,
+      "learning_rate": 1.4e-07,
+      "loss": 0.0,
+      "reward": 1.2986854314804077,
+      "reward_std": 0.694359702989459,
+      "rewards/cosine_scaled_reward": 0.2951760431751609,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 8
+    },
+    {
+      "completion_length": 2716.9375,
+      "epoch": 0.010285714285714285,
+      "grad_norm": 0.031250353902578354,
+      "kl": 0.00015756487846374512,
+      "learning_rate": 1.6e-07,
+      "loss": 0.0,
+      "reward": 0.4805310983210802,
+      "reward_std": 0.6337715331465006,
+      "rewards/cosine_scaled_reward": 0.021515536587685347,
+      "rewards/format_reward": 0.4375,
+      "step": 9
+    },
+    {
+      "completion_length": 2326.1458625793457,
+      "epoch": 0.011428571428571429,
+      "grad_norm": 0.02834513410925865,
+      "kl": 0.00010342895984649658,
+      "learning_rate": 1.8e-07,
+      "loss": 0.0,
+      "reward": 0.7450611963868141,
+      "reward_std": 0.7835288420319557,
+      "rewards/cosine_scaled_reward": 0.09128060017246753,
+      "rewards/format_reward": 0.5625000093132257,
+      "step": 10
+    },
+    {
+      "completion_length": 3093.3541870117188,
+      "epoch": 0.012571428571428572,
+      "grad_norm": 0.023482978343963623,
+      "kl": 0.00014767050743103027,
+      "learning_rate": 2e-07,
+      "loss": 0.0,
+      "reward": 0.3454847726970911,
+      "reward_std": 0.5903383195400238,
+      "rewards/cosine_scaled_reward": -0.0043409522622823715,
+      "rewards/format_reward": 0.35416666977107525,
+      "step": 11
+    },
+    {
+      "completion_length": 2152.541748046875,
+      "epoch": 0.013714285714285714,
+      "grad_norm": 0.028229378163814545,
+      "kl": 0.0001628100872039795,
+      "learning_rate": 2.1999999999999998e-07,
+      "loss": 0.0,
+      "reward": 0.7333892993628979,
+      "reward_std": 0.9000732153654099,
+      "rewards/cosine_scaled_reward": 0.0437779575586319,
+      "rewards/format_reward": 0.6458333488553762,
+      "step": 12
+    },
+    {
+      "completion_length": 2562.145866394043,
+      "epoch": 0.014857142857142857,
+      "grad_norm": 0.022799858823418617,
+      "kl": 0.00014079362154006958,
+      "learning_rate": 2.4e-07,
+      "loss": 0.0,
+      "reward": 0.5581501871347427,
+      "reward_std": 0.8053405769169331,
+      "rewards/cosine_scaled_reward": 0.018658424261957407,
+      "rewards/format_reward": 0.5208333432674408,
+      "step": 13
+    },
+    {
+      "completion_length": 2327.166679382324,
+      "epoch": 0.016,
+      "grad_norm": 0.025850726291537285,
+      "kl": 0.00013312697410583496,
+      "learning_rate": 2.6e-07,
+      "loss": 0.0,
+      "reward": 0.4661823809146881,
+      "reward_std": 0.7131906598806381,
+      "rewards/cosine_scaled_reward": -0.04815882258117199,
+      "rewards/format_reward": 0.5625000018626451,
+      "step": 14
+    },
+    {
+      "completion_length": 2491.4791717529297,
+      "epoch": 0.017142857142857144,
+      "grad_norm": 0.025151003152132034,
+      "kl": 0.00011346954852342606,
+      "learning_rate": 2.8e-07,
+      "loss": 0.0,
+      "reward": 0.5728016346693039,
+      "reward_std": 0.4929979182779789,
+      "rewards/cosine_scaled_reward": 0.036400796845555305,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 15
+    },
+    {
+      "completion_length": 3466.8958740234375,
+      "epoch": 0.018285714285714287,
+      "grad_norm": 0.01853562705218792,
+      "kl": 0.00017964839935302734,
+      "learning_rate": 3e-07,
+      "loss": 0.0,
+      "reward": 0.04977674409747124,
+      "reward_std": 0.8271799795329571,
+      "rewards/cosine_scaled_reward": -0.06886161956936121,
+      "rewards/format_reward": 0.1875,
+      "step": 16
+    },
+    {
+      "completion_length": 1964.208366394043,
+      "epoch": 0.019428571428571427,
+      "grad_norm": 0.039788853377103806,
+      "kl": 0.00011803209781646729,
+      "learning_rate": 3.2e-07,
+      "loss": 0.0,
+      "reward": 0.9614612013101578,
+      "reward_std": 0.5819262592121959,
+      "rewards/cosine_scaled_reward": 0.1786472648382187,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 17
+    },
+    {
+      "completion_length": 2595.437545776367,
+      "epoch": 0.02057142857142857,
+      "grad_norm": 0.01998710259795189,
+      "kl": 0.00012928247451782227,
+      "learning_rate": 3.4000000000000003e-07,
+      "loss": 0.0,
+      "reward": 0.7032174468040466,
+      "reward_std": 0.616347186267376,
+      "rewards/cosine_scaled_reward": 0.03910870919935405,
+      "rewards/format_reward": 0.625,
+      "step": 18
+    },
+    {
+      "completion_length": 2346.3542098999023,
+      "epoch": 0.021714285714285714,
+      "grad_norm": 0.02497478388249874,
+      "kl": 0.00013540685176849365,
+      "learning_rate": 3.6e-07,
+      "loss": 0.0,
+      "reward": 1.3847788944840431,
+      "reward_std": 1.1934307999908924,
+      "rewards/cosine_scaled_reward": 0.3382227774709463,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 19
+    },
+    {
+      "completion_length": 1532.4166984558105,
+      "epoch": 0.022857142857142857,
+      "grad_norm": 0.030825506895780563,
+      "kl": 7.693842053413391e-05,
+      "learning_rate": 3.7999999999999996e-07,
+      "loss": 0.0,
+      "reward": 1.009512271732092,
+      "reward_std": 0.6844168081879616,
+      "rewards/cosine_scaled_reward": 0.04642279585823417,
+      "rewards/format_reward": 0.916666679084301,
+      "step": 20
+    },
+    {
+      "completion_length": 2422.31254196167,
+      "epoch": 0.024,
+      "grad_norm": 0.032373763620853424,
+      "kl": 0.0001367330551147461,
+      "learning_rate": 4e-07,
+      "loss": 0.0,
+      "reward": 0.6270023807883263,
+      "reward_std": 0.6962997727096081,
+      "rewards/cosine_scaled_reward": 0.053084509272594005,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 21
+    },
+    {
+      "completion_length": 1439.500057220459,
+      "epoch": 0.025142857142857144,
+      "grad_norm": 0.03851747140288353,
+      "kl": 0.00011110305786132812,
+      "learning_rate": 4.1999999999999995e-07,
+      "loss": 0.0,
+      "reward": 0.7128502167761326,
+      "reward_std": 0.4558130237273872,
+      "rewards/cosine_scaled_reward": -0.09149156883358955,
+      "rewards/format_reward": 0.8958333358168602,
+      "step": 22
+    },
+    {
+      "completion_length": 2568.8750534057617,
+      "epoch": 0.026285714285714287,
+      "grad_norm": 0.027865994721651077,
+      "kl": 0.0001246929168701172,
+      "learning_rate": 4.3999999999999997e-07,
+      "loss": 0.0,
+      "reward": 0.39115861523896456,
+      "reward_std": 0.9601399153470993,
+      "rewards/cosine_scaled_reward": -0.05442069785203785,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 23
+    },
+    {
+      "completion_length": 1739.6875534057617,
+      "epoch": 0.027428571428571427,
+      "grad_norm": 0.02797676995396614,
+      "kl": 7.119029760360718e-05,
+      "learning_rate": 4.6e-07,
+      "loss": 0.0,
+      "reward": 0.8130059950053692,
+      "reward_std": 0.8428283482789993,
+      "rewards/cosine_scaled_reward": 0.02108631655573845,
+      "rewards/format_reward": 0.770833345130086,
+      "step": 24
+    },
+    {
+      "completion_length": 2243.5625381469727,
+      "epoch": 0.02857142857142857,
+      "grad_norm": 0.02834572270512581,
+      "kl": 0.00014713406562805176,
+      "learning_rate": 4.8e-07,
+      "loss": 0.0,
+      "reward": 0.7691881991922855,
+      "reward_std": 0.6148425601422787,
+      "rewards/cosine_scaled_reward": 0.10334407165646553,
+      "rewards/format_reward": 0.5625000018626451,
+      "step": 25
+    },
+    {
+      "completion_length": 2384.479202270508,
+      "epoch": 0.029714285714285714,
+      "grad_norm": 0.022393781691789627,
+      "kl": 0.00015068799257278442,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "reward": 0.4031391516327858,
+      "reward_std": 0.6596824564039707,
+      "rewards/cosine_scaled_reward": -0.06926377443596721,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 26
+    },
+    {
+      "completion_length": 2295.479202270508,
+      "epoch": 0.030857142857142857,
+      "grad_norm": 0.027790380641818047,
+      "kl": 0.0001430213451385498,
+      "learning_rate": 5.2e-07,
+      "loss": 0.0,
+      "reward": 0.7962075062096119,
+      "reward_std": 0.5945973135530949,
+      "rewards/cosine_scaled_reward": 0.0439370833337307,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 27
+    },
+    {
+      "completion_length": 2273.5000534057617,
+      "epoch": 0.032,
+      "grad_norm": 0.03035018779337406,
+      "kl": 0.00013148784637451172,
+      "learning_rate": 5.4e-07,
+      "loss": 0.0,
+      "reward": 0.9480000095209107,
+      "reward_std": 0.969454187899828,
+      "rewards/cosine_scaled_reward": 0.15108334203250706,
+      "rewards/format_reward": 0.6458333358168602,
+      "step": 28
+    },
+    {
+      "completion_length": 2891.291748046875,
+      "epoch": 0.03314285714285714,
+      "grad_norm": 0.024835970252752304,
+      "kl": 0.00016313791275024414,
+      "learning_rate": 5.6e-07,
+      "loss": 0.0,
+      "reward": 0.31715210899710655,
+      "reward_std": 0.7158625386655331,
+      "rewards/cosine_scaled_reward": -0.060173945501446724,
+      "rewards/format_reward": 0.43750000931322575,
+      "step": 29
+    },
+    {
+      "completion_length": 2167.875045776367,
+      "epoch": 0.03428571428571429,
+      "grad_norm": 0.023409895598888397,
+      "kl": 0.00010243058204650879,
+      "learning_rate": 5.8e-07,
+      "loss": 0.0,
+      "reward": 1.1814709827303886,
+      "reward_std": 0.9771759286522865,
+      "rewards/cosine_scaled_reward": 0.22615211736410856,
+      "rewards/format_reward": 0.7291666865348816,
+      "step": 30
+    },
+    {
+      "completion_length": 2741.916702270508,
+      "epoch": 0.03542857142857143,
+      "grad_norm": 0.020507322624325752,
+      "kl": 0.00014102458953857422,
+      "learning_rate": 6e-07,
+      "loss": 0.0,
+      "reward": 0.5180935077369213,
+      "reward_std": 0.8897004313766956,
+      "rewards/cosine_scaled_reward": 0.019463416654616594,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 31
+    },
+    {
+      "completion_length": 2284.687545776367,
+      "epoch": 0.036571428571428574,
+      "grad_norm": 0.02322392910718918,
+      "kl": 0.00012177228927612305,
+      "learning_rate": 6.2e-07,
+      "loss": 0.0,
+      "reward": 1.1874850988388062,
+      "reward_std": 0.6209478713572025,
+      "rewards/cosine_scaled_reward": 0.23957587592303753,
+      "rewards/format_reward": 0.7083333395421505,
+      "step": 32
+    },
+    {
+      "completion_length": 2879.3750610351562,
+      "epoch": 0.037714285714285714,
+      "grad_norm": 0.019906090572476387,
+      "kl": 0.00011980533599853516,
+      "learning_rate": 6.4e-07,
+      "loss": 0.0,
+      "reward": 0.8293444700539112,
+      "reward_std": 0.8636636873707175,
+      "rewards/cosine_scaled_reward": 0.11258888803422451,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 33
+    },
+    {
+      "completion_length": 1965.4375076293945,
+      "epoch": 0.038857142857142854,
+      "grad_norm": 0.02797047235071659,
+      "kl": 0.0001359265297651291,
+      "learning_rate": 6.6e-07,
+      "loss": 0.0,
+      "reward": 1.257663020864129,
+      "reward_std": 0.8084600828588009,
+      "rewards/cosine_scaled_reward": 0.2746648136526346,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 34
+    },
+    {
+      "completion_length": 2429.000045776367,
+      "epoch": 0.04,
+      "grad_norm": 0.03363242745399475,
+      "kl": 0.0001286640763282776,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0,
+      "reward": 0.9006999442353845,
+      "reward_std": 0.8478060588240623,
+      "rewards/cosine_scaled_reward": 0.15868327533826232,
+      "rewards/format_reward": 0.5833333395421505,
+      "step": 35
+    },
+    {
+      "completion_length": 2975.083366394043,
+      "epoch": 0.04114285714285714,
+      "grad_norm": 0.029229290783405304,
+      "kl": 0.00017908215522766113,
+      "learning_rate": 7e-07,
+      "loss": 0.0,
+      "reward": 0.09833091939799488,
+      "reward_std": 0.9010875336825848,
+      "rewards/cosine_scaled_reward": -0.14875120925717056,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 36
+    },
+    {
+      "completion_length": 2954.0833435058594,
+      "epoch": 0.04228571428571429,
+      "grad_norm": 0.019327132031321526,
+      "kl": 0.0001398026943206787,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0,
+      "reward": 0.2552360990084708,
+      "reward_std": 0.6516154278069735,
+      "rewards/cosine_scaled_reward": -0.04946526139974594,
+      "rewards/format_reward": 0.3541666679084301,
+      "step": 37
+    },
+    {
+      "completion_length": 3235.125030517578,
+      "epoch": 0.04342857142857143,
+      "grad_norm": 0.019981542602181435,
+      "kl": 0.0001823529601097107,
+      "learning_rate": 7.4e-07,
+      "loss": 0.0,
+      "reward": 0.13104849308729172,
+      "reward_std": 0.4905168265104294,
+      "rewards/cosine_scaled_reward": -0.03864241763949394,
+      "rewards/format_reward": 0.2083333358168602,
+      "step": 38
+    },
+    {
+      "completion_length": 2411.5417289733887,
+      "epoch": 0.044571428571428574,
+      "grad_norm": 0.026408830657601357,
+      "kl": 0.00011394917964935303,
+      "learning_rate": 7.599999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.5625039599835873,
+      "reward_std": 0.3937423247843981,
+      "rewards/cosine_scaled_reward": -0.020831378176808357,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 39
+    },
+    {
+      "completion_length": 2098.062545776367,
+      "epoch": 0.045714285714285714,
+      "grad_norm": 0.02573023922741413,
+      "kl": 0.00011499226093292236,
+      "learning_rate": 7.799999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.657089076936245,
+      "reward_std": 0.7896895445883274,
+      "rewards/cosine_scaled_reward": -0.025622128508985043,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 40
+    },
+    {
+      "completion_length": 2488.229217529297,
+      "epoch": 0.046857142857142854,
+      "grad_norm": 0.02382344752550125,
+      "kl": 0.00011301040649414062,
+      "learning_rate": 8e-07,
+      "loss": 0.0,
+      "reward": 0.9531193277798593,
+      "reward_std": 0.625364888459444,
+      "rewards/cosine_scaled_reward": 0.15364299900829792,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 41
+    },
+    {
+      "completion_length": 2591.5416984558105,
+      "epoch": 0.048,
+      "grad_norm": 0.0474100187420845,
+      "kl": 0.00018638372421264648,
+      "learning_rate": 8.199999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.019108548760414124,
+      "reward_std": 0.5053375829011202,
+      "rewards/cosine_scaled_reward": -0.20919574238359928,
+      "rewards/format_reward": 0.4375,
+      "step": 42
+    },
+    {
+      "completion_length": 2745.6250610351562,
+      "epoch": 0.04914285714285714,
+      "grad_norm": 0.02300211787223816,
+      "kl": 0.00014187395572662354,
+      "learning_rate": 8.399999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.3337301954161376,
+      "reward_std": 1.0558787789195776,
+      "rewards/cosine_scaled_reward": -0.04146823566406965,
+      "rewards/format_reward": 0.416666679084301,
+      "step": 43
+    },
+    {
+      "completion_length": 2167.33341217041,
+      "epoch": 0.05028571428571429,
+      "grad_norm": 0.038022760301828384,
+      "kl": 0.00011192262172698975,
+      "learning_rate": 8.599999999999999e-07,
+      "loss": 0.0,
+      "reward": 1.0215463279746473,
+      "reward_std": 0.37458088528364897,
+      "rewards/cosine_scaled_reward": 0.14618983678519726,
+      "rewards/format_reward": 0.7291666697710752,
+      "step": 44
+    },
+    {
+      "completion_length": 3106.7083740234375,
+      "epoch": 0.05142857142857143,
+      "grad_norm": 0.02269629016518593,
+      "kl": 0.00013549625873565674,
+      "learning_rate": 8.799999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.7414790373295546,
+      "reward_std": 0.6533729061484337,
+      "rewards/cosine_scaled_reward": 0.11032285634428263,
+      "rewards/format_reward": 0.5208333432674408,
+      "step": 45
+    },
+    {
+      "completion_length": 2975.0208587646484,
+      "epoch": 0.052571428571428575,
+      "grad_norm": 0.023037103936076164,
+      "kl": 0.0001526474952697754,
+      "learning_rate": 9e-07,
+      "loss": 0.0,
+      "reward": 0.13235431909561157,
+      "reward_std": 0.6402451954782009,
+      "rewards/cosine_scaled_reward": -0.15257284580729902,
+      "rewards/format_reward": 0.43750000558793545,
+      "step": 46
+    },
+    {
+      "completion_length": 2202.0417098999023,
+      "epoch": 0.053714285714285714,
+      "grad_norm": 0.03050912357866764,
+      "kl": 9.85860824584961e-05,
+      "learning_rate": 9.2e-07,
+      "loss": 0.0,
+      "reward": 1.0235360227525234,
+      "reward_std": 0.6361051239073277,
+      "rewards/cosine_scaled_reward": 0.15760133787989616,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 47
+    },
+    {
+      "completion_length": 2482.937530517578,
+      "epoch": 0.054857142857142854,
+      "grad_norm": 0.028333164751529694,
+      "kl": 0.00011826306581497192,
+      "learning_rate": 9.399999999999999e-07,
+      "loss": 0.0,
+      "reward": 0.8281691037118435,
+      "reward_std": 0.8235839083790779,
+      "rewards/cosine_scaled_reward": 0.14325121929869056,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 48
+    },
+    {
+      "completion_length": 1686.7708778381348,
+      "epoch": 0.056,
+      "grad_norm": 0.03345295041799545,
+      "kl": 9.24915075302124e-05,
+      "learning_rate": 9.6e-07,
+      "loss": 0.0,
+      "reward": 1.1297309827059507,
+      "reward_std": 0.8849144196137786,
+      "rewards/cosine_scaled_reward": 0.12736546620726585,
+      "rewards/format_reward": 0.8750000074505806,
+      "step": 49
+    },
+    {
+      "completion_length": 2702.166679382324,
+      "epoch": 0.05714285714285714,
+      "grad_norm": 0.025781678035855293,
+      "kl": 0.00011380016803741455,
+      "learning_rate": 9.8e-07,
+      "loss": 0.0,
+      "reward": 0.774685425683856,
+      "reward_std": 0.6316370200365782,
+      "rewards/cosine_scaled_reward": 0.12692604400217533,
+      "rewards/format_reward": 0.5208333376795053,
+      "step": 50
+    },
+    {
+      "completion_length": 2162.75,
+      "epoch": 0.05828571428571429,
+      "grad_norm": 0.032990217208862305,
+      "kl": 0.0001423358917236328,
+      "learning_rate": 1e-06,
+      "loss": 0.0,
+      "reward": 0.395895104855299,
+      "reward_std": 0.5788541845977306,
+      "rewards/cosine_scaled_reward": -0.062469134107232094,
+      "rewards/format_reward": 0.520833333954215,
+      "step": 51
+    },
+    {
+      "completion_length": 2481.3334045410156,
+      "epoch": 0.05942857142857143,
+      "grad_norm": 0.027054639533162117,
+      "kl": 0.00010142475366592407,
+      "learning_rate": 9.999890338174275e-07,
+      "loss": 0.0,
+      "reward": 1.1616107635200024,
+      "reward_std": 0.6760222055017948,
+      "rewards/cosine_scaled_reward": 0.23705538734793663,
+      "rewards/format_reward": 0.6875000055879354,
+      "step": 52
+    },
+    {
+      "completion_length": 2468.5208892822266,
+      "epoch": 0.060571428571428575,
+      "grad_norm": 0.026934338733553886,
+      "kl": 0.000138893723487854,
+      "learning_rate": 9.999561358041868e-07,
+      "loss": 0.0,
+      "reward": 1.0014648959040642,
+      "reward_std": 1.0805351361632347,
+      "rewards/cosine_scaled_reward": 0.15698243118822575,
+      "rewards/format_reward": 0.6875000055879354,
+      "step": 53
+    },
+    {
+      "completion_length": 1780.3333435058594,
+      "epoch": 0.061714285714285715,
+      "grad_norm": 0.030859723687171936,
+      "kl": 9.08970832824707e-05,
+      "learning_rate": 9.999013075636804e-07,
+      "loss": 0.0,
+      "reward": 1.5217821262776852,
+      "reward_std": 0.8481816910207272,
+      "rewards/cosine_scaled_reward": 0.35464105010032654,
+      "rewards/format_reward": 0.8125000074505806,
+      "step": 54
+    },
+    {
+      "completion_length": 2687.4166946411133,
+      "epoch": 0.06285714285714286,
+      "grad_norm": 0.021781187504529953,
+      "kl": 0.00013262033462524414,
+      "learning_rate": 9.998245517681593e-07,
+      "loss": 0.0,
+      "reward": 1.0466104643419385,
+      "reward_std": 0.878080858848989,
+      "rewards/cosine_scaled_reward": 0.20038855448365211,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 55
+    },
+    {
+      "completion_length": 2628.770866394043,
+      "epoch": 0.064,
+      "grad_norm": 0.02273709513247013,
+      "kl": 0.0001190677285194397,
+      "learning_rate": 9.997258721585931e-07,
+      "loss": 0.0,
+      "reward": 0.7287426479160786,
+      "reward_std": 0.6065099984407425,
+      "rewards/cosine_scaled_reward": 0.06228797510266304,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 56
+    },
+    {
+      "completion_length": 2883.854202270508,
+      "epoch": 0.06514285714285714,
+      "grad_norm": 0.017316030338406563,
+      "kl": 0.00010225176811218262,
+      "learning_rate": 9.996052735444862e-07,
+      "loss": 0.0,
+      "reward": 0.4611471053212881,
+      "reward_std": 1.0018693208694458,
+      "rewards/cosine_scaled_reward": -0.019426452228799462,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 57
+    },
+    {
+      "completion_length": 1413.4791793823242,
+      "epoch": 0.06628571428571428,
+      "grad_norm": 0.03232548013329506,
+      "kl": 6.647780537605286e-05,
+      "learning_rate": 9.994627618036452e-07,
+      "loss": 0.0,
+      "reward": 0.8900517076253891,
+      "reward_std": 0.7214644104242325,
+      "rewards/cosine_scaled_reward": 0.049192506819963455,
+      "rewards/format_reward": 0.7916666716337204,
+      "step": 58
+    },
+    {
+      "completion_length": 2609.520851135254,
+      "epoch": 0.06742857142857143,
+      "grad_norm": 0.02341977320611477,
+      "kl": 0.00011394917964935303,
+      "learning_rate": 9.992983438818915e-07,
+      "loss": 0.0,
+      "reward": 0.6282028257846832,
+      "reward_std": 0.7330185724422336,
+      "rewards/cosine_scaled_reward": 0.08493474265560508,
+      "rewards/format_reward": 0.45833333395421505,
+      "step": 59
+    },
+    {
+      "completion_length": 2260.875030517578,
+      "epoch": 0.06857142857142857,
+      "grad_norm": 0.025622840970754623,
+      "kl": 0.00010037794709205627,
+      "learning_rate": 9.991120277927223e-07,
+      "loss": 0.0,
+      "reward": 0.7927640732377768,
+      "reward_std": 0.5676146075129509,
+      "rewards/cosine_scaled_reward": 0.04221535753458738,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 60
+    },
+    {
+      "completion_length": 2364.1042098999023,
+      "epoch": 0.06971428571428571,
+      "grad_norm": 0.023006901144981384,
+      "kl": 8.575618267059326e-05,
+      "learning_rate": 9.989038226169207e-07,
+      "loss": 0.0,
+      "reward": 0.5264723375439644,
+      "reward_std": 0.4836166054010391,
+      "rewards/cosine_scaled_reward": -0.0388471744954586,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 61
+    },
+    {
+      "completion_length": 2127.2500228881836,
+      "epoch": 0.07085714285714285,
+      "grad_norm": 0.024150002747774124,
+      "kl": 8.403509855270386e-05,
+      "learning_rate": 9.98673738502114e-07,
+      "loss": 0.0,
+      "reward": 1.038158729672432,
+      "reward_std": 0.8189139366149902,
+      "rewards/cosine_scaled_reward": 0.14407933596521616,
+      "rewards/format_reward": 0.7500000074505806,
+      "step": 62
+    },
+    {
+      "completion_length": 1480.7500228881836,
+      "epoch": 0.072,
+      "grad_norm": 0.03183520957827568,
+      "kl": 7.95125961303711e-05,
+      "learning_rate": 9.98421786662277e-07,
+      "loss": 0.0,
+      "reward": 1.3979634009301662,
+      "reward_std": 0.5990209653973579,
+      "rewards/cosine_scaled_reward": 0.261481698602438,
+      "rewards/format_reward": 0.875,
+      "step": 63
+    },
+    {
+      "completion_length": 2504.666679382324,
+      "epoch": 0.07314285714285715,
+      "grad_norm": 0.021520959213376045,
+      "kl": 0.0001504272222518921,
+      "learning_rate": 9.981479793771866e-07,
+      "loss": 0.0,
+      "reward": 0.8372010216116905,
+      "reward_std": 0.6612134985625744,
+      "rewards/cosine_scaled_reward": 0.1373505061492324,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 64
+    },
+    {
+      "completion_length": 2580.6458702087402,
+      "epoch": 0.07428571428571429,
+      "grad_norm": 0.028981182724237442,
+      "kl": 0.00011400878429412842,
+      "learning_rate": 9.97852329991824e-07,
+      "loss": 0.0,
+      "reward": 0.45997084584087133,
+      "reward_std": 0.7114468142390251,
+      "rewards/cosine_scaled_reward": -0.030431261286139488,
+      "rewards/format_reward": 0.520833345130086,
+      "step": 65
+    },
+    {
+      "completion_length": 2067.458335876465,
+      "epoch": 0.07542857142857143,
+      "grad_norm": 0.03032701462507248,
+      "kl": 9.172409772872925e-05,
+      "learning_rate": 9.975348529157229e-07,
+      "loss": 0.0,
+      "reward": 0.301434013992548,
+      "reward_std": 0.645634401589632,
+      "rewards/cosine_scaled_reward": -0.09928299766033888,
+      "rewards/format_reward": 0.5,
+      "step": 66
+    },
+    {
+      "completion_length": 3204.3541717529297,
+      "epoch": 0.07657142857142857,
+      "grad_norm": 0.016771739348769188,
+      "kl": 0.00011616945266723633,
+      "learning_rate": 9.971955636222684e-07,
+      "loss": 0.0,
+      "reward": 0.028863554820418358,
+      "reward_std": 0.6313999556005001,
+      "rewards/cosine_scaled_reward": -0.14181822165846825,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 67
+    },
+    {
+      "completion_length": 1526.895866394043,
+      "epoch": 0.07771428571428571,
+      "grad_norm": 0.03267313167452812,
+      "kl": 8.743256330490112e-05,
+      "learning_rate": 9.968344786479415e-07,
+      "loss": 0.0,
+      "reward": 0.8832469508051872,
+      "reward_std": 0.6210112273693085,
+      "rewards/cosine_scaled_reward": 0.06662345677614212,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 68
+    },
+    {
+      "completion_length": 1829.0416946411133,
+      "epoch": 0.07885714285714286,
+      "grad_norm": 0.031190019100904465,
+      "kl": 9.609758853912354e-05,
+      "learning_rate": 9.964516155915151e-07,
+      "loss": 0.0,
+      "reward": 0.6412256211042404,
+      "reward_std": 0.839095912873745,
+      "rewards/cosine_scaled_reward": -0.03355386573821306,
+      "rewards/format_reward": 0.7083333395421505,
+      "step": 69
+    },
+    {
+      "completion_length": 2334.7083740234375,
+      "epoch": 0.08,
+      "grad_norm": 0.023025985807180405,
+      "kl": 9.860843420028687e-05,
+      "learning_rate": 9.960469931131936e-07,
+      "loss": 0.0,
+      "reward": 0.6220458149909973,
+      "reward_std": 0.6576723717153072,
+      "rewards/cosine_scaled_reward": -0.011893758550286293,
+      "rewards/format_reward": 0.645833333954215,
+      "step": 70
+    },
+    {
+      "completion_length": 2430.8750228881836,
+      "epoch": 0.08114285714285714,
+      "grad_norm": 0.03443196043372154,
+      "kl": 0.00010547041893005371,
+      "learning_rate": 9.956206309337066e-07,
+      "loss": 0.0,
+      "reward": 0.6613975018262863,
+      "reward_std": 0.5863231625407934,
+      "rewards/cosine_scaled_reward": 0.11194872949272394,
+      "rewards/format_reward": 0.43750000186264515,
+      "step": 71
+    },
+    {
+      "completion_length": 2218.250045776367,
+      "epoch": 0.08228571428571428,
+      "grad_norm": 0.02904508076608181,
+      "kl": 9.149312973022461e-05,
+      "learning_rate": 9.951725498333448e-07,
+      "loss": 0.0,
+      "reward": 0.39750963170081377,
+      "reward_std": 0.5485289674252272,
+      "rewards/cosine_scaled_reward": -0.15541186556220055,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 72
+    },
+    {
+      "completion_length": 3065.2708892822266,
+      "epoch": 0.08342857142857144,
+      "grad_norm": 0.01883835718035698,
+      "kl": 0.0001595020294189453,
+      "learning_rate": 9.947027716509488e-07,
+      "loss": 0.0,
+      "reward": 0.3231031158939004,
+      "reward_std": 0.7491481080651283,
+      "rewards/cosine_scaled_reward": -0.03636510670185089,
+      "rewards/format_reward": 0.3958333395421505,
+      "step": 73
+    },
+    {
+      "completion_length": 2371.104202270508,
+      "epoch": 0.08457142857142858,
+      "grad_norm": 0.024749629199504852,
+      "kl": 0.00012037158012390137,
+      "learning_rate": 9.942113192828444e-07,
+      "loss": 0.0,
+      "reward": 0.6487875208258629,
+      "reward_std": 0.839644305408001,
+      "rewards/cosine_scaled_reward": 0.05356042645871639,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 74
+    },
+    {
+      "completion_length": 2677.5,
+      "epoch": 0.08571428571428572,
+      "grad_norm": 0.02110193856060505,
+      "kl": 0.0001303553581237793,
+      "learning_rate": 9.93698216681727e-07,
+      "loss": 0.0,
+      "reward": 0.6778632658533752,
+      "reward_std": 0.7005478721112013,
+      "rewards/cosine_scaled_reward": 0.07851496431976557,
+      "rewards/format_reward": 0.5208333414047956,
+      "step": 75
+    },
+    {
+      "completion_length": 2419.2291984558105,
+      "epoch": 0.08685714285714285,
+      "grad_norm": 0.02441054955124855,
+      "kl": 0.0001291334629058838,
+      "learning_rate": 9.931634888554935e-07,
+      "loss": 0.0,
+      "reward": 0.41384580731391907,
+      "reward_std": 0.6501213163137436,
+      "rewards/cosine_scaled_reward": -0.053493766114115715,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 76
+    },
+    {
+      "completion_length": 2707.1042098999023,
+      "epoch": 0.088,
+      "grad_norm": 0.020344527438282967,
+      "kl": 0.00013850629329681396,
+      "learning_rate": 9.926071618660237e-07,
+      "loss": 0.0,
+      "reward": 0.2681877203285694,
+      "reward_std": 0.5029121972620487,
+      "rewards/cosine_scaled_reward": -0.11590614821761847,
+      "rewards/format_reward": 0.5000000037252903,
+      "step": 77
+    },
+    {
+      "completion_length": 2731.479232788086,
+      "epoch": 0.08914285714285715,
+      "grad_norm": 0.01844891719520092,
+      "kl": 0.00013375282287597656,
+      "learning_rate": 9.9202926282791e-07,
+      "loss": 0.0,
+      "reward": 0.7639368114178069,
+      "reward_std": 0.8214902877807617,
+      "rewards/cosine_scaled_reward": 0.12155175860971212,
+      "rewards/format_reward": 0.520833345130086,
+      "step": 78
+    },
+    {
+      "completion_length": 2004.6041946411133,
+      "epoch": 0.09028571428571429,
+      "grad_norm": 0.028990577906370163,
+      "kl": 8.181855082511902e-05,
+      "learning_rate": 9.91429819907136e-07,
+      "loss": 0.0,
+      "reward": 0.7587422616779804,
+      "reward_std": 0.9989555478096008,
+      "rewards/cosine_scaled_reward": 0.04603780619800091,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 79
+    },
+    {
+      "completion_length": 2782.375045776367,
+      "epoch": 0.09142857142857143,
+      "grad_norm": 0.023466741666197777,
+      "kl": 0.00015717744827270508,
+      "learning_rate": 9.908088623197048e-07,
+      "loss": 0.0,
+      "reward": 0.6499133557081223,
+      "reward_std": 0.7965570129454136,
+      "rewards/cosine_scaled_reward": 0.05412333086133003,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 80
+    },
+    {
+      "completion_length": 2665.3542137145996,
+      "epoch": 0.09257142857142857,
+      "grad_norm": 0.028318636119365692,
+      "kl": 0.0001677870750427246,
+      "learning_rate": 9.901664203302124e-07,
+      "loss": 0.0,
+      "reward": 0.4032784029841423,
+      "reward_std": 0.6976380785927176,
+      "rewards/cosine_scaled_reward": -0.03794414922595024,
+      "rewards/format_reward": 0.4791666753590107,
+      "step": 81
+    },
+    {
+      "completion_length": 2347.4583702087402,
+      "epoch": 0.09371428571428571,
+      "grad_norm": 0.025593508034944534,
+      "kl": 0.00010985136032104492,
+      "learning_rate": 9.895025252503755e-07,
+      "loss": 0.0,
+      "reward": 0.8746878691017628,
+      "reward_std": 0.9405775591731071,
+      "rewards/cosine_scaled_reward": 0.14567727083340287,
+      "rewards/format_reward": 0.5833333376795053,
+      "step": 82
+    },
+    {
+      "completion_length": 2405.333366394043,
+      "epoch": 0.09485714285714286,
+      "grad_norm": 0.03943740949034691,
+      "kl": 0.00012260675430297852,
+      "learning_rate": 9.888172094375033e-07,
+      "loss": 0.0,
+      "reward": 0.6102542355656624,
+      "reward_std": 0.8917163014411926,
+      "rewards/cosine_scaled_reward": 0.03429375775158405,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 83
+    },
+    {
+      "completion_length": 2356.6666984558105,
+      "epoch": 0.096,
+      "grad_norm": 0.023660918697714806,
+      "kl": 0.00010949373245239258,
+      "learning_rate": 9.881105062929221e-07,
+      "loss": 0.0,
+      "reward": 0.49356625229120255,
+      "reward_std": 0.6785021275281906,
+      "rewards/cosine_scaled_reward": -0.03446688875555992,
+      "rewards/format_reward": 0.5625,
+      "step": 84
+    },
+    {
+      "completion_length": 2653.9791870117188,
+      "epoch": 0.09714285714285714,
+      "grad_norm": 0.01803443394601345,
+      "kl": 9.615719318389893e-05,
+      "learning_rate": 9.873824502603459e-07,
+      "loss": 0.0,
+      "reward": 0.8409973373636603,
+      "reward_std": 0.8465002179145813,
+      "rewards/cosine_scaled_reward": 0.10799866930756252,
+      "rewards/format_reward": 0.6250000093132257,
+      "step": 85
+    },
+    {
+      "completion_length": 2493.8542098999023,
+      "epoch": 0.09828571428571428,
+      "grad_norm": 0.026323335245251656,
+      "kl": 0.00012114644050598145,
+      "learning_rate": 9.866330768241983e-07,
+      "loss": 0.0,
+      "reward": 0.48225442320108414,
+      "reward_std": 0.4767768904566765,
+      "rewards/cosine_scaled_reward": -0.00887279398739338,
+      "rewards/format_reward": 0.5,
+      "step": 86
+    },
+    {
+      "completion_length": 2238.4583587646484,
+      "epoch": 0.09942857142857142,
+      "grad_norm": 0.02708311937749386,
+      "kl": 0.00014419853687286377,
+      "learning_rate": 9.85862422507884e-07,
+      "loss": 0.0,
+      "reward": 0.8275710507296026,
+      "reward_std": 0.5411144681274891,
+      "rewards/cosine_scaled_reward": 0.09086884278804064,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 87
+    },
+    {
+      "completion_length": 1580.8958854675293,
+      "epoch": 0.10057142857142858,
+      "grad_norm": 0.029665708541870117,
+      "kl": 8.863955736160278e-05,
+      "learning_rate": 9.850705248720068e-07,
+      "loss": 0.0,
+      "reward": 0.9847916029393673,
+      "reward_std": 0.9228500425815582,
+      "rewards/cosine_scaled_reward": 0.08614580077119172,
+      "rewards/format_reward": 0.812500013038516,
+      "step": 88
+    },
+    {
+      "completion_length": 2665.1666870117188,
+      "epoch": 0.10171428571428572,
+      "grad_norm": 0.019765589386224747,
+      "kl": 0.00011581182479858398,
+      "learning_rate": 9.8425742251254e-07,
+      "loss": 0.0,
+      "reward": 0.6976313246414065,
+      "reward_std": 0.7233806774020195,
+      "rewards/cosine_scaled_reward": 0.09881562972441316,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 89
+    },
+    {
+      "completion_length": 2352.7291870117188,
+      "epoch": 0.10285714285714286,
+      "grad_norm": 0.04337649419903755,
+      "kl": 0.00018215179443359375,
+      "learning_rate": 9.83423155058946e-07,
+      "loss": 0.0,
+      "reward": 0.2358827404677868,
+      "reward_std": 0.5784358195960522,
+      "rewards/cosine_scaled_reward": -0.15289196744561195,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 90
+    },
+    {
+      "completion_length": 2579.8750534057617,
+      "epoch": 0.104,
+      "grad_norm": 0.024008560925722122,
+      "kl": 0.00012639164924621582,
+      "learning_rate": 9.825677631722435e-07,
+      "loss": 0.0,
+      "reward": 0.5804571909829974,
+      "reward_std": 0.7776114530861378,
+      "rewards/cosine_scaled_reward": 0.029811910664648167,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 91
+    },
+    {
+      "completion_length": 2127.750015258789,
+      "epoch": 0.10514285714285715,
+      "grad_norm": 0.03029821440577507,
+      "kl": 9.115040302276611e-05,
+      "learning_rate": 9.816912885430258e-07,
+      "loss": 0.0,
+      "reward": 0.6724662911146879,
+      "reward_std": 0.40310859866440296,
+      "rewards/cosine_scaled_reward": -0.059600187465548515,
+      "rewards/format_reward": 0.7916666716337204,
+      "step": 92
+    },
+    {
+      "completion_length": 3542.4791870117188,
+      "epoch": 0.10628571428571429,
+      "grad_norm": 0.01921100728213787,
+      "kl": 0.00022071599960327148,
+      "learning_rate": 9.807937738894303e-07,
+      "loss": 0.0,
+      "reward": -0.3310100920498371,
+      "reward_std": 0.44439256377518177,
+      "rewards/cosine_scaled_reward": -0.19675504975020885,
+      "rewards/format_reward": 0.06250000186264515,
+      "step": 93
+    },
+    {
+      "completion_length": 2373.1667251586914,
+      "epoch": 0.10742857142857143,
+      "grad_norm": 0.02489403821527958,
+      "kl": 0.0001574680209159851,
+      "learning_rate": 9.798752629550546e-07,
+      "loss": 0.0,
+      "reward": 0.7052749944850802,
+      "reward_std": 0.8667173758149147,
+      "rewards/cosine_scaled_reward": 0.05055414792150259,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 94
+    },
+    {
+      "completion_length": 3027.5208740234375,
+      "epoch": 0.10857142857142857,
+      "grad_norm": 0.02021307684481144,
+      "kl": 0.0001308917999267578,
+      "learning_rate": 9.78935800506826e-07,
+      "loss": 0.0,
+      "reward": 0.38812466710805893,
+      "reward_std": 0.7133069280534983,
+      "rewards/cosine_scaled_reward": -0.03510434227064252,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 95
+    },
+    {
+      "completion_length": 2473.1875762939453,
+      "epoch": 0.10971428571428571,
+      "grad_norm": 0.030283445492386818,
+      "kl": 9.927153587341309e-05,
+      "learning_rate": 9.779754323328192e-07,
+      "loss": 0.0,
+      "reward": 0.6445761788636446,
+      "reward_std": 0.8469636254012585,
+      "rewards/cosine_scaled_reward": 0.04103806708008051,
+      "rewards/format_reward": 0.5625000055879354,
+      "step": 96
+    },
+    {
+      "completion_length": 2826.166732788086,
+      "epoch": 0.11085714285714286,
+      "grad_norm": 0.022094760090112686,
+      "kl": 0.00012320280075073242,
+      "learning_rate": 9.769942052400235e-07,
+      "loss": 0.0,
+      "reward": 0.33709235209971666,
+      "reward_std": 0.8025432769209146,
+      "rewards/cosine_scaled_reward": -0.03978714719414711,
+      "rewards/format_reward": 0.4166666753590107,
+      "step": 97
+    },
+    {
+      "completion_length": 2312.9166946411133,
+      "epoch": 0.112,
+      "grad_norm": 0.032762184739112854,
+      "kl": 9.972602128982544e-05,
+      "learning_rate": 9.759921670520634e-07,
+      "loss": 0.0,
+      "reward": 0.34186697006225586,
+      "reward_std": 0.7580292820930481,
+      "rewards/cosine_scaled_reward": -0.06864983681589365,
+      "rewards/format_reward": 0.47916667722165585,
+      "step": 98
+    },
+    {
+      "completion_length": 2789.12504196167,
+      "epoch": 0.11314285714285714,
+      "grad_norm": 0.0221230611205101,
+      "kl": 0.00011648237705230713,
+      "learning_rate": 9.749693666068663e-07,
+      "loss": 0.0,
+      "reward": 0.3996657454408705,
+      "reward_std": 0.7801091782748699,
+      "rewards/cosine_scaled_reward": -0.02933379588648677,
+      "rewards/format_reward": 0.45833334140479565,
+      "step": 99
+    },
+    {
+      "completion_length": 2257.708381652832,
+      "epoch": 0.11428571428571428,
+      "grad_norm": 0.023916251957416534,
+      "kl": 0.00011393427848815918,
+      "learning_rate": 9.739258537542835e-07,
+      "loss": 0.0,
+      "reward": 1.030206087976694,
+      "reward_std": 0.9744261428713799,
+      "rewards/cosine_scaled_reward": 0.1921863555908203,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 100
+    },
+    {
+      "completion_length": 2282.979202270508,
+      "epoch": 0.11542857142857142,
+      "grad_norm": 0.029999660328030586,
+      "kl": 0.0001456141471862793,
+      "learning_rate": 9.728616793536587e-07,
+      "loss": 0.0,
+      "reward": 0.869966953061521,
+      "reward_std": 0.784905806183815,
+      "rewards/cosine_scaled_reward": 0.11206682212650776,
+      "rewards/format_reward": 0.6458333488553762,
+      "step": 101
+    },
+    {
+      "completion_length": 2075.9792098999023,
+      "epoch": 0.11657142857142858,
+      "grad_norm": 0.030493436381220818,
+      "kl": 0.00013086199760437012,
+      "learning_rate": 9.717768952713511e-07,
+      "loss": 0.0,
+      "reward": 0.8771389909088612,
+      "reward_std": 0.7964347116649151,
+      "rewards/cosine_scaled_reward": 0.07398613449186087,
+      "rewards/format_reward": 0.7291666753590107,
+      "step": 102
+    },
+    {
+      "completion_length": 2310.7500228881836,
+      "epoch": 0.11771428571428572,
+      "grad_norm": 0.029018821194767952,
+      "kl": 0.00010515749454498291,
+      "learning_rate": 9.706715543782064e-07,
+      "loss": 0.0,
+      "reward": 0.5800389759242535,
+      "reward_std": 0.5602545086294413,
+      "rewards/cosine_scaled_reward": -0.06414720602333546,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 103
+    },
+    {
+      "completion_length": 2372.104263305664,
+      "epoch": 0.11885714285714286,
+      "grad_norm": 0.027296504005789757,
+      "kl": 0.0001329481601715088,
+      "learning_rate": 9.695457105469804e-07,
+      "loss": 0.0,
+      "reward": 0.5882483441382647,
+      "reward_std": 0.730236142873764,
+      "rewards/cosine_scaled_reward": -0.028792519122362137,
+      "rewards/format_reward": 0.6458333395421505,
+      "step": 104
+    },
+    {
+      "completion_length": 2313.625030517578,
+      "epoch": 0.12,
+      "grad_norm": 0.025855960324406624,
+      "kl": 0.00012581050395965576,
+      "learning_rate": 9.683994186497132e-07,
+      "loss": 0.0,
+      "reward": 1.154417909681797,
+      "reward_std": 0.6686892919242382,
+      "rewards/cosine_scaled_reward": 0.25429226853884757,
+      "rewards/format_reward": 0.6458333395421505,
+      "step": 105
+    },
+    {
+      "completion_length": 1879.3125457763672,
+      "epoch": 0.12114285714285715,
+      "grad_norm": 0.030279330909252167,
+      "kl": 7.063150405883789e-05,
+      "learning_rate": 9.672327345550543e-07,
+      "loss": 0.0,
+      "reward": 1.0762072280049324,
+      "reward_std": 0.7403409704566002,
+      "rewards/cosine_scaled_reward": 0.1735202744603157,
+      "rewards/format_reward": 0.7291666697710752,
+      "step": 106
+    },
+    {
+      "completion_length": 2561.50004196167,
+      "epoch": 0.12228571428571429,
+      "grad_norm": 0.023640897125005722,
+      "kl": 0.00015266239643096924,
+      "learning_rate": 9.66045715125541e-07,
+      "loss": 0.0,
+      "reward": 0.47847325541079044,
+      "reward_std": 0.5093330107629299,
+      "rewards/cosine_scaled_reward": -0.02118003647774458,
+      "rewards/format_reward": 0.520833333954215,
+      "step": 107
+    },
+    {
+      "completion_length": 2796.229179382324,
+      "epoch": 0.12342857142857143,
+      "grad_norm": 0.02216712199151516,
+      "kl": 0.00013267993927001953,
+      "learning_rate": 9.648384182148252e-07,
+      "loss": 0.0,
+      "reward": 0.4137321300804615,
+      "reward_std": 0.7223830446600914,
+      "rewards/cosine_scaled_reward": -0.0014672763645648956,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 108
+    },
+    {
+      "completion_length": 2795.687545776367,
+      "epoch": 0.12457142857142857,
+      "grad_norm": 0.022554874420166016,
+      "kl": 0.0001379549503326416,
+      "learning_rate": 9.636109026648554e-07,
+      "loss": 0.0,
+      "reward": 0.27347924932837486,
+      "reward_std": 0.6205065222457051,
+      "rewards/cosine_scaled_reward": -0.050760375801473856,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 109
+    },
+    {
+      "completion_length": 2653.0417442321777,
+      "epoch": 0.12571428571428572,
+      "grad_norm": 0.02770637348294258,
+      "kl": 0.00011685490608215332,
+      "learning_rate": 9.623632283030077e-07,
+      "loss": 0.0,
+      "reward": 1.0205977819859982,
+      "reward_std": 0.9722634367644787,
+      "rewards/cosine_scaled_reward": 0.17696558311581612,
+      "rewards/format_reward": 0.6666666828095913,
+      "step": 110
+    },
+    {
+      "completion_length": 2789.4583587646484,
+      "epoch": 0.12685714285714286,
+      "grad_norm": 0.0231799129396677,
+      "kl": 0.00014919042587280273,
+      "learning_rate": 9.610954559391704e-07,
+      "loss": 0.0,
+      "reward": 0.5839154049754143,
+      "reward_std": 0.6967787519097328,
+      "rewards/cosine_scaled_reward": 0.07320770528167486,
+      "rewards/format_reward": 0.43750001303851604,
+      "step": 111
+    },
+    {
+      "completion_length": 3039.875030517578,
+      "epoch": 0.128,
+      "grad_norm": 0.021811209619045258,
+      "kl": 0.00016495585441589355,
+      "learning_rate": 9.598076473627796e-07,
+      "loss": 0.0,
+      "reward": 0.43579965829849243,
+      "reward_std": 0.8692431841045618,
+      "rewards/cosine_scaled_reward": -0.0008501838892698288,
+      "rewards/format_reward": 0.4375000111758709,
+      "step": 112
+    },
+    {
+      "completion_length": 2002.0000305175781,
+      "epoch": 0.12914285714285714,
+      "grad_norm": 0.03178806230425835,
+      "kl": 0.00013971328735351562,
+      "learning_rate": 9.58499865339809e-07,
+      "loss": 0.0,
+      "reward": 0.7515672761946917,
+      "reward_std": 0.8758837655186653,
+      "rewards/cosine_scaled_reward": 0.0007836292497813702,
+      "rewards/format_reward": 0.7500000093132257,
+      "step": 113
+    },
+    {
+      "completion_length": 2049.791717529297,
+      "epoch": 0.13028571428571428,
+      "grad_norm": 0.030137214809656143,
+      "kl": 8.422881364822388e-05,
+      "learning_rate": 9.571721736097088e-07,
+      "loss": 0.0,
+      "reward": 0.46851017512381077,
+      "reward_std": 0.3973765755072236,
+      "rewards/cosine_scaled_reward": -0.11991160549223423,
+      "rewards/format_reward": 0.708333333954215,
+      "step": 114
+    },
+    {
+      "completion_length": 2772.979217529297,
+      "epoch": 0.13142857142857142,
+      "grad_norm": 0.031248675659298897,
+      "kl": 0.00012229382991790771,
+      "learning_rate": 9.55824636882301e-07,
+      "loss": 0.0,
+      "reward": 0.7633647555485368,
+      "reward_std": 0.8922017849981785,
+      "rewards/cosine_scaled_reward": 0.14209904265590012,
+      "rewards/format_reward": 0.47916667349636555,
+      "step": 115
+    },
+    {
+      "completion_length": 3117.1875534057617,
+      "epoch": 0.13257142857142856,
+      "grad_norm": 0.025444043800234795,
+      "kl": 0.00015610456466674805,
+      "learning_rate": 9.54457320834625e-07,
+      "loss": 0.0,
+      "reward": 0.1580726346001029,
+      "reward_std": 0.614492192864418,
+      "rewards/cosine_scaled_reward": -0.08763036131858826,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 116
+    },
+    {
+      "completion_length": 2826.8958892822266,
+      "epoch": 0.1337142857142857,
+      "grad_norm": 0.02326161414384842,
+      "kl": 0.00017911195755004883,
+      "learning_rate": 9.530702921077358e-07,
+      "loss": 0.0,
+      "reward": 0.10269369930028915,
+      "reward_std": 0.7328709326684475,
+      "rewards/cosine_scaled_reward": -0.14656982268206775,
+      "rewards/format_reward": 0.3958333469927311,
+      "step": 117
+    },
+    {
+      "completion_length": 2824.666717529297,
+      "epoch": 0.13485714285714287,
+      "grad_norm": 0.020892256870865822,
+      "kl": 0.00012445449829101562,
+      "learning_rate": 9.516636183034564e-07,
+      "loss": 0.0,
+      "reward": 1.3870177865028381,
+      "reward_std": 1.0715553350746632,
+      "rewards/cosine_scaled_reward": 0.3601755518466234,
+      "rewards/format_reward": 0.6666666753590107,
+      "step": 118
+    },
+    {
+      "completion_length": 1789.0000267028809,
+      "epoch": 0.136,
+      "grad_norm": 0.042789019644260406,
+      "kl": 9.068846702575684e-05,
+      "learning_rate": 9.502373679810839e-07,
+      "loss": 0.0,
+      "reward": 0.9923834577202797,
+      "reward_std": 0.4148991871625185,
+      "rewards/cosine_scaled_reward": 0.1420250441879034,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 119
+    },
+    {
+      "completion_length": 2159.7292098999023,
+      "epoch": 0.13714285714285715,
+      "grad_norm": 0.030027495697140694,
+      "kl": 0.00016351789236068726,
+      "learning_rate": 9.487916106540465e-07,
+      "loss": 0.0,
+      "reward": 0.6492169927805662,
+      "reward_std": 0.827868428081274,
+      "rewards/cosine_scaled_reward": 0.032941827550530434,
+      "rewards/format_reward": 0.5833333432674408,
+      "step": 120
+    },
+    {
+      "completion_length": 1691.541690826416,
+      "epoch": 0.1382857142857143,
+      "grad_norm": 0.03286458179354668,
+      "kl": 0.00010415911674499512,
+      "learning_rate": 9.473264167865171e-07,
+      "loss": 0.0,
+      "reward": 0.7581329848617315,
+      "reward_std": 0.4227004833519459,
+      "rewards/cosine_scaled_reward": 0.03531647473573685,
+      "rewards/format_reward": 0.6875000018626451,
+      "step": 121
+    },
+    {
+      "completion_length": 2760.31254196167,
+      "epoch": 0.13942857142857143,
+      "grad_norm": 0.024439577013254166,
+      "kl": 0.0001647472381591797,
+      "learning_rate": 9.458418577899774e-07,
+      "loss": 0.0,
+      "reward": 0.755975735373795,
+      "reward_std": 0.785768523812294,
+      "rewards/cosine_scaled_reward": 0.08632119931280613,
+      "rewards/format_reward": 0.583333333954215,
+      "step": 122
+    },
+    {
+      "completion_length": 2559.270896911621,
+      "epoch": 0.14057142857142857,
+      "grad_norm": 0.021681685000658035,
+      "kl": 0.00013074278831481934,
+      "learning_rate": 9.443380060197385e-07,
+      "loss": 0.0,
+      "reward": 0.5207496341317892,
+      "reward_std": 0.6123828925192356,
+      "rewards/cosine_scaled_reward": -0.010458520613610744,
+      "rewards/format_reward": 0.5416666772216558,
+      "step": 123
+    },
+    {
+      "completion_length": 1917.479175567627,
+      "epoch": 0.1417142857142857,
+      "grad_norm": 0.028799263760447502,
+      "kl": 7.950514554977417e-05,
+      "learning_rate": 9.428149347714143e-07,
+      "loss": 0.0,
+      "reward": 1.3661205926910043,
+      "reward_std": 0.6406003027223051,
+      "rewards/cosine_scaled_reward": 0.3288935967721045,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 124
+    },
+    {
+      "completion_length": 2284.5833435058594,
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.026874002069234848,
+      "kl": 9.514018893241882e-05,
+      "learning_rate": 9.412727182773486e-07,
+      "loss": 0.0,
+      "reward": 0.4183652549982071,
+      "reward_std": 0.5504244212061167,
+      "rewards/cosine_scaled_reward": -0.019984053447842598,
+      "rewards/format_reward": 0.4583333432674408,
+      "step": 125
+    },
+    {
+      "completion_length": 2366.625045776367,
+      "epoch": 0.144,
+      "grad_norm": 0.025750989094376564,
+      "kl": 0.0001022443175315857,
+      "learning_rate": 9.397114317029974e-07,
+      "loss": 0.0,
+      "reward": 0.8881729152053595,
+      "reward_std": 0.8806649167090654,
+      "rewards/cosine_scaled_reward": 0.14200311340391636,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 126
+    },
+    {
+      "completion_length": 3196.8750610351562,
+      "epoch": 0.14514285714285713,
+      "grad_norm": 0.01978475973010063,
+      "kl": 0.000141829252243042,
+      "learning_rate": 9.381311511432658e-07,
+      "loss": 0.0,
+      "reward": -0.040465060621500015,
+      "reward_std": 0.6252349764108658,
+      "rewards/cosine_scaled_reward": -0.17648251354694366,
+      "rewards/format_reward": 0.31250000931322575,
+      "step": 127
+    },
+    {
+      "completion_length": 2271.8958740234375,
+      "epoch": 0.1462857142857143,
+      "grad_norm": 0.029708651825785637,
+      "kl": 0.00015082955360412598,
+      "learning_rate": 9.36531953618799e-07,
+      "loss": 0.0,
+      "reward": 0.825566089246422,
+      "reward_std": 0.7982673496007919,
+      "rewards/cosine_scaled_reward": 0.11069970577955246,
+      "rewards/format_reward": 0.6041666753590107,
+      "step": 128
+    },
+    {
+      "completion_length": 3237.4583435058594,
+      "epoch": 0.14742857142857144,
+      "grad_norm": 0.019838884472846985,
+      "kl": 0.0001277327537536621,
+      "learning_rate": 9.34913917072228e-07,
+      "loss": 0.0,
+      "reward": 0.4639533148147166,
+      "reward_std": 0.9510225504636765,
+      "rewards/cosine_scaled_reward": 0.03405998833477497,
+      "rewards/format_reward": 0.39583334140479565,
+      "step": 129
+    },
+    {
+      "completion_length": 3055.3333587646484,
+      "epoch": 0.14857142857142858,
+      "grad_norm": 0.020251061767339706,
+      "kl": 0.0001876354217529297,
+      "learning_rate": 9.332771203643714e-07,
+      "loss": 0.0,
+      "reward": 0.29083502665162086,
+      "reward_std": 0.9767868965864182,
+      "rewards/cosine_scaled_reward": -0.00041582807898521423,
+      "rewards/format_reward": 0.2916666679084301,
+      "step": 130
+    },
+    {
+      "completion_length": 2579.000068664551,
+      "epoch": 0.14971428571428572,
+      "grad_norm": 0.023820001631975174,
+      "kl": 0.0001468360424041748,
+      "learning_rate": 9.316216432703916e-07,
+      "loss": 0.0,
+      "reward": 0.7045300118625164,
+      "reward_std": 0.47794024273753166,
+      "rewards/cosine_scaled_reward": 0.0814316663891077,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 131
+    },
+    {
+      "completion_length": 2430.1250228881836,
+      "epoch": 0.15085714285714286,
+      "grad_norm": 0.024699199944734573,
+      "kl": 0.00010822713375091553,
+      "learning_rate": 9.299475664759068e-07,
+      "loss": 0.0,
+      "reward": 0.9790843389928341,
+      "reward_std": 0.9423319976776838,
+      "rewards/cosine_scaled_reward": 0.22912549204193056,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 132
+    },
+    {
+      "completion_length": 3047.2083587646484,
+      "epoch": 0.152,
+      "grad_norm": 0.020359130576252937,
+      "kl": 0.00017851591110229492,
+      "learning_rate": 9.282549715730579e-07,
+      "loss": 0.0,
+      "reward": 0.1808095509186387,
+      "reward_std": 0.9317280426621437,
+      "rewards/cosine_scaled_reward": -0.08667857074760832,
+      "rewards/format_reward": 0.3541666753590107,
+      "step": 133
+    },
+    {
+      "completion_length": 2291.5833892822266,
+      "epoch": 0.15314285714285714,
+      "grad_norm": 0.02574433758854866,
+      "kl": 0.00011102622374892235,
+      "learning_rate": 9.265439410565328e-07,
+      "loss": 0.0,
+      "reward": 0.8931092359125614,
+      "reward_std": 0.7652852088212967,
+      "rewards/cosine_scaled_reward": 0.14447128726169467,
+      "rewards/format_reward": 0.6041666734963655,
+      "step": 134
+    },
+    {
+      "completion_length": 1257.270839691162,
+      "epoch": 0.15428571428571428,
+      "grad_norm": 0.037061907351017,
+      "kl": 6.86943531036377e-05,
+      "learning_rate": 9.248145583195447e-07,
+      "loss": 0.0,
+      "reward": 1.4347281754016876,
+      "reward_std": 0.5897002862766385,
+      "rewards/cosine_scaled_reward": 0.2798640683759004,
+      "rewards/format_reward": 0.8750000055879354,
+      "step": 135
+    },
+    {
+      "completion_length": 2301.437515258789,
+      "epoch": 0.15542857142857142,
+      "grad_norm": 0.029598237946629524,
+      "kl": 0.00010699033737182617,
+      "learning_rate": 9.230669076497687e-07,
+      "loss": 0.0,
+      "reward": 1.0288100242614746,
+      "reward_std": 0.6650557238608599,
+      "rewards/cosine_scaled_reward": 0.2227383404970169,
+      "rewards/format_reward": 0.5833333358168602,
+      "step": 136
+    },
+    {
+      "completion_length": 2888.0833740234375,
+      "epoch": 0.15657142857142858,
+      "grad_norm": 0.021254906430840492,
+      "kl": 0.00012347102165222168,
+      "learning_rate": 9.213010742252327e-07,
+      "loss": 0.0,
+      "reward": 0.15984412096440792,
+      "reward_std": 0.8047252930700779,
+      "rewards/cosine_scaled_reward": -0.12841127801220864,
+      "rewards/format_reward": 0.41666667722165585,
+      "step": 137
+    },
+    {
+      "completion_length": 2664.2500610351562,
+      "epoch": 0.15771428571428572,
+      "grad_norm": 0.04367408901453018,
+      "kl": 0.00011092424392700195,
+      "learning_rate": 9.195171441101668e-07,
+      "loss": 0.0,
+      "reward": 0.13648264110088348,
+      "reward_std": 0.6316529109608382,
+      "rewards/cosine_scaled_reward": -0.12967534782364964,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 138
+    },
+    {
+      "completion_length": 2937.541717529297,
+      "epoch": 0.15885714285714286,
+      "grad_norm": 0.02476118691265583,
+      "kl": 0.00015249848365783691,
+      "learning_rate": 9.177152042508077e-07,
+      "loss": 0.0,
+      "reward": 0.49358947202563286,
+      "reward_std": 0.7895509861409664,
+      "rewards/cosine_scaled_reward": -0.003205273300409317,
+      "rewards/format_reward": 0.5000000018626451,
+      "step": 139
+    },
+    {
+      "completion_length": 2945.854202270508,
+      "epoch": 0.16,
+      "grad_norm": 0.021247655153274536,
+      "kl": 0.00016349554061889648,
+      "learning_rate": 9.158953424711624e-07,
+      "loss": 0.0,
+      "reward": 0.7320265481248498,
+      "reward_std": 1.0867303423583508,
+      "rewards/cosine_scaled_reward": 0.12642992846667767,
+      "rewards/format_reward": 0.4791666828095913,
+      "step": 140
+    },
+    {
+      "completion_length": 2825.7292251586914,
+      "epoch": 0.16114285714285714,
+      "grad_norm": 0.02051922306418419,
+      "kl": 0.0001229420304298401,
+      "learning_rate": 9.140576474687263e-07,
+      "loss": 0.0,
+      "reward": 0.4059164673089981,
+      "reward_std": 0.9311309345066547,
+      "rewards/cosine_scaled_reward": -0.047041770070791245,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 141
+    },
+    {
+      "completion_length": 2637.9583892822266,
+      "epoch": 0.16228571428571428,
+      "grad_norm": 0.021125253289937973,
+      "kl": 0.0001480579376220703,
+      "learning_rate": 9.122022088101613e-07,
+      "loss": 0.0,
+      "reward": 0.559799462556839,
+      "reward_std": 0.6969177909195423,
+      "rewards/cosine_scaled_reward": -0.0221836119890213,
+      "rewards/format_reward": 0.6041666697710752,
+      "step": 142
+    },
+    {
+      "completion_length": 2216.8750762939453,
+      "epoch": 0.16342857142857142,
+      "grad_norm": 0.028433097526431084,
+      "kl": 0.00013668090105056763,
+      "learning_rate": 9.103291169269299e-07,
+      "loss": 0.0,
+      "reward": 0.6880653910338879,
+      "reward_std": 0.7187580987811089,
+      "rewards/cosine_scaled_reward": -0.020550659857690334,
+      "rewards/format_reward": 0.7291666697710752,
+      "step": 143
+    },
+    {
+      "completion_length": 2552.895881652832,
+      "epoch": 0.16457142857142856,
+      "grad_norm": 0.026991255581378937,
+      "kl": 0.00011313706636428833,
+      "learning_rate": 9.084384631108882e-07,
+      "loss": 0.0,
+      "reward": 0.8902260856702924,
+      "reward_std": 1.0206835009157658,
+      "rewards/cosine_scaled_reward": 0.16386302933096886,
+      "rewards/format_reward": 0.5625000111758709,
+      "step": 144
+    },
+    {
+      "completion_length": 1938.041732788086,
+      "epoch": 0.1657142857142857,
+      "grad_norm": 0.032270271331071854,
+      "kl": 0.00011307001113891602,
+      "learning_rate": 9.065303395098358e-07,
+      "loss": 0.0,
+      "reward": 0.7428423997480422,
+      "reward_std": 0.5232174023985863,
+      "rewards/cosine_scaled_reward": 0.05892119184136391,
+      "rewards/format_reward": 0.6250000055879354,
+      "step": 145
+    },
+    {
+      "completion_length": 2219.791702270508,
+      "epoch": 0.16685714285714287,
+      "grad_norm": 0.02581549808382988,
+      "kl": 0.00010230764746665955,
+      "learning_rate": 9.046048391230247e-07,
+      "loss": 0.0,
+      "reward": 0.07503729220479727,
+      "reward_std": 0.5477688852697611,
+      "rewards/cosine_scaled_reward": -0.2541480294894427,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 146
+    },
+    {
+      "completion_length": 3379.812530517578,
+      "epoch": 0.168,
+      "grad_norm": 0.01836305670440197,
+      "kl": 0.00017067790031433105,
+      "learning_rate": 9.026620557966279e-07,
+      "loss": 0.0,
+      "reward": 0.33875247836112976,
+      "reward_std": 0.7588643953204155,
+      "rewards/cosine_scaled_reward": 0.013126220554113388,
+      "rewards/format_reward": 0.31250001303851604,
+      "step": 147
+    },
+    {
+      "completion_length": 2358.4166870117188,
+      "epoch": 0.16914285714285715,
+      "grad_norm": 0.02416214346885681,
+      "kl": 0.0001198500394821167,
+      "learning_rate": 9.007020842191634e-07,
+      "loss": 0.0,
+      "reward": 0.6435984447598457,
+      "reward_std": 0.7261539697647095,
+      "rewards/cosine_scaled_reward": 0.009299194440245628,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 148
+    },
+    {
+      "completion_length": 2277.3750534057617,
+      "epoch": 0.1702857142857143,
+      "grad_norm": 0.021401530131697655,
+      "kl": 8.6270272731781e-05,
+      "learning_rate": 8.987250199168808e-07,
+      "loss": 0.0,
+      "reward": 1.1666594594717026,
+      "reward_std": 0.8260795883834362,
+      "rewards/cosine_scaled_reward": 0.18749637342989445,
+      "rewards/format_reward": 0.7916666753590107,
+      "step": 149
+    },
+    {
+      "completion_length": 2718.6667098999023,
+      "epoch": 0.17142857142857143,
+      "grad_norm": 0.026421265676617622,
+      "kl": 0.0001287609338760376,
+      "learning_rate": 8.967309592491052e-07,
+      "loss": 0.0,
+      "reward": 0.5824301056563854,
+      "reward_std": 0.8789757378399372,
+      "rewards/cosine_scaled_reward": 0.07246502116322517,
+      "rewards/format_reward": 0.4375000037252903,
+      "step": 150
+    },
+    {
+      "completion_length": 2435.0834045410156,
+      "epoch": 0.17257142857142857,
+      "grad_norm": 0.022044222801923752,
+      "kl": 0.00015279650688171387,
+      "learning_rate": 8.9471999940354e-07,
+      "loss": 0.0,
+      "reward": 1.020676076412201,
+      "reward_std": 0.8065766766667366,
+      "rewards/cosine_scaled_reward": 0.1770047047175467,
+      "rewards/format_reward": 0.6666666772216558,
+      "step": 151
+    },
+    {
+      "completion_length": 2837.125030517578,
+      "epoch": 0.1737142857142857,
+      "grad_norm": 0.03187905251979828,
+      "kl": 0.00020840764045715332,
+      "learning_rate": 8.926922383915315e-07,
+      "loss": 0.0,
+      "reward": -0.09739869087934494,
+      "reward_std": 0.588227073661983,
+      "rewards/cosine_scaled_reward": -0.21536601521074772,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 152
+    },
+    {
+      "completion_length": 2711.1666984558105,
+      "epoch": 0.17485714285714285,
+      "grad_norm": 0.02911432646214962,
+      "kl": 0.00015884637832641602,
+      "learning_rate": 8.906477750432903e-07,
+      "loss": 0.0,
+      "reward": 0.37669877288863063,
+      "reward_std": 0.8355071609839797,
+      "rewards/cosine_scaled_reward": -0.051233954494819045,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 153
+    },
+    {
+      "completion_length": 2756.3750228881836,
+      "epoch": 0.176,
+      "grad_norm": 0.02044747956097126,
+      "kl": 0.00012248754501342773,
+      "learning_rate": 8.88586709003076e-07,
+      "loss": 0.0,
+      "reward": 0.9225640147924423,
+      "reward_std": 0.8925285078585148,
+      "rewards/cosine_scaled_reward": 0.18003201112151146,
+      "rewards/format_reward": 0.5625000055879354,
+      "step": 154
+    },
+    {
+      "completion_length": 2223.312526702881,
+      "epoch": 0.17714285714285713,
+      "grad_norm": 0.03391764312982559,
+      "kl": 0.00013962388038635254,
+      "learning_rate": 8.865091407243394e-07,
+      "loss": 0.0,
+      "reward": 0.7340945638716221,
+      "reward_std": 0.9665955938398838,
+      "rewards/cosine_scaled_reward": 0.0649639368057251,
+      "rewards/format_reward": 0.6041666753590107,
+      "step": 155
+    },
+    {
+      "completion_length": 2733.0417098999023,
+      "epoch": 0.1782857142857143,
+      "grad_norm": 0.024309324100613594,
+      "kl": 0.00012956559658050537,
+      "learning_rate": 8.844151714648274e-07,
+      "loss": 0.0,
+      "reward": 0.21320721879601479,
+      "reward_std": 0.7121351584792137,
+      "rewards/cosine_scaled_reward": -0.08089640364050865,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 156
+    },
+    {
+      "completion_length": 2663.6041870117188,
+      "epoch": 0.17942857142857144,
+      "grad_norm": 0.023216916248202324,
+      "kl": 0.00014782696962356567,
+      "learning_rate": 8.823049032816478e-07,
+      "loss": 0.0,
+      "reward": 0.14814166724681854,
+      "reward_std": 0.3913048207759857,
+      "rewards/cosine_scaled_reward": -0.16551249055191875,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 157
+    },
+    {
+      "completion_length": 2328.541702270508,
+      "epoch": 0.18057142857142858,
+      "grad_norm": 0.03534715995192528,
+      "kl": 0.00013265013694763184,
+      "learning_rate": 8.801784390262943e-07,
+      "loss": 0.0,
+      "reward": 0.9912874735891819,
+      "reward_std": 0.8466089218854904,
+      "rewards/cosine_scaled_reward": 0.1727270409464836,
+      "rewards/format_reward": 0.645833345130086,
+      "step": 158
+    },
+    {
+      "completion_length": 3219.0833587646484,
+      "epoch": 0.18171428571428572,
+      "grad_norm": 0.02012258768081665,
+      "kl": 0.00016486644744873047,
+      "learning_rate": 8.780358823396352e-07,
+      "loss": 0.0,
+      "reward": 0.21142571535892785,
+      "reward_std": 0.7264946773648262,
+      "rewards/cosine_scaled_reward": -0.019287142204120755,
+      "rewards/format_reward": 0.25000000186264515,
+      "step": 159
+    },
+    {
+      "completion_length": 2777.000045776367,
+      "epoch": 0.18285714285714286,
+      "grad_norm": 0.025370828807353973,
+      "kl": 0.0001633763313293457,
+      "learning_rate": 8.758773376468604e-07,
+      "loss": 0.0,
+      "reward": 0.5651849992573261,
+      "reward_std": 0.7188206501305103,
+      "rewards/cosine_scaled_reward": 0.06384249404072762,
+      "rewards/format_reward": 0.43750000558793545,
+      "step": 160
+    },
+    {
+      "completion_length": 2474.541732788086,
+      "epoch": 0.184,
+      "grad_norm": 0.030321095138788223,
+      "kl": 0.00015471689403057098,
+      "learning_rate": 8.737029101523929e-07,
+      "loss": 0.0,
+      "reward": 0.7223290950059891,
+      "reward_std": 0.8424487709999084,
+      "rewards/cosine_scaled_reward": 0.05908120982348919,
+      "rewards/format_reward": 0.6041666828095913,
+      "step": 161
+    },
+    {
+      "completion_length": 3007.937530517578,
+      "epoch": 0.18514285714285714,
+      "grad_norm": 0.023185160011053085,
+      "kl": 0.00018095970153808594,
+      "learning_rate": 8.715127058347614e-07,
+      "loss": 0.0,
+      "reward": 0.43497929722070694,
+      "reward_std": 0.8792226985096931,
+      "rewards/cosine_scaled_reward": 0.009156312793493271,
+      "rewards/format_reward": 0.4166666753590107,
+      "step": 162
+    },
+    {
+      "completion_length": 2370.145866394043,
+      "epoch": 0.18628571428571428,
+      "grad_norm": 0.025774484500288963,
+      "kl": 0.00015078485012054443,
+      "learning_rate": 8.693068314414344e-07,
+      "loss": 0.0,
+      "reward": 0.9850538782775402,
+      "reward_std": 0.45399810932576656,
+      "rewards/cosine_scaled_reward": 0.1904435846954584,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 163
+    },
+    {
+      "completion_length": 2291.64591217041,
+      "epoch": 0.18742857142857142,
+      "grad_norm": 0.032293785363435745,
+      "kl": 0.00010757520794868469,
+      "learning_rate": 8.670853944836176e-07,
+      "loss": 0.0,
+      "reward": 0.8117690533399582,
+      "reward_std": 0.7221045140177011,
+      "rewards/cosine_scaled_reward": 0.11421786062419415,
+      "rewards/format_reward": 0.5833333376795053,
+      "step": 164
+    },
+    {
+      "completion_length": 2779.3125534057617,
+      "epoch": 0.18857142857142858,
+      "grad_norm": 0.02514166198670864,
+      "kl": 0.0001589655876159668,
+      "learning_rate": 8.648485032310144e-07,
+      "loss": 0.0,
+      "reward": 0.16848786175251007,
+      "reward_std": 0.7497027926146984,
+      "rewards/cosine_scaled_reward": -0.11367274331860244,
+      "rewards/format_reward": 0.39583333395421505,
+      "step": 165
+    },
+    {
+      "completion_length": 2580.3542251586914,
+      "epoch": 0.18971428571428572,
+      "grad_norm": 0.01945258490741253,
+      "kl": 0.00012123212218284607,
+      "learning_rate": 8.625962667065487e-07,
+      "loss": 0.0,
+      "reward": 0.7501739612780511,
+      "reward_std": 1.0527099072933197,
+      "rewards/cosine_scaled_reward": 0.10425363201647997,
+      "rewards/format_reward": 0.5416666753590107,
+      "step": 166
+    },
+    {
+      "completion_length": 2261.9166946411133,
+      "epoch": 0.19085714285714286,
+      "grad_norm": 0.024124927818775177,
+      "kl": 0.00012174248695373535,
+      "learning_rate": 8.603287946810513e-07,
+      "loss": 0.0,
+      "reward": 0.5743089579045773,
+      "reward_std": 0.7672125902026892,
+      "rewards/cosine_scaled_reward": -0.014928878052160144,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 167
+    },
+    {
+      "completion_length": 2618.8958587646484,
+      "epoch": 0.192,
+      "grad_norm": 0.02464590221643448,
+      "kl": 0.00015217065811157227,
+      "learning_rate": 8.580461976679099e-07,
+      "loss": 0.0,
+      "reward": 0.6550418548285961,
+      "reward_std": 0.8852942362427711,
+      "rewards/cosine_scaled_reward": 0.056687585078179836,
+      "rewards/format_reward": 0.5416666753590107,
+      "step": 168
+    },
+    {
+      "completion_length": 1920.0208740234375,
+      "epoch": 0.19314285714285714,
+      "grad_norm": 0.02443297766149044,
+      "kl": 0.00012825848534703255,
+      "learning_rate": 8.557485869176825e-07,
+      "loss": 0.0,
+      "reward": 1.19900780916214,
+      "reward_std": 0.48580580204725266,
+      "rewards/cosine_scaled_reward": 0.2870038769906387,
+      "rewards/format_reward": 0.6250000055879354,
+      "step": 169
+    },
+    {
+      "completion_length": 2522.541702270508,
+      "epoch": 0.19428571428571428,
+      "grad_norm": 0.028783444315195084,
+      "kl": 0.00019599497318267822,
+      "learning_rate": 8.534360744126753e-07,
+      "loss": 0.0,
+      "reward": 0.48285481217317283,
+      "reward_std": 0.5786700565367937,
+      "rewards/cosine_scaled_reward": 0.0018440568819642067,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 170
+    },
+    {
+      "completion_length": 2279.4166870117188,
+      "epoch": 0.19542857142857142,
+      "grad_norm": 0.028025727719068527,
+      "kl": 0.0001086406409740448,
+      "learning_rate": 8.511087728614862e-07,
+      "loss": 0.0,
+      "reward": 0.6512754829600453,
+      "reward_std": 0.7119449377059937,
+      "rewards/cosine_scaled_reward": 0.06522107869386673,
+      "rewards/format_reward": 0.520833345130086,
+      "step": 171
+    },
+    {
+      "completion_length": 2717.416732788086,
+      "epoch": 0.19657142857142856,
+      "grad_norm": 0.0323350615799427,
+      "kl": 0.00017327070236206055,
+      "learning_rate": 8.487667956935087e-07,
+      "loss": 0.0,
+      "reward": 0.43108027055859566,
+      "reward_std": 0.6473873369395733,
+      "rewards/cosine_scaled_reward": 0.007206789217889309,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 172
+    },
+    {
+      "completion_length": 1855.1458549499512,
+      "epoch": 0.1977142857142857,
+      "grad_norm": 0.05176561325788498,
+      "kl": 0.00010795891284942627,
+      "learning_rate": 8.464102570534061e-07,
+      "loss": 0.0,
+      "reward": 0.4854015044402331,
+      "reward_std": 0.7094262056052685,
+      "rewards/cosine_scaled_reward": -0.059382592036854476,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 173
+    },
+    {
+      "completion_length": 1968.1458587646484,
+      "epoch": 0.19885714285714284,
+      "grad_norm": 0.026019148528575897,
+      "kl": 0.0001271367073059082,
+      "learning_rate": 8.440392717955475e-07,
+      "loss": 0.0,
+      "reward": 0.9800305366516113,
+      "reward_std": 0.8449381552636623,
+      "rewards/cosine_scaled_reward": 0.09418194030877203,
+      "rewards/format_reward": 0.791666679084301,
+      "step": 174
+    },
+    {
+      "completion_length": 2335.395835876465,
+      "epoch": 0.2,
+      "grad_norm": 0.02863323502242565,
+      "kl": 0.00011295080184936523,
+      "learning_rate": 8.416539554784089e-07,
+      "loss": 0.0,
+      "reward": 0.6143988557159901,
+      "reward_std": 0.4556568693369627,
+      "rewards/cosine_scaled_reward": 0.05719940923154354,
+      "rewards/format_reward": 0.5,
+      "step": 175
+    },
+    {
+      "completion_length": 2449.2708778381348,
+      "epoch": 0.20114285714285715,
+      "grad_norm": 0.02869235724210739,
+      "kl": 0.00014215707778930664,
+      "learning_rate": 8.392544243589427e-07,
+      "loss": 0.0,
+      "reward": 1.1555413324385881,
+      "reward_std": 1.0306697580963373,
+      "rewards/cosine_scaled_reward": 0.24443731893552467,
+      "rewards/format_reward": 0.6666666753590107,
+      "step": 176
+    },
+    {
+      "completion_length": 2694.562515258789,
+      "epoch": 0.2022857142857143,
+      "grad_norm": 0.023254889994859695,
+      "kl": 0.00017695128917694092,
+      "learning_rate": 8.368407953869103e-07,
+      "loss": 0.0,
+      "reward": 0.26045722933486104,
+      "reward_std": 0.7560887522995472,
+      "rewards/cosine_scaled_reward": -0.11977138603106141,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 177
+    },
+    {
+      "completion_length": 2326.645896911621,
+      "epoch": 0.20342857142857143,
+      "grad_norm": 0.05164008587598801,
+      "kl": 0.0001480579376220703,
+      "learning_rate": 8.344131861991828e-07,
+      "loss": 0.0,
+      "reward": 0.6920488923788071,
+      "reward_std": 0.6076631285250187,
+      "rewards/cosine_scaled_reward": 0.0439411043189466,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 178
+    },
+    {
+      "completion_length": 2577.729202270508,
+      "epoch": 0.20457142857142857,
+      "grad_norm": 0.025095604360103607,
+      "kl": 0.0001359879970550537,
+      "learning_rate": 8.319717151140072e-07,
+      "loss": 0.0,
+      "reward": 0.22021218854933977,
+      "reward_std": 0.6630701459944248,
+      "rewards/cosine_scaled_reward": -0.1294772457331419,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 179
+    },
+    {
+      "completion_length": 2143.1666870117188,
+      "epoch": 0.2057142857142857,
+      "grad_norm": 0.03228743001818657,
+      "kl": 0.00014454126358032227,
+      "learning_rate": 8.295165011252396e-07,
+      "loss": 0.0,
+      "reward": 0.7503797290846705,
+      "reward_std": 0.2802760358899832,
+      "rewards/cosine_scaled_reward": 0.06268983148038387,
+      "rewards/format_reward": 0.625,
+      "step": 180
+    },
+    {
+      "completion_length": 2969.541702270508,
+      "epoch": 0.20685714285714285,
+      "grad_norm": 0.022628450766205788,
+      "kl": 0.00018687546253204346,
+      "learning_rate": 8.270476638965461e-07,
+      "loss": 0.0,
+      "reward": 0.40104219946078956,
+      "reward_std": 0.6805736906826496,
+      "rewards/cosine_scaled_reward": 0.002604421228170395,
+      "rewards/format_reward": 0.3958333358168602,
+      "step": 181
+    },
+    {
+      "completion_length": 1776.2708587646484,
+      "epoch": 0.208,
+      "grad_norm": 0.025273211300373077,
+      "kl": 9.524030610918999e-05,
+      "learning_rate": 8.245653237555705e-07,
+      "loss": 0.0,
+      "reward": 0.8596987724304199,
+      "reward_std": 0.5180976707488298,
+      "rewards/cosine_scaled_reward": 0.05484938062727451,
+      "rewards/format_reward": 0.75,
+      "step": 182
+    },
+    {
+      "completion_length": 1851.4792022705078,
+      "epoch": 0.20914285714285713,
+      "grad_norm": 0.027847742661833763,
+      "kl": 9.390711784362793e-05,
+      "learning_rate": 8.220696016880687e-07,
+      "loss": 0.0,
+      "reward": 0.8440241813659668,
+      "reward_std": 0.6227804413065314,
+      "rewards/cosine_scaled_reward": 0.06784541811794043,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 183
+    },
+    {
+      "completion_length": 2858.520854949951,
+      "epoch": 0.2102857142857143,
+      "grad_norm": 0.035307928919792175,
+      "kl": 0.0001779794692993164,
+      "learning_rate": 8.195606193320136e-07,
+      "loss": 0.0,
+      "reward": 0.06458963826298714,
+      "reward_std": 0.34594947658479214,
+      "rewards/cosine_scaled_reward": -0.14478851668536663,
+      "rewards/format_reward": 0.3541666716337204,
+      "step": 184
+    },
+    {
+      "completion_length": 2389.1250038146973,
+      "epoch": 0.21142857142857144,
+      "grad_norm": 0.02831236459314823,
+      "kl": 0.00010818615555763245,
+      "learning_rate": 8.170384989716657e-07,
+      "loss": 0.0,
+      "reward": 0.5413765218108892,
+      "reward_std": 0.4714433569461107,
+      "rewards/cosine_scaled_reward": -0.00014507770538330078,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 185
+    },
+    {
+      "completion_length": 2767.6666870117188,
+      "epoch": 0.21257142857142858,
+      "grad_norm": 0.023117488250136375,
+      "kl": 0.0001722574234008789,
+      "learning_rate": 8.145033635316128e-07,
+      "loss": 0.0,
+      "reward": 0.41337184607982635,
+      "reward_std": 0.5102072861045599,
+      "rewards/cosine_scaled_reward": 0.01918593794107437,
+      "rewards/format_reward": 0.375,
+      "step": 186
+    },
+    {
+      "completion_length": 2513.5000381469727,
+      "epoch": 0.21371428571428572,
+      "grad_norm": 0.028418753296136856,
+      "kl": 0.00018349289894104004,
+      "learning_rate": 8.119553365707802e-07,
+      "loss": 0.0,
+      "reward": 0.25105898082256317,
+      "reward_std": 0.609873728826642,
+      "rewards/cosine_scaled_reward": -0.10363717749714851,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 187
+    },
+    {
+      "completion_length": 3422.1458435058594,
+      "epoch": 0.21485714285714286,
+      "grad_norm": 0.017747724428772926,
+      "kl": 0.00021284818649291992,
+      "learning_rate": 8.093945422764069e-07,
+      "loss": 0.0,
+      "reward": 0.0721919247880578,
+      "reward_std": 0.6335567105561495,
+      "rewards/cosine_scaled_reward": -0.07848738227039576,
+      "rewards/format_reward": 0.2291666716337204,
+      "step": 188
+    },
+    {
+      "completion_length": 1912.0000305175781,
+      "epoch": 0.216,
+      "grad_norm": 0.03303361311554909,
+      "kl": 0.00013190507888793945,
+      "learning_rate": 8.068211054579943e-07,
+      "loss": 0.0,
+      "reward": 0.6457886800635606,
+      "reward_std": 0.5706375325098634,
+      "rewards/cosine_scaled_reward": -0.010438980534672737,
+      "rewards/format_reward": 0.6666666679084301,
+      "step": 189
+    },
+    {
+      "completion_length": 2566.1666831970215,
+      "epoch": 0.21714285714285714,
+      "grad_norm": 0.02325809746980667,
+      "kl": 0.00013524293899536133,
+      "learning_rate": 8.04235151541222e-07,
+      "loss": 0.0,
+      "reward": 0.6750274561345577,
+      "reward_std": 0.8541564792394638,
+      "rewards/cosine_scaled_reward": 0.08751372992992401,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 190
+    },
+    {
+      "completion_length": 2105.062530517578,
+      "epoch": 0.21828571428571428,
+      "grad_norm": 0.029074614867568016,
+      "kl": 0.0001640021800994873,
+      "learning_rate": 8.01636806561836e-07,
+      "loss": 0.0,
+      "reward": 1.0512720253318548,
+      "reward_std": 0.7890468016266823,
+      "rewards/cosine_scaled_reward": 0.1923026624135673,
+      "rewards/format_reward": 0.6666666734963655,
+      "step": 191
+    },
+    {
+      "completion_length": 3006.625045776367,
+      "epoch": 0.21942857142857142,
+      "grad_norm": 0.018196215853095055,
+      "kl": 0.0001971721649169922,
+      "learning_rate": 7.990261971595048e-07,
+      "loss": 0.0,
+      "reward": 0.1856545265763998,
+      "reward_std": 0.5431043151766062,
+      "rewards/cosine_scaled_reward": -0.11550608882680535,
+      "rewards/format_reward": 0.41666667722165585,
+      "step": 192
+    },
+    {
+      "completion_length": 2791.708396911621,
+      "epoch": 0.22057142857142858,
+      "grad_norm": 0.026738908141851425,
+      "kl": 0.00015562772750854492,
+      "learning_rate": 7.964034505716476e-07,
+      "loss": 0.0,
+      "reward": 0.6532938107848167,
+      "reward_std": 0.8042124956846237,
+      "rewards/cosine_scaled_reward": 0.03498022351413965,
+      "rewards/format_reward": 0.5833333432674408,
+      "step": 193
+    },
+    {
+      "completion_length": 2549.3333740234375,
+      "epoch": 0.22171428571428572,
+      "grad_norm": 0.02125261351466179,
+      "kl": 0.00017492473125457764,
+      "learning_rate": 7.93768694627233e-07,
+      "loss": 0.0,
+      "reward": 1.3440267005935311,
+      "reward_std": 0.6356805637478828,
+      "rewards/cosine_scaled_reward": 0.33868001215159893,
+      "rewards/format_reward": 0.6666666734963655,
+      "step": 194
+    },
+    {
+      "completion_length": 2541.9375610351562,
+      "epoch": 0.22285714285714286,
+      "grad_norm": 0.021625977009534836,
+      "kl": 0.00013524293899536133,
+      "learning_rate": 7.911220577405484e-07,
+      "loss": 0.0,
+      "reward": 0.45053595933131874,
+      "reward_std": 0.7725311703979969,
+      "rewards/cosine_scaled_reward": -0.055982012301683426,
+      "rewards/format_reward": 0.5625000018626451,
+      "step": 195
+    },
+    {
+      "completion_length": 3335.7083435058594,
+      "epoch": 0.224,
+      "grad_norm": 0.019518742337822914,
+      "kl": 0.00020563602447509766,
+      "learning_rate": 7.884636689049422e-07,
+      "loss": 0.0,
+      "reward": 0.07744309306144714,
+      "reward_std": 0.6891645342111588,
+      "rewards/cosine_scaled_reward": -0.13836179114878178,
+      "rewards/format_reward": 0.354166679084301,
+      "step": 196
+    },
+    {
+      "completion_length": 2233.312545776367,
+      "epoch": 0.22514285714285714,
+      "grad_norm": 0.031853772699832916,
+      "kl": 0.00016062520444393158,
+      "learning_rate": 7.857936576865356e-07,
+      "loss": 0.0,
+      "reward": 1.7223949134349823,
+      "reward_std": 1.0976359993219376,
+      "rewards/cosine_scaled_reward": 0.47578078508377075,
+      "rewards/format_reward": 0.7708333432674408,
+      "step": 197
+    },
+    {
+      "completion_length": 2403.312545776367,
+      "epoch": 0.22628571428571428,
+      "grad_norm": 0.02528354711830616,
+      "kl": 0.0001506805419921875,
+      "learning_rate": 7.831121542179086e-07,
+      "loss": 0.0,
+      "reward": 0.725980069488287,
+      "reward_std": 0.9397312067449093,
+      "rewards/cosine_scaled_reward": 0.07132336869835854,
+      "rewards/format_reward": 0.583333345130086,
+      "step": 198
+    },
+    {
+      "completion_length": 3550.500030517578,
+      "epoch": 0.22742857142857142,
+      "grad_norm": 0.01737813465297222,
+      "kl": 0.00018775463104248047,
+      "learning_rate": 7.804192891917571e-07,
+      "loss": 0.0,
+      "reward": -0.2171228351071477,
+      "reward_std": 0.7574310433119535,
+      "rewards/cosine_scaled_reward": -0.17106142034754157,
+      "rewards/format_reward": 0.12500000186264515,
+      "step": 199
+    },
+    {
+      "completion_length": 1939.5833740234375,
+      "epoch": 0.22857142857142856,
+      "grad_norm": 0.025537090376019478,
+      "kl": 0.00011917948722839355,
+      "learning_rate": 7.777151938545235e-07,
+      "loss": 0.0,
+      "reward": 1.1004857262596488,
+      "reward_std": 0.7319156341254711,
+      "rewards/cosine_scaled_reward": 0.16482618637382984,
+      "rewards/format_reward": 0.7708333358168602,
+      "step": 200
+    },
+    {
+      "completion_length": 2528.5209045410156,
+      "epoch": 0.2297142857142857,
+      "grad_norm": 0.026324188336730003,
+      "kl": 0.00013250857591629028,
+      "learning_rate": 7.75e-07,
+      "loss": 0.0,
+      "reward": 1.465837650001049,
+      "reward_std": 0.9725578310899436,
+      "rewards/cosine_scaled_reward": 0.3683354835957289,
+      "rewards/format_reward": 0.7291666828095913,
+      "step": 201
+    },
+    {
+      "completion_length": 2129.250030517578,
+      "epoch": 0.23085714285714284,
+      "grad_norm": 0.027547147125005722,
+      "kl": 0.00012653321027755737,
+      "learning_rate": 7.72273839962904e-07,
+      "loss": 0.0,
+      "reward": 1.2678888477385044,
+      "reward_std": 0.42415672820061445,
+      "rewards/cosine_scaled_reward": 0.3422777857631445,
+      "rewards/format_reward": 0.5833333358168602,
+      "step": 202
+    },
+    {
+      "completion_length": 3210.416717529297,
+      "epoch": 0.232,
+      "grad_norm": 0.02026360109448433,
+      "kl": 0.0002213120460510254,
+      "learning_rate": 7.695368466124296e-07,
+      "loss": 0.0,
+      "reward": 0.32417835108935833,
+      "reward_std": 0.8184224963188171,
+      "rewards/cosine_scaled_reward": -0.014994161203503609,
+      "rewards/format_reward": 0.35416667349636555,
+      "step": 203
+    },
+    {
+      "completion_length": 1677.7083892822266,
+      "epoch": 0.23314285714285715,
+      "grad_norm": 0.03866266459226608,
+      "kl": 0.0001564323902130127,
+      "learning_rate": 7.667891533457718e-07,
+      "loss": 0.0,
+      "reward": 0.9787286967039108,
+      "reward_std": 0.7050071842968464,
+      "rewards/cosine_scaled_reward": 0.11436433251947165,
+      "rewards/format_reward": 0.7500000018626451,
+      "step": 204
+    },
+    {
+      "completion_length": 2191.7500762939453,
+      "epoch": 0.2342857142857143,
+      "grad_norm": 0.026666179299354553,
+      "kl": 0.00016039609909057617,
+      "learning_rate": 7.640308940816239e-07,
+      "loss": 0.0,
+      "reward": 1.4601695500314236,
+      "reward_std": 0.8596113231033087,
+      "rewards/cosine_scaled_reward": 0.3863347489386797,
+      "rewards/format_reward": 0.687500013038516,
+      "step": 205
+    },
+    {
+      "completion_length": 2625.3958435058594,
+      "epoch": 0.23542857142857143,
+      "grad_norm": 0.02177976816892624,
+      "kl": 0.00015854835510253906,
+      "learning_rate": 7.612622032536507e-07,
+      "loss": 0.0,
+      "reward": 0.12173939868807793,
+      "reward_std": 0.6548620201647282,
+      "rewards/cosine_scaled_reward": -0.15788030996918678,
+      "rewards/format_reward": 0.4375000074505806,
+      "step": 206
+    },
+    {
+      "completion_length": 2883.0625610351562,
+      "epoch": 0.23657142857142857,
+      "grad_norm": 0.026248862966895103,
+      "kl": 0.00018286705017089844,
+      "learning_rate": 7.584832158039378e-07,
+      "loss": 0.0,
+      "reward": 0.2657412812113762,
+      "reward_std": 0.7520406804978848,
+      "rewards/cosine_scaled_reward": -0.11712937615811825,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 207
+    },
+    {
+      "completion_length": 2729.125015258789,
+      "epoch": 0.2377142857142857,
+      "grad_norm": 0.022093147039413452,
+      "kl": 0.0001411736011505127,
+      "learning_rate": 7.556940671764124e-07,
+      "loss": 0.0,
+      "reward": 0.47902350744698197,
+      "reward_std": 0.7772372476756573,
+      "rewards/cosine_scaled_reward": 0.020761748775839806,
+      "rewards/format_reward": 0.43750000558793545,
+      "step": 208
+    },
+    {
+      "completion_length": 2192.9792289733887,
+      "epoch": 0.23885714285714285,
+      "grad_norm": 0.026345793157815933,
+      "kl": 0.00014121830463409424,
+      "learning_rate": 7.528948933102438e-07,
+      "loss": 0.0,
+      "reward": 1.030539353378117,
+      "reward_std": 0.5581800434738398,
+      "rewards/cosine_scaled_reward": 0.16110298968851566,
+      "rewards/format_reward": 0.7083333395421505,
+      "step": 209
+    },
+    {
+      "completion_length": 2763.8541870117188,
+      "epoch": 0.24,
+      "grad_norm": 0.0209537073969841,
+      "kl": 0.00013713538646697998,
+      "learning_rate": 7.500858306332172e-07,
+      "loss": 0.0,
+      "reward": 0.7295061359182,
+      "reward_std": 0.8547591110691428,
+      "rewards/cosine_scaled_reward": 0.10433638375252485,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 210
+    },
+    {
+      "completion_length": 2132.6041870117188,
+      "epoch": 0.24114285714285713,
+      "grad_norm": 0.029190730303525925,
+      "kl": 0.00014358758926391602,
+      "learning_rate": 7.472670160550848e-07,
+      "loss": 0.0,
+      "reward": 0.844088938087225,
+      "reward_std": 0.7269210359081626,
+      "rewards/cosine_scaled_reward": 0.09912779554724693,
+      "rewards/format_reward": 0.6458333358168602,
+      "step": 211
+    },
+    {
+      "completion_length": 1901.6458587646484,
+      "epoch": 0.2422857142857143,
+      "grad_norm": 0.03538208082318306,
+      "kl": 0.00014218688011169434,
+      "learning_rate": 7.444385869608921e-07,
+      "loss": 0.0,
+      "reward": 0.6045566711109132,
+      "reward_std": 0.5201306939125061,
+      "rewards/cosine_scaled_reward": -0.010221696458756924,
+      "rewards/format_reward": 0.6250000037252903,
+      "step": 212
+    },
+    {
+      "completion_length": 1943.0000381469727,
+      "epoch": 0.24342857142857144,
+      "grad_norm": 0.03535102307796478,
+      "kl": 0.00016012787818908691,
+      "learning_rate": 7.416006812042827e-07,
+      "loss": 0.0,
+      "reward": 1.4567882642149925,
+      "reward_std": 0.7881541578099132,
+      "rewards/cosine_scaled_reward": 0.3742274232208729,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 213
+    },
+    {
+      "completion_length": 2420.416702270508,
+      "epoch": 0.24457142857142858,
+      "grad_norm": 0.03149150311946869,
+      "kl": 0.00019598007202148438,
+      "learning_rate": 7.387534371007797e-07,
+      "loss": 0.0,
+      "reward": 0.46870883740484715,
+      "reward_std": 0.7327337116003036,
+      "rewards/cosine_scaled_reward": -0.06772891082800925,
+      "rewards/format_reward": 0.6041666809469461,
+      "step": 214
+    },
+    {
+      "completion_length": 1979.6250610351562,
+      "epoch": 0.24571428571428572,
+      "grad_norm": 0.03919277340173721,
+      "kl": 0.00014135241508483887,
+      "learning_rate": 7.358969934210438e-07,
+      "loss": 0.0,
+      "reward": 0.5261248834431171,
+      "reward_std": 0.8126789703965187,
+      "rewards/cosine_scaled_reward": -0.09110424015671015,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 215
+    },
+    {
+      "completion_length": 1884.812515258789,
+      "epoch": 0.24685714285714286,
+      "grad_norm": 0.0325796902179718,
+      "kl": 0.0001468956470489502,
+      "learning_rate": 7.330314893841101e-07,
+      "loss": 0.0,
+      "reward": 0.7279386296868324,
+      "reward_std": 0.569173252210021,
+      "rewards/cosine_scaled_reward": 0.030635960400104523,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 216
+    },
+    {
+      "completion_length": 2459.5833435058594,
+      "epoch": 0.248,
+      "grad_norm": 0.025857582688331604,
+      "kl": 0.00016328692436218262,
+      "learning_rate": 7.301570646506027e-07,
+      "loss": 0.0,
+      "reward": 0.7756138246040791,
+      "reward_std": 0.9014986008405685,
+      "rewards/cosine_scaled_reward": 0.09614021610468626,
+      "rewards/format_reward": 0.583333333954215,
+      "step": 217
+    },
+    {
+      "completion_length": 2637.5208740234375,
+      "epoch": 0.24914285714285714,
+      "grad_norm": 0.02500126138329506,
+      "kl": 0.00015026330947875977,
+      "learning_rate": 7.27273859315928e-07,
+      "loss": 0.0,
+      "reward": 0.6237838268280029,
+      "reward_std": 0.6200420390814543,
+      "rewards/cosine_scaled_reward": 0.07230857852846384,
+      "rewards/format_reward": 0.47916667722165585,
+      "step": 218
+    },
+    {
+      "completion_length": 2157.6666870117188,
+      "epoch": 0.2502857142857143,
+      "grad_norm": 0.02829655073583126,
+      "kl": 0.00017982721328735352,
+      "learning_rate": 7.243820139034464e-07,
+      "loss": 0.0,
+      "reward": 1.0598554275929928,
+      "reward_std": 0.7442222200334072,
+      "rewards/cosine_scaled_reward": 0.1757610123604536,
+      "rewards/format_reward": 0.7083333395421505,
+      "step": 219
+    },
+    {
+      "completion_length": 2727.3541946411133,
+      "epoch": 0.25142857142857145,
+      "grad_norm": 0.0359327532351017,
+      "kl": 0.00015404075384140015,
+      "learning_rate": 7.214816693576234e-07,
+      "loss": 0.0,
+      "reward": -0.1477277409285307,
+      "reward_std": 0.410174666903913,
+      "rewards/cosine_scaled_reward": -0.2509472072124481,
+      "rewards/format_reward": 0.35416667722165585,
+      "step": 220
+    },
+    {
+      "completion_length": 1798.3125038146973,
+      "epoch": 0.25257142857142856,
+      "grad_norm": 0.02666349895298481,
+      "kl": 0.00010177493095397949,
+      "learning_rate": 7.185729670371604e-07,
+      "loss": 0.0,
+      "reward": 0.9729608483612537,
+      "reward_std": 0.3803430050611496,
+      "rewards/cosine_scaled_reward": 0.13231376186013222,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 221
+    },
+    {
+      "completion_length": 1928.5000228881836,
+      "epoch": 0.2537142857142857,
+      "grad_norm": 0.02487647347152233,
+      "kl": 0.00014640390872955322,
+      "learning_rate": 7.156560487081051e-07,
+      "loss": 0.0,
+      "reward": 1.0013896897435188,
+      "reward_std": 0.6300395149737597,
+      "rewards/cosine_scaled_reward": 0.14652817510068417,
+      "rewards/format_reward": 0.708333333954215,
+      "step": 222
+    },
+    {
+      "completion_length": 1702.7708435058594,
+      "epoch": 0.25485714285714284,
+      "grad_norm": 0.02845071442425251,
+      "kl": 0.00010716915130615234,
+      "learning_rate": 7.127310565369415e-07,
+      "loss": 0.0,
+      "reward": 0.9212910756468773,
+      "reward_std": 0.6102632470428944,
+      "rewards/cosine_scaled_reward": 0.08564550150185823,
+      "rewards/format_reward": 0.75,
+      "step": 223
+    },
+    {
+      "completion_length": 2827.3959045410156,
+      "epoch": 0.256,
+      "grad_norm": 0.021246662363409996,
+      "kl": 0.00013241171836853027,
+      "learning_rate": 7.097981330836616e-07,
+      "loss": 0.0,
+      "reward": 0.6548797897994518,
+      "reward_std": 0.7224124409258366,
+      "rewards/cosine_scaled_reward": 0.06702320463955402,
+      "rewards/format_reward": 0.5208333358168602,
+      "step": 224
+    },
+    {
+      "completion_length": 2509.5625228881836,
+      "epoch": 0.2571428571428571,
+      "grad_norm": 0.027184953913092613,
+      "kl": 0.00013971328735351562,
+      "learning_rate": 7.068574212948169e-07,
+      "loss": 0.0,
+      "reward": 0.44291812740266323,
+      "reward_std": 0.8240737412124872,
+      "rewards/cosine_scaled_reward": -0.028540948405861855,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 225
+    },
+    {
+      "completion_length": 2216.375011444092,
+      "epoch": 0.2582857142857143,
+      "grad_norm": 0.026975193992257118,
+      "kl": 0.00015223026275634766,
+      "learning_rate": 7.039090644965509e-07,
+      "loss": 0.0,
+      "reward": 0.8716886155307293,
+      "reward_std": 0.5759567692875862,
+      "rewards/cosine_scaled_reward": 0.15459428541362286,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 226
+    },
+    {
+      "completion_length": 1686.8958702087402,
+      "epoch": 0.25942857142857145,
+      "grad_norm": 0.03848085552453995,
+      "kl": 0.00011584162712097168,
+      "learning_rate": 7.009532063876148e-07,
+      "loss": 0.0,
+      "reward": 1.1075771823525429,
+      "reward_std": 0.913255337625742,
+      "rewards/cosine_scaled_reward": 0.1475385595113039,
+      "rewards/format_reward": 0.8125000149011612,
+      "step": 227
+    },
+    {
+      "completion_length": 1939.8958435058594,
+      "epoch": 0.26057142857142856,
+      "grad_norm": 0.03354226425290108,
+      "kl": 0.00010298937559127808,
+      "learning_rate": 6.979899910323624e-07,
+      "loss": 0.0,
+      "reward": 0.8404038224834949,
+      "reward_std": 0.31906691938638687,
+      "rewards/cosine_scaled_reward": 0.13895191438496113,
+      "rewards/format_reward": 0.5625,
+      "step": 228
+    },
+    {
+      "completion_length": 3084.041717529297,
+      "epoch": 0.26171428571428573,
+      "grad_norm": 0.02156384289264679,
+      "kl": 0.00019919872283935547,
+      "learning_rate": 6.950195628537299e-07,
+      "loss": 0.0,
+      "reward": 0.2632536366581917,
+      "reward_std": 0.7106823846697807,
+      "rewards/cosine_scaled_reward": -0.02462319377809763,
+      "rewards/format_reward": 0.3125000037252903,
+      "step": 229
+    },
+    {
+      "completion_length": 2789.5208892822266,
+      "epoch": 0.26285714285714284,
+      "grad_norm": 0.022447209805250168,
+      "kl": 0.0001659989356994629,
+      "learning_rate": 6.920420666261961e-07,
+      "loss": 0.0,
+      "reward": 0.6103229657746851,
+      "reward_std": 0.9964038021862507,
+      "rewards/cosine_scaled_reward": 0.034328147768974304,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 230
+    },
+    {
+      "completion_length": 2335.125030517578,
+      "epoch": 0.264,
+      "grad_norm": 0.02745773270726204,
+      "kl": 0.00013434886932373047,
+      "learning_rate": 6.890576474687263e-07,
+      "loss": 0.0,
+      "reward": 0.8026296477764845,
+      "reward_std": 0.6375624313950539,
+      "rewards/cosine_scaled_reward": 0.08881480386480689,
+      "rewards/format_reward": 0.6250000018626451,
+      "step": 231
+    },
+    {
+      "completion_length": 2833.791763305664,
+      "epoch": 0.2651428571428571,
+      "grad_norm": 0.02423485741019249,
+      "kl": 0.00019976496696472168,
+      "learning_rate": 6.860664508377001e-07,
+      "loss": 0.0,
+      "reward": 0.7053144201636314,
+      "reward_std": 0.9186714664101601,
+      "rewards/cosine_scaled_reward": 0.08182389056310058,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 232
+    },
+    {
+      "completion_length": 1837.6666946411133,
+      "epoch": 0.2662857142857143,
+      "grad_norm": 0.03410692512989044,
+      "kl": 0.0001348555088043213,
+      "learning_rate": 6.83068622519821e-07,
+      "loss": 0.0,
+      "reward": 1.0225979089736938,
+      "reward_std": 1.1002402231097221,
+      "rewards/cosine_scaled_reward": 0.15713229309767485,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 233
+    },
+    {
+      "completion_length": 2508.9791889190674,
+      "epoch": 0.2674285714285714,
+      "grad_norm": 0.03137635439634323,
+      "kl": 0.0001417398452758789,
+      "learning_rate": 6.800643086250121e-07,
+      "loss": 0.0,
+      "reward": 0.4594912249594927,
+      "reward_std": 0.34390021953731775,
+      "rewards/cosine_scaled_reward": -0.009837735444307327,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 234
+    },
+    {
+      "completion_length": 2293.812526702881,
+      "epoch": 0.26857142857142857,
+      "grad_norm": 0.03317144513130188,
+      "kl": 0.00013634003698825836,
+      "learning_rate": 6.770536555792944e-07,
+      "loss": 0.0,
+      "reward": 0.8775238930247724,
+      "reward_std": 0.8305205693468451,
+      "rewards/cosine_scaled_reward": 0.15751194162294269,
+      "rewards/format_reward": 0.5625000074505806,
+      "step": 235
+    },
+    {
+      "completion_length": 2535.791702270508,
+      "epoch": 0.26971428571428574,
+      "grad_norm": 0.022067207843065262,
+      "kl": 0.0001373887062072754,
+      "learning_rate": 6.740368101176495e-07,
+      "loss": 0.0,
+      "reward": 1.103066761046648,
+      "reward_std": 0.9327066205441952,
+      "rewards/cosine_scaled_reward": 0.23903337866067886,
+      "rewards/format_reward": 0.6250000111758709,
+      "step": 236
+    },
+    {
+      "completion_length": 2465.583335876465,
+      "epoch": 0.27085714285714285,
+      "grad_norm": 0.023853298276662827,
+      "kl": 0.00014454126358032227,
+      "learning_rate": 6.710139192768694e-07,
+      "loss": 0.0,
+      "reward": 0.7351281112059951,
+      "reward_std": 0.7459862437099218,
+      "rewards/cosine_scaled_reward": 0.12798071585712023,
+      "rewards/format_reward": 0.47916667349636555,
+      "step": 237
+    },
+    {
+      "completion_length": 2457.104217529297,
+      "epoch": 0.272,
+      "grad_norm": 0.022526247426867485,
+      "kl": 0.0001602470874786377,
+      "learning_rate": 6.679851303883891e-07,
+      "loss": 0.0,
+      "reward": 1.184366210596636,
+      "reward_std": 0.9232276640832424,
+      "rewards/cosine_scaled_reward": 0.2171830991283059,
+      "rewards/format_reward": 0.7500000111758709,
+      "step": 238
+    },
+    {
+      "completion_length": 1609.354206085205,
+      "epoch": 0.27314285714285713,
+      "grad_norm": 0.03424397110939026,
+      "kl": 8.738785982131958e-05,
+      "learning_rate": 6.649505910711058e-07,
+      "loss": 0.0,
+      "reward": 1.3734241500496864,
+      "reward_std": 0.5677254348993301,
+      "rewards/cosine_scaled_reward": 0.30129539873450994,
+      "rewards/format_reward": 0.7708333395421505,
+      "step": 239
+    },
+    {
+      "completion_length": 2939.1458740234375,
+      "epoch": 0.2742857142857143,
+      "grad_norm": 0.028905417770147324,
+      "kl": 0.00021439790725708008,
+      "learning_rate": 6.619104492241847e-07,
+      "loss": 0.0,
+      "reward": -0.0536014367826283,
+      "reward_std": 0.5374163910746574,
+      "rewards/cosine_scaled_reward": -0.21430072654038668,
+      "rewards/format_reward": 0.37500000558793545,
+      "step": 240
+    },
+    {
+      "completion_length": 2732.6250228881836,
+      "epoch": 0.2754285714285714,
+      "grad_norm": 0.02268056571483612,
+      "kl": 0.00020313262939453125,
+      "learning_rate": 6.588648530198504e-07,
+      "loss": 0.0,
+      "reward": 0.2373307328671217,
+      "reward_std": 0.628166763111949,
+      "rewards/cosine_scaled_reward": -0.13133463938720524,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 241
+    },
+    {
+      "completion_length": 1986.0000228881836,
+      "epoch": 0.2765714285714286,
+      "grad_norm": 0.03629198670387268,
+      "kl": 0.00025428086519241333,
+      "learning_rate": 6.558139508961654e-07,
+      "loss": 0.0,
+      "reward": 0.5092753138160333,
+      "reward_std": 0.674851905554533,
+      "rewards/cosine_scaled_reward": -0.047445693984627724,
+      "rewards/format_reward": 0.6041666828095913,
+      "step": 242
+    },
+    {
+      "completion_length": 2734.229217529297,
+      "epoch": 0.2777142857142857,
+      "grad_norm": 0.021753275766968727,
+      "kl": 0.00011421740055084229,
+      "learning_rate": 6.527578915497951e-07,
+      "loss": 0.0,
+      "reward": 0.9028994496911764,
+      "reward_std": 1.0264187045395374,
+      "rewards/cosine_scaled_reward": 0.2014497071504593,
+      "rewards/format_reward": 0.5000000111758709,
+      "step": 243
+    },
+    {
+      "completion_length": 2789.687545776367,
+      "epoch": 0.27885714285714286,
+      "grad_norm": 0.02406812459230423,
+      "kl": 0.00016814470291137695,
+      "learning_rate": 6.496968239287603e-07,
+      "loss": 0.0,
+      "reward": 0.5570749193429947,
+      "reward_std": 1.1161103919148445,
+      "rewards/cosine_scaled_reward": 0.05978744977619499,
+      "rewards/format_reward": 0.43750000931322575,
+      "step": 244
+    },
+    {
+      "completion_length": 2515.312568664551,
+      "epoch": 0.28,
+      "grad_norm": 0.029088586568832397,
+      "kl": 0.0001869797706604004,
+      "learning_rate": 6.466308972251785e-07,
+      "loss": 0.0,
+      "reward": 1.0382240321487188,
+      "reward_std": 1.1980111673474312,
+      "rewards/cosine_scaled_reward": 0.18577865794941317,
+      "rewards/format_reward": 0.6666666772216558,
+      "step": 245
+    },
+    {
+      "completion_length": 2277.875045776367,
+      "epoch": 0.28114285714285714,
+      "grad_norm": 0.02349865809082985,
+      "kl": 0.00016105175018310547,
+      "learning_rate": 6.435602608679916e-07,
+      "loss": 0.0,
+      "reward": 0.9948470462113619,
+      "reward_std": 0.784628352150321,
+      "rewards/cosine_scaled_reward": 0.13284017331898212,
+      "rewards/format_reward": 0.7291666679084301,
+      "step": 246
+    },
+    {
+      "completion_length": 2917.416717529297,
+      "epoch": 0.2822857142857143,
+      "grad_norm": 0.019329527392983437,
+      "kl": 0.0001506805419921875,
+      "learning_rate": 6.404850645156841e-07,
+      "loss": 0.0,
+      "reward": 0.3844575481489301,
+      "reward_std": 0.7838251367211342,
+      "rewards/cosine_scaled_reward": -0.005687899887561798,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 247
+    },
+    {
+      "completion_length": 1923.4375381469727,
+      "epoch": 0.2834285714285714,
+      "grad_norm": 0.034683942794799805,
+      "kl": 0.0001239478588104248,
+      "learning_rate": 6.374054580489873e-07,
+      "loss": 0.0,
+      "reward": 1.2442171084694564,
+      "reward_std": 0.8035479206591845,
+      "rewards/cosine_scaled_reward": 0.2887751990929246,
+      "rewards/format_reward": 0.6666666697710752,
+      "step": 248
+    },
+    {
+      "completion_length": 1708.2916946411133,
+      "epoch": 0.2845714285714286,
+      "grad_norm": 0.03058278001844883,
+      "kl": 0.00011420249938964844,
+      "learning_rate": 6.343215915635761e-07,
+      "loss": 0.0,
+      "reward": 0.7377499938011169,
+      "reward_std": 0.5643534660339355,
+      "rewards/cosine_scaled_reward": 0.02512497268617153,
+      "rewards/format_reward": 0.6875,
+      "step": 249
+    },
+    {
+      "completion_length": 2321.4167137145996,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 0.02951645292341709,
+      "kl": 0.00019127130508422852,
+      "learning_rate": 6.31233615362752e-07,
+      "loss": 0.0,
+      "reward": 0.5674178429762833,
+      "reward_std": 0.8846789188683033,
+      "rewards/cosine_scaled_reward": -0.04962442675605416,
+      "rewards/format_reward": 0.6666666772216558,
+      "step": 250
+    },
+    {
+      "completion_length": 1895.1875228881836,
+      "epoch": 0.28685714285714287,
+      "grad_norm": 0.02950862981379032,
+      "kl": 0.0001430809497833252,
+      "learning_rate": 6.281416799501187e-07,
+      "loss": 0.0,
+      "reward": 1.1478381529450417,
+      "reward_std": 0.9535709973424673,
+      "rewards/cosine_scaled_reward": 0.17808573693037033,
+      "rewards/format_reward": 0.7916666772216558,
+      "step": 251
+    },
+    {
+      "completion_length": 2560.3125534057617,
+      "epoch": 0.288,
+      "grad_norm": 0.026902787387371063,
+      "kl": 0.00019630789756774902,
+      "learning_rate": 6.25045936022246e-07,
+      "loss": 0.0,
+      "reward": 0.29892047587782145,
+      "reward_std": 0.697446895763278,
+      "rewards/cosine_scaled_reward": -0.09012311231344938,
+      "rewards/format_reward": 0.47916667349636555,
+      "step": 252
+    },
+    {
+      "completion_length": 2656.145854949951,
+      "epoch": 0.28914285714285715,
+      "grad_norm": 0.03214016929268837,
+      "kl": 0.00017839670181274414,
+      "learning_rate": 6.219465344613258e-07,
+      "loss": 0.0,
+      "reward": 0.3197309598326683,
+      "reward_std": 0.8267754539847374,
+      "rewards/cosine_scaled_reward": -0.11096786396956304,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 253
+    },
+    {
+      "completion_length": 2232.6666984558105,
+      "epoch": 0.29028571428571426,
+      "grad_norm": 0.03257149085402489,
+      "kl": 0.00014886260032653809,
+      "learning_rate": 6.188436263278172e-07,
+      "loss": 0.0,
+      "reward": 0.8183548748493195,
+      "reward_std": 0.742323987185955,
+      "rewards/cosine_scaled_reward": 0.09667741693556309,
+      "rewards/format_reward": 0.6250000037252903,
+      "step": 254
+    },
+    {
+      "completion_length": 3091.645854949951,
+      "epoch": 0.2914285714285714,
+      "grad_norm": 0.02308414690196514,
+      "kl": 0.00019991397857666016,
+      "learning_rate": 6.157373628530852e-07,
+      "loss": 0.0,
+      "reward": 0.48197503201663494,
+      "reward_std": 1.0647086016833782,
+      "rewards/cosine_scaled_reward": 0.03265417801867443,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 255
+    },
+    {
+      "completion_length": 2500.2917404174805,
+      "epoch": 0.2925714285714286,
+      "grad_norm": 0.025716979056596756,
+      "kl": 0.0002094060182571411,
+      "learning_rate": 6.126278954320294e-07,
+      "loss": 0.0,
+      "reward": 0.6078239146154374,
+      "reward_std": 0.9901378192007542,
+      "rewards/cosine_scaled_reward": 0.01224527694284916,
+      "rewards/format_reward": 0.5833333469927311,
+      "step": 256
+    },
+    {
+      "completion_length": 2928.791702270508,
+      "epoch": 0.2937142857142857,
+      "grad_norm": 0.020812608301639557,
+      "kl": 0.00019612908363342285,
+      "learning_rate": 6.095153756157051e-07,
+      "loss": 0.0,
+      "reward": 1.0758514516055584,
+      "reward_std": 0.8948536142706871,
+      "rewards/cosine_scaled_reward": 0.26709236670285463,
+      "rewards/format_reward": 0.5416666772216558,
+      "step": 257
+    },
+    {
+      "completion_length": 3086.6875915527344,
+      "epoch": 0.2948571428571429,
+      "grad_norm": 0.022219646722078323,
+      "kl": 0.0002516508102416992,
+      "learning_rate": 6.06399955103937e-07,
+      "loss": 0.0,
+      "reward": 0.5924380738288164,
+      "reward_std": 0.8064648397266865,
+      "rewards/cosine_scaled_reward": 0.035802360624074936,
+      "rewards/format_reward": 0.5208333414047956,
+      "step": 258
+    },
+    {
+      "completion_length": 2551.250026702881,
+      "epoch": 0.296,
+      "grad_norm": 0.03364958614110947,
+      "kl": 0.00019508600234985352,
+      "learning_rate": 6.032817857379256e-07,
+      "loss": 0.0,
+      "reward": 0.6504997052252293,
+      "reward_std": 0.7922412864863873,
+      "rewards/cosine_scaled_reward": 0.07524984562769532,
+      "rewards/format_reward": 0.5000000093132257,
+      "step": 259
+    },
+    {
+      "completion_length": 1881.8541831970215,
+      "epoch": 0.29714285714285715,
+      "grad_norm": 0.03499652445316315,
+      "kl": 0.00013124942779541016,
+      "learning_rate": 6.001610194928464e-07,
+      "loss": 0.0,
+      "reward": 1.0665137059986591,
+      "reward_std": 0.5399355934932828,
+      "rewards/cosine_scaled_reward": 0.231173490639776,
+      "rewards/format_reward": 0.6041666697710752,
+      "step": 260
+    },
+    {
+      "completion_length": 2637.4166870117188,
+      "epoch": 0.29828571428571427,
+      "grad_norm": 0.025031670928001404,
+      "kl": 0.00015434622764587402,
+      "learning_rate": 5.97037808470444e-07,
+      "loss": 0.0,
+      "reward": 0.3631602041423321,
+      "reward_std": 0.6417339015752077,
+      "rewards/cosine_scaled_reward": -0.047586577478796244,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 261
+    },
+    {
+      "completion_length": 3057.458366394043,
+      "epoch": 0.29942857142857143,
+      "grad_norm": 0.022654270753264427,
+      "kl": 0.00020933151245117188,
+      "learning_rate": 5.939123048916173e-07,
+      "loss": 0.0,
+      "reward": 0.11393738351762295,
+      "reward_std": 0.7356228828430176,
+      "rewards/cosine_scaled_reward": -0.10969797242432833,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 262
+    },
+    {
+      "completion_length": 2784.5000228881836,
+      "epoch": 0.30057142857142854,
+      "grad_norm": 0.027890196070075035,
+      "kl": 0.00014328956604003906,
+      "learning_rate": 5.907846610890011e-07,
+      "loss": 0.0,
+      "reward": 0.5555706415325403,
+      "reward_std": 0.6993023809045553,
+      "rewards/cosine_scaled_reward": 0.05903530679643154,
+      "rewards/format_reward": 0.4375000037252903,
+      "step": 263
+    },
+    {
+      "completion_length": 2711.916702270508,
+      "epoch": 0.3017142857142857,
+      "grad_norm": 0.026740290224552155,
+      "kl": 0.0001666545867919922,
+      "learning_rate": 5.87655029499542e-07,
+      "loss": 0.0,
+      "reward": 0.5835818080231547,
+      "reward_std": 0.9015247318893671,
+      "rewards/cosine_scaled_reward": 0.010540907853282988,
+      "rewards/format_reward": 0.5625000093132257,
+      "step": 264
+    },
+    {
+      "completion_length": 1977.6875305175781,
+      "epoch": 0.3028571428571429,
+      "grad_norm": 0.03000440075993538,
+      "kl": 0.00016647577285766602,
+      "learning_rate": 5.845235626570683e-07,
+      "loss": 0.0,
+      "reward": 1.0214002854190767,
+      "reward_std": 0.7950426787137985,
+      "rewards/cosine_scaled_reward": 0.16695012245327234,
+      "rewards/format_reward": 0.6875000018626451,
+      "step": 265
+    },
+    {
+      "completion_length": 3163.958354949951,
+      "epoch": 0.304,
+      "grad_norm": 0.01840229332447052,
+      "kl": 0.00019887089729309082,
+      "learning_rate": 5.813904131848564e-07,
+      "loss": 0.0,
+      "reward": -0.22563371248543262,
+      "reward_std": 0.4246583506464958,
+      "rewards/cosine_scaled_reward": -0.21698353067040443,
+      "rewards/format_reward": 0.2083333358168602,
+      "step": 266
+    },
+    {
+      "completion_length": 2904.750045776367,
+      "epoch": 0.30514285714285716,
+      "grad_norm": 0.024483945220708847,
+      "kl": 0.0001894235610961914,
+      "learning_rate": 5.78255733788191e-07,
+      "loss": 0.0,
+      "reward": 0.3056447245180607,
+      "reward_std": 0.7651229426264763,
+      "rewards/cosine_scaled_reward": -0.024260970763862133,
+      "rewards/format_reward": 0.3541666753590107,
+      "step": 267
+    },
+    {
+      "completion_length": 2707.229202270508,
+      "epoch": 0.3062857142857143,
+      "grad_norm": 0.033471375703811646,
+      "kl": 0.0002181529998779297,
+      "learning_rate": 5.751196772469237e-07,
+      "loss": 0.0,
+      "reward": 0.31767203425988555,
+      "reward_std": 1.0267981998622417,
+      "rewards/cosine_scaled_reward": -0.07033066404983401,
+      "rewards/format_reward": 0.4583333469927311,
+      "step": 268
+    },
+    {
+      "completion_length": 2601.145881652832,
+      "epoch": 0.30742857142857144,
+      "grad_norm": 0.023748284205794334,
+      "kl": 0.00017061829566955566,
+      "learning_rate": 5.71982396408026e-07,
+      "loss": 0.0,
+      "reward": 1.1178178992122412,
+      "reward_std": 0.6580545753240585,
+      "rewards/cosine_scaled_reward": 0.2568256063386798,
+      "rewards/format_reward": 0.6041666697710752,
+      "step": 269
+    },
+    {
+      "completion_length": 2286.145851135254,
+      "epoch": 0.30857142857142855,
+      "grad_norm": 0.026643814519047737,
+      "kl": 0.00014588236808776855,
+      "learning_rate": 5.688440441781398e-07,
+      "loss": 0.0,
+      "reward": 1.167864153161645,
+      "reward_std": 0.851657796651125,
+      "rewards/cosine_scaled_reward": 0.21934875007718801,
+      "rewards/format_reward": 0.7291666809469461,
+      "step": 270
+    },
+    {
+      "completion_length": 1784.645866394043,
+      "epoch": 0.3097142857142857,
+      "grad_norm": 0.030084380879998207,
+      "kl": 0.00011153519153594971,
+      "learning_rate": 5.657047735161255e-07,
+      "loss": 0.0,
+      "reward": 1.4007908403873444,
+      "reward_std": 0.4155320357531309,
+      "rewards/cosine_scaled_reward": 0.29414540342986584,
+      "rewards/format_reward": 0.8125,
+      "step": 271
+    },
+    {
+      "completion_length": 2623.0208587646484,
+      "epoch": 0.31085714285714283,
+      "grad_norm": 0.02625565230846405,
+      "kl": 0.00020056962966918945,
+      "learning_rate": 5.625647374256061e-07,
+      "loss": 0.0,
+      "reward": 0.664887816645205,
+      "reward_std": 1.0364688262343407,
+      "rewards/cosine_scaled_reward": 0.09286056295968592,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 272
+    },
+    {
+      "completion_length": 2523.1875534057617,
+      "epoch": 0.312,
+      "grad_norm": 0.02682247757911682,
+      "kl": 0.00015234388411045074,
+      "learning_rate": 5.594240889475106e-07,
+      "loss": 0.0,
+      "reward": 0.8108466335106641,
+      "reward_std": 0.6180602237582207,
+      "rewards/cosine_scaled_reward": 0.1762566501274705,
+      "rewards/format_reward": 0.45833334140479565,
+      "step": 273
+    },
+    {
+      "completion_length": 1638.1875190734863,
+      "epoch": 0.31314285714285717,
+      "grad_norm": 0.035664211958646774,
+      "kl": 0.0001033395528793335,
+      "learning_rate": 5.562829811526154e-07,
+      "loss": 0.0,
+      "reward": 1.2543680891394615,
+      "reward_std": 0.6733489837497473,
+      "rewards/cosine_scaled_reward": 0.21051735104992986,
+      "rewards/format_reward": 0.8333333432674408,
+      "step": 274
+    },
+    {
+      "completion_length": 2227.7500381469727,
+      "epoch": 0.3142857142857143,
+      "grad_norm": 0.024416198953986168,
+      "kl": 0.00014771521091461182,
+      "learning_rate": 5.531415671340826e-07,
+      "loss": 0.0,
+      "reward": 1.0997284352779388,
+      "reward_std": 0.6753085609525442,
+      "rewards/cosine_scaled_reward": 0.21653086133301258,
+      "rewards/format_reward": 0.6666666697710752,
+      "step": 275
+    },
+    {
+      "completion_length": 2410.375045776367,
+      "epoch": 0.31542857142857145,
+      "grad_norm": 0.03159765899181366,
+      "kl": 0.00018554925918579102,
+      "learning_rate": 5.5e-07,
+      "loss": 0.0,
+      "reward": 1.0730885528028011,
+      "reward_std": 0.8632807899266481,
+      "rewards/cosine_scaled_reward": 0.24487759917974472,
+      "rewards/format_reward": 0.583333333954215,
+      "step": 276
+    },
+    {
+      "completion_length": 2498.7708587646484,
+      "epoch": 0.31657142857142856,
+      "grad_norm": 0.024099338799715042,
+      "kl": 0.00020748376846313477,
+      "learning_rate": 5.468584328659172e-07,
+      "loss": 0.0,
+      "reward": 0.6968964114785194,
+      "reward_std": 0.8357117287814617,
+      "rewards/cosine_scaled_reward": 0.09844818594865501,
+      "rewards/format_reward": 0.5000000037252903,
+      "step": 277
+    },
+    {
+      "completion_length": 2121.0209426879883,
+      "epoch": 0.3177142857142857,
+      "grad_norm": 0.032011423259973526,
+      "kl": 0.00018835067749023438,
+      "learning_rate": 5.437170188473847e-07,
+      "loss": 0.0,
+      "reward": 1.1160824708640575,
+      "reward_std": 0.7977323867380619,
+      "rewards/cosine_scaled_reward": 0.17262454610317945,
+      "rewards/format_reward": 0.770833345130086,
+      "step": 278
+    },
+    {
+      "completion_length": 3179.7708435058594,
+      "epoch": 0.31885714285714284,
+      "grad_norm": 0.022884871810674667,
+      "kl": 0.0002695918083190918,
+      "learning_rate": 5.405759110524894e-07,
+      "loss": 0.0,
+      "reward": 0.020895134657621384,
+      "reward_std": 0.8185425326228142,
+      "rewards/cosine_scaled_reward": -0.09371909964829683,
+      "rewards/format_reward": 0.2083333358168602,
+      "step": 279
+    },
+    {
+      "completion_length": 2095.4583778381348,
+      "epoch": 0.32,
+      "grad_norm": 0.03684168681502342,
+      "kl": 0.0001882314682006836,
+      "learning_rate": 5.37435262574394e-07,
+      "loss": 0.0,
+      "reward": 0.6300602070987225,
+      "reward_std": 0.8308645784854889,
+      "rewards/cosine_scaled_reward": -0.007886571809649467,
+      "rewards/format_reward": 0.6458333395421505,
+      "step": 280
+    },
+    {
+      "completion_length": 3198.8958587646484,
+      "epoch": 0.3211428571428571,
+      "grad_norm": 0.021219369024038315,
+      "kl": 0.00019353628158569336,
+      "learning_rate": 5.342952264838747e-07,
+      "loss": 0.0,
+      "reward": -0.09903884382219985,
+      "reward_std": 0.7667136080563068,
+      "rewards/cosine_scaled_reward": -0.20576944231288508,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 281
+    },
+    {
+      "completion_length": 2406.458366394043,
+      "epoch": 0.3222857142857143,
+      "grad_norm": 0.02495969459414482,
+      "kl": 0.00018447637557983398,
+      "learning_rate": 5.311559558218603e-07,
+      "loss": 0.0,
+      "reward": 0.7009545154869556,
+      "reward_std": 0.7124308422207832,
+      "rewards/cosine_scaled_reward": 0.10047726146876812,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 282
+    },
+    {
+      "completion_length": 2349.270896911621,
+      "epoch": 0.32342857142857145,
+      "grad_norm": 0.025876298546791077,
+      "kl": 0.00015360116958618164,
+      "learning_rate": 5.28017603591974e-07,
+      "loss": 0.0,
+      "reward": 1.3722519651055336,
+      "reward_std": 0.653431911021471,
+      "rewards/cosine_scaled_reward": 0.34237595461308956,
+      "rewards/format_reward": 0.687500013038516,
+      "step": 283
+    },
+    {
+      "completion_length": 2050.7708587646484,
+      "epoch": 0.32457142857142857,
+      "grad_norm": 0.03173566982150078,
+      "kl": 0.00013002753257751465,
+      "learning_rate": 5.248803227530763e-07,
+      "loss": 0.0,
+      "reward": 0.6497855484485626,
+      "reward_std": 0.8422477524727583,
+      "rewards/cosine_scaled_reward": 0.012392764445394278,
+      "rewards/format_reward": 0.6250000111758709,
+      "step": 284
+    },
+    {
+      "completion_length": 1926.0208435058594,
+      "epoch": 0.32571428571428573,
+      "grad_norm": 0.02867991477251053,
+      "kl": 0.00016066431999206543,
+      "learning_rate": 5.21744266211809e-07,
+      "loss": 0.0,
+      "reward": 0.4057514422456734,
+      "reward_std": 0.3730186829343438,
+      "rewards/cosine_scaled_reward": -0.13045761734247208,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 285
+    },
+    {
+      "completion_length": 2342.687530517578,
+      "epoch": 0.32685714285714285,
+      "grad_norm": 0.02181842550635338,
+      "kl": 0.00020366907119750977,
+      "learning_rate": 5.186095868151436e-07,
+      "loss": 0.0,
+      "reward": 0.7377639599144459,
+      "reward_std": 0.7825022768229246,
+      "rewards/cosine_scaled_reward": 0.07721533998847008,
+      "rewards/format_reward": 0.5833333488553762,
+      "step": 286
+    },
+    {
+      "completion_length": 1700.1250076293945,
+      "epoch": 0.328,
+      "grad_norm": 0.03624531999230385,
+      "kl": 0.00015243887901306152,
+      "learning_rate": 5.154764373429315e-07,
+      "loss": 0.0,
+      "reward": 0.8391638435423374,
+      "reward_std": 0.34983737021684647,
+      "rewards/cosine_scaled_reward": 0.11749858036637306,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 287
+    },
+    {
+      "completion_length": 2821.4167098999023,
+      "epoch": 0.3291428571428571,
+      "grad_norm": 0.021799219772219658,
+      "kl": 0.0002630949020385742,
+      "learning_rate": 5.123449705004581e-07,
+      "loss": 0.0,
+      "reward": 0.45475615188479424,
+      "reward_std": 0.5843030996620655,
+      "rewards/cosine_scaled_reward": 0.01904474012553692,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 288
+    },
+    {
+      "completion_length": 2083.333351135254,
+      "epoch": 0.3302857142857143,
+      "grad_norm": 0.03328165039420128,
+      "kl": 0.00016808509826660156,
+      "learning_rate": 5.09215338910999e-07,
+      "loss": 0.0,
+      "reward": 0.7187244035303593,
+      "reward_std": 0.586519805714488,
+      "rewards/cosine_scaled_reward": 0.0676955422386527,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 289
+    },
+    {
+      "completion_length": 1441.3333778381348,
+      "epoch": 0.3314285714285714,
+      "grad_norm": 0.041413385421037674,
+      "kl": 0.0001710951328277588,
+      "learning_rate": 5.060876951083828e-07,
+      "loss": 0.0,
+      "reward": 1.352583286818117,
+      "reward_std": 0.8790040239691734,
+      "rewards/cosine_scaled_reward": 0.21795828046742827,
+      "rewards/format_reward": 0.9166666716337204,
+      "step": 290
+    },
+    {
+      "completion_length": 2162.9167098999023,
+      "epoch": 0.3325714285714286,
+      "grad_norm": 0.034360699355602264,
+      "kl": 0.00018029659986495972,
+      "learning_rate": 5.02962191529556e-07,
+      "loss": 0.0,
+      "reward": 1.1365532241761684,
+      "reward_std": 0.8855068720877171,
+      "rewards/cosine_scaled_reward": 0.22452658973634243,
+      "rewards/format_reward": 0.6875000055879354,
+      "step": 291
+    },
+    {
+      "completion_length": 2644.9583587646484,
+      "epoch": 0.33371428571428574,
+      "grad_norm": 0.036109767854213715,
+      "kl": 0.0002117753028869629,
+      "learning_rate": 4.998389805071536e-07,
+      "loss": 0.0,
+      "reward": 0.4815543070435524,
+      "reward_std": 0.9521236941218376,
+      "rewards/cosine_scaled_reward": -0.01963951502693817,
+      "rewards/format_reward": 0.5208333488553762,
+      "step": 292
+    },
+    {
+      "completion_length": 2065.250030517578,
+      "epoch": 0.33485714285714285,
+      "grad_norm": 0.02345646359026432,
+      "kl": 0.00020043551921844482,
+      "learning_rate": 4.967182142620745e-07,
+      "loss": 0.0,
+      "reward": 0.7920062579214573,
+      "reward_std": 0.5047565586864948,
+      "rewards/cosine_scaled_reward": 0.06266978848725557,
+      "rewards/format_reward": 0.6666666716337204,
+      "step": 293
+    },
+    {
+      "completion_length": 3125.937545776367,
+      "epoch": 0.336,
+      "grad_norm": 0.021944746375083923,
+      "kl": 0.00028936564922332764,
+      "learning_rate": 4.93600044896063e-07,
+      "loss": 0.0,
+      "reward": 0.09450298827141523,
+      "reward_std": 0.7673522084951401,
+      "rewards/cosine_scaled_reward": -0.11941516539081931,
+      "rewards/format_reward": 0.33333334140479565,
+      "step": 294
+    },
+    {
+      "completion_length": 2895.0416870117188,
+      "epoch": 0.33714285714285713,
+      "grad_norm": 0.02375158853828907,
+      "kl": 0.00023639202117919922,
+      "learning_rate": 4.904846243842949e-07,
+      "loss": 0.0,
+      "reward": 0.26469022780656815,
+      "reward_std": 0.7072394695132971,
+      "rewards/cosine_scaled_reward": -0.03432156052440405,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 295
+    },
+    {
+      "completion_length": 2651.1250610351562,
+      "epoch": 0.3382857142857143,
+      "grad_norm": 0.035691626369953156,
+      "kl": 0.00022995471954345703,
+      "learning_rate": 4.873721045679706e-07,
+      "loss": 0.0,
+      "reward": 0.3209421820938587,
+      "reward_std": 0.5486197881400585,
+      "rewards/cosine_scaled_reward": -0.08952891454100609,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 296
+    },
+    {
+      "completion_length": 3260.8958740234375,
+      "epoch": 0.3394285714285714,
+      "grad_norm": 0.01949848048388958,
+      "kl": 0.0002536773681640625,
+      "learning_rate": 4.842626371469149e-07,
+      "loss": 0.0,
+      "reward": 0.2345700664445758,
+      "reward_std": 0.6523050926625729,
+      "rewards/cosine_scaled_reward": -0.0077149709686636925,
+      "rewards/format_reward": 0.2500000037252903,
+      "step": 297
+    },
+    {
+      "completion_length": 2739.8333740234375,
+      "epoch": 0.3405714285714286,
+      "grad_norm": 0.02277456782758236,
+      "kl": 0.0001857280731201172,
+      "learning_rate": 4.811563736721829e-07,
+      "loss": 0.0,
+      "reward": 0.43556486954912543,
+      "reward_std": 0.6449983278289437,
+      "rewards/cosine_scaled_reward": 0.009449097327888012,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 298
+    },
+    {
+      "completion_length": 2909.4791870117188,
+      "epoch": 0.3417142857142857,
+      "grad_norm": 0.019031217321753502,
+      "kl": 0.00021719932556152344,
+      "learning_rate": 4.780534655386743e-07,
+      "loss": 0.0,
+      "reward": 0.6691161692142487,
+      "reward_std": 0.7011800315231085,
+      "rewards/cosine_scaled_reward": 0.11580806598067284,
+      "rewards/format_reward": 0.4375000074505806,
+      "step": 299
+    },
+    {
+      "completion_length": 3504.541717529297,
+      "epoch": 0.34285714285714286,
+      "grad_norm": 0.02002292312681675,
+      "kl": 0.0002887248992919922,
+      "learning_rate": 4.749540639777539e-07,
+      "loss": 0.0,
+      "reward": -0.11457789549604058,
+      "reward_std": 0.6568729132413864,
+      "rewards/cosine_scaled_reward": -0.21353895403444767,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 300
+    },
+    {
+      "completion_length": 2172.7291870117188,
+      "epoch": 0.344,
+      "grad_norm": 0.029103700071573257,
+      "kl": 0.00020676851272583008,
+      "learning_rate": 4.7185832004988133e-07,
+      "loss": 0.0,
+      "reward": 0.6181785631924868,
+      "reward_std": 0.7141751162707806,
+      "rewards/cosine_scaled_reward": -0.045077390037477016,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 301
+    },
+    {
+      "completion_length": 2186.4792251586914,
+      "epoch": 0.34514285714285714,
+      "grad_norm": 0.03206675127148628,
+      "kl": 0.00018966197967529297,
+      "learning_rate": 4.68766384637248e-07,
+      "loss": 0.0,
+      "reward": 0.9613751322031021,
+      "reward_std": 0.7258251551538706,
+      "rewards/cosine_scaled_reward": 0.14735423121601343,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 302
+    },
+    {
+      "completion_length": 2475.937545776367,
+      "epoch": 0.3462857142857143,
+      "grad_norm": 0.030688228085637093,
+      "kl": 0.00026735663414001465,
+      "learning_rate": 4.656784084364238e-07,
+      "loss": 0.0,
+      "reward": 0.7674907688051462,
+      "reward_std": 0.7924170382320881,
+      "rewards/cosine_scaled_reward": 0.11291203834116459,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 303
+    },
+    {
+      "completion_length": 2449.791717529297,
+      "epoch": 0.3474285714285714,
+      "grad_norm": 0.024618245661258698,
+      "kl": 0.00020569562911987305,
+      "learning_rate": 4.6259454195101267e-07,
+      "loss": 0.0,
+      "reward": 0.6075898297131062,
+      "reward_std": 0.6916676126420498,
+      "rewards/cosine_scaled_reward": 0.032961574383080006,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 304
+    },
+    {
+      "completion_length": 2968.2916870117188,
+      "epoch": 0.3485714285714286,
+      "grad_norm": 0.02341613359749317,
+      "kl": 0.00027740001678466797,
+      "learning_rate": 4.59514935484316e-07,
+      "loss": 0.0,
+      "reward": 0.38752281852066517,
+      "reward_std": 0.653439100831747,
+      "rewards/cosine_scaled_reward": -0.024988610297441483,
+      "rewards/format_reward": 0.43750001303851604,
+      "step": 305
+    },
+    {
+      "completion_length": 2223.770851135254,
+      "epoch": 0.3497142857142857,
+      "grad_norm": 0.03363551199436188,
+      "kl": 0.000197485089302063,
+      "learning_rate": 4.5643973913200837e-07,
+      "loss": 0.0,
+      "reward": 0.603302130009979,
+      "reward_std": 0.6513868616893888,
+      "rewards/cosine_scaled_reward": 0.009984390810132027,
+      "rewards/format_reward": 0.5833333358168602,
+      "step": 306
+    },
+    {
+      "completion_length": 2278.1250495910645,
+      "epoch": 0.35085714285714287,
+      "grad_norm": 0.02858709916472435,
+      "kl": 0.00022980570793151855,
+      "learning_rate": 4.5336910277482155e-07,
+      "loss": 0.0,
+      "reward": 0.6608727481216192,
+      "reward_std": 0.9192832596600056,
+      "rewards/cosine_scaled_reward": 0.05960302893072367,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 307
+    },
+    {
+      "completion_length": 3100.4583587646484,
+      "epoch": 0.352,
+      "grad_norm": 0.020364860072731972,
+      "kl": 0.0002155900001525879,
+      "learning_rate": 4.503031760712397e-07,
+      "loss": 0.0,
+      "reward": 0.0357023049145937,
+      "reward_std": 0.8593032034114003,
+      "rewards/cosine_scaled_reward": -0.10714884800836444,
+      "rewards/format_reward": 0.2500000037252903,
+      "step": 308
+    },
+    {
+      "completion_length": 2716.1666946411133,
+      "epoch": 0.35314285714285715,
+      "grad_norm": 0.021565675735473633,
+      "kl": 0.00018143653869628906,
+      "learning_rate": 4.4724210845020494e-07,
+      "loss": 0.0,
+      "reward": 0.9010052662342787,
+      "reward_std": 0.7911521699279547,
+      "rewards/cosine_scaled_reward": 0.19008595822378993,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 309
+    },
+    {
+      "completion_length": 2119.1667098999023,
+      "epoch": 0.35428571428571426,
+      "grad_norm": 0.03625396639108658,
+      "kl": 0.00021964311599731445,
+      "learning_rate": 4.441860491038345e-07,
+      "loss": 0.0,
+      "reward": 0.7360105197876692,
+      "reward_std": 0.6620105188339949,
+      "rewards/cosine_scaled_reward": 0.0555052665488347,
+      "rewards/format_reward": 0.6250000055879354,
+      "step": 310
+    },
+    {
+      "completion_length": 2410.833354949951,
+      "epoch": 0.3554285714285714,
+      "grad_norm": 0.02665293589234352,
+      "kl": 0.00021332502365112305,
+      "learning_rate": 4.4113514698014953e-07,
+      "loss": 0.0,
+      "reward": 0.9606177415698767,
+      "reward_std": 0.5332410894334316,
+      "rewards/cosine_scaled_reward": 0.20947551727294922,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 311
+    },
+    {
+      "completion_length": 2079.7500228881836,
+      "epoch": 0.3565714285714286,
+      "grad_norm": 0.039660558104515076,
+      "kl": 0.00025030970573425293,
+      "learning_rate": 4.3808955077581546e-07,
+      "loss": 0.0,
+      "reward": 0.9072759561240673,
+      "reward_std": 0.7400492213200778,
+      "rewards/cosine_scaled_reward": 0.18280465085990727,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 312
+    },
+    {
+      "completion_length": 2780.166778564453,
+      "epoch": 0.3577142857142857,
+      "grad_norm": 0.024134520441293716,
+      "kl": 0.0002815723419189453,
+      "learning_rate": 4.350494089288943e-07,
+      "loss": 0.0,
+      "reward": 0.7847088079433888,
+      "reward_std": 0.7853051656857133,
+      "rewards/cosine_scaled_reward": 0.15277107199653983,
+      "rewards/format_reward": 0.4791666753590107,
+      "step": 313
+    },
+    {
+      "completion_length": 2324.958354949951,
+      "epoch": 0.3588571428571429,
+      "grad_norm": 0.03832564502954483,
+      "kl": 0.00017490237951278687,
+      "learning_rate": 4.3201486961161093e-07,
+      "loss": 0.0,
+      "reward": 0.5692981313914061,
+      "reward_std": 0.5818128418177366,
+      "rewards/cosine_scaled_reward": 0.03464904520660639,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 314
+    },
+    {
+      "completion_length": 2754.1875534057617,
+      "epoch": 0.36,
+      "grad_norm": 0.02288965694606304,
+      "kl": 0.0002474188804626465,
+      "learning_rate": 4.2898608072313045e-07,
+      "loss": 0.0,
+      "reward": 0.4873616211116314,
+      "reward_std": 0.7167357690632343,
+      "rewards/cosine_scaled_reward": 0.03534747939556837,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 315
+    },
+    {
+      "completion_length": 3277.7291870117188,
+      "epoch": 0.36114285714285715,
+      "grad_norm": 0.021539807319641113,
+      "kl": 0.0003350973129272461,
+      "learning_rate": 4.2596318988235037e-07,
+      "loss": 0.0,
+      "reward": 0.06327132880687714,
+      "reward_std": 0.722619965672493,
+      "rewards/cosine_scaled_reward": -0.09336433373391628,
+      "rewards/format_reward": 0.25000000931322575,
+      "step": 316
+    },
+    {
+      "completion_length": 2702.958351135254,
+      "epoch": 0.36228571428571427,
+      "grad_norm": 0.02301914617419243,
+      "kl": 0.00023549795150756836,
+      "learning_rate": 4.2294634442070553e-07,
+      "loss": 0.0,
+      "reward": 0.5955107100307941,
+      "reward_std": 0.7853559423238039,
+      "rewards/cosine_scaled_reward": 0.06858869083225727,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 317
+    },
+    {
+      "completion_length": 1185.270866394043,
+      "epoch": 0.36342857142857143,
+      "grad_norm": 0.041411470621824265,
+      "kl": 0.00015148520469665527,
+      "learning_rate": 4.1993569137498776e-07,
+      "loss": 0.0,
+      "reward": 1.0035362336784601,
+      "reward_std": 0.6542131118476391,
+      "rewards/cosine_scaled_reward": 0.0642680861055851,
+      "rewards/format_reward": 0.8750000055879354,
+      "step": 318
+    },
+    {
+      "completion_length": 2636.291690826416,
+      "epoch": 0.36457142857142855,
+      "grad_norm": 0.027852993458509445,
+      "kl": 0.00036531686782836914,
+      "learning_rate": 4.1693137748017915e-07,
+      "loss": 0.0,
+      "reward": -0.07456635776907206,
+      "reward_std": 0.6103027909994125,
+      "rewards/cosine_scaled_reward": -0.23519985377788544,
+      "rewards/format_reward": 0.39583333767950535,
+      "step": 319
+    },
+    {
+      "completion_length": 1750.3958740234375,
+      "epoch": 0.3657142857142857,
+      "grad_norm": 0.040690820664167404,
+      "kl": 0.00023043155670166016,
+      "learning_rate": 4.1393354916230005e-07,
+      "loss": 0.0,
+      "reward": 0.6239464022219181,
+      "reward_std": 0.7162524946033955,
+      "rewards/cosine_scaled_reward": -0.04219348356127739,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 320
+    },
+    {
+      "completion_length": 1433.7083702087402,
+      "epoch": 0.3668571428571429,
+      "grad_norm": 0.03097819723188877,
+      "kl": 0.00015997886657714844,
+      "learning_rate": 4.1094235253127374e-07,
+      "loss": 0.0,
+      "reward": 1.5650277510285378,
+      "reward_std": 0.6967961974442005,
+      "rewards/cosine_scaled_reward": 0.35543054959271103,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 321
+    },
+    {
+      "completion_length": 2536.6250343322754,
+      "epoch": 0.368,
+      "grad_norm": 0.035577192902565,
+      "kl": 0.0003066062927246094,
+      "learning_rate": 4.079579333738039e-07,
+      "loss": 0.0,
+      "reward": 0.3695586025714874,
+      "reward_std": 0.85175746306777,
+      "rewards/cosine_scaled_reward": -0.05480404058471322,
+      "rewards/format_reward": 0.47916666977107525,
+      "step": 322
+    },
+    {
+      "completion_length": 2828.7083587646484,
+      "epoch": 0.36914285714285716,
+      "grad_norm": 0.022153332829475403,
+      "kl": 0.0002874135971069336,
+      "learning_rate": 4.0498043714627006e-07,
+      "loss": 0.0,
+      "reward": 0.35160677476233104,
+      "reward_std": 0.49565985053777695,
+      "rewards/cosine_scaled_reward": -0.03252994408831,
+      "rewards/format_reward": 0.41666666977107525,
+      "step": 323
+    },
+    {
+      "completion_length": 2622.7084045410156,
+      "epoch": 0.3702857142857143,
+      "grad_norm": 0.025416357442736626,
+      "kl": 0.00023853778839111328,
+      "learning_rate": 4.020100089676376e-07,
+      "loss": 0.0,
+      "reward": 0.6543685221113265,
+      "reward_std": 0.733675017952919,
+      "rewards/cosine_scaled_reward": 0.01468425802886486,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 324
+    },
+    {
+      "completion_length": 2451.312568664551,
+      "epoch": 0.37142857142857144,
+      "grad_norm": 0.030104102566838264,
+      "kl": 0.00023043155670166016,
+      "learning_rate": 3.9904679361238526e-07,
+      "loss": 0.0,
+      "reward": 1.3238378688693047,
+      "reward_std": 0.8181280642747879,
+      "rewards/cosine_scaled_reward": 0.3077522525563836,
+      "rewards/format_reward": 0.7083333469927311,
+      "step": 325
+    },
+    {
+      "completion_length": 2106.645866394043,
+      "epoch": 0.37257142857142855,
+      "grad_norm": 0.027758557349443436,
+      "kl": 0.0002227574586868286,
+      "learning_rate": 3.9609093550344907e-07,
+      "loss": 0.0,
+      "reward": 0.5596632058732212,
+      "reward_std": 0.41956991609185934,
+      "rewards/cosine_scaled_reward": -0.01183508150279522,
+      "rewards/format_reward": 0.583333333954215,
+      "step": 326
+    },
+    {
+      "completion_length": 2574.145851135254,
+      "epoch": 0.3737142857142857,
+      "grad_norm": 0.022373398765921593,
+      "kl": 0.0002663135528564453,
+      "learning_rate": 3.931425787051832e-07,
+      "loss": 0.0,
+      "reward": 0.6893210001289845,
+      "reward_std": 0.5786230899393559,
+      "rewards/cosine_scaled_reward": 0.09466049098409712,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 327
+    },
+    {
+      "completion_length": 3245.9791870117188,
+      "epoch": 0.37485714285714283,
+      "grad_norm": 0.020722128450870514,
+      "kl": 0.0002732276916503906,
+      "learning_rate": 3.902018669163384e-07,
+      "loss": 0.0,
+      "reward": -0.09898044681176543,
+      "reward_std": 0.7348660603165627,
+      "rewards/cosine_scaled_reward": -0.16407354921102524,
+      "rewards/format_reward": 0.2291666716337204,
+      "step": 328
+    },
+    {
+      "completion_length": 1781.6875381469727,
+      "epoch": 0.376,
+      "grad_norm": 0.03358970955014229,
+      "kl": 0.00023666024208068848,
+      "learning_rate": 3.872689434630585e-07,
+      "loss": 0.0,
+      "reward": 0.9239476323127747,
+      "reward_std": 0.6006343699991703,
+      "rewards/cosine_scaled_reward": 0.12864043563604355,
+      "rewards/format_reward": 0.6666666679084301,
+      "step": 329
+    },
+    {
+      "completion_length": 2251.4166870117188,
+      "epoch": 0.37714285714285717,
+      "grad_norm": 0.03271754831075668,
+      "kl": 0.00033169984817504883,
+      "learning_rate": 3.843439512918949e-07,
+      "loss": 0.0,
+      "reward": 0.4894188493490219,
+      "reward_std": 0.6065305564552546,
+      "rewards/cosine_scaled_reward": -0.026123913936316967,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 330
+    },
+    {
+      "completion_length": 2500.625026702881,
+      "epoch": 0.3782857142857143,
+      "grad_norm": 0.046767521649599075,
+      "kl": 0.0003387928009033203,
+      "learning_rate": 3.8142703296283953e-07,
+      "loss": 0.0,
+      "reward": 0.19016977865248919,
+      "reward_std": 0.5361207164824009,
+      "rewards/cosine_scaled_reward": -0.09241510927677155,
+      "rewards/format_reward": 0.3750000111758709,
+      "step": 331
+    },
+    {
+      "completion_length": 2354.2917251586914,
+      "epoch": 0.37942857142857145,
+      "grad_norm": 0.025751523673534393,
+      "kl": 0.00024452805519104004,
+      "learning_rate": 3.785183306423767e-07,
+      "loss": 0.0,
+      "reward": 0.5813850574195385,
+      "reward_std": 0.7453097756952047,
+      "rewards/cosine_scaled_reward": -0.03222414234187454,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 332
+    },
+    {
+      "completion_length": 1749.708366394043,
+      "epoch": 0.38057142857142856,
+      "grad_norm": 0.030186345800757408,
+      "kl": 0.00021500885486602783,
+      "learning_rate": 3.7561798609655373e-07,
+      "loss": 0.0,
+      "reward": 0.9438134930096567,
+      "reward_std": 0.7434254898689687,
+      "rewards/cosine_scaled_reward": 0.08649008721113205,
+      "rewards/format_reward": 0.7708333488553762,
+      "step": 333
+    },
+    {
+      "completion_length": 3011.7916870117188,
+      "epoch": 0.38171428571428573,
+      "grad_norm": 0.020333390682935715,
+      "kl": 0.0002646446228027344,
+      "learning_rate": 3.72726140684072e-07,
+      "loss": 0.0,
+      "reward": 0.013087651343084872,
+      "reward_std": 0.8183173164725304,
+      "rewards/cosine_scaled_reward": -0.19137283554300666,
+      "rewards/format_reward": 0.3958333432674408,
+      "step": 334
+    },
+    {
+      "completion_length": 2112.37508392334,
+      "epoch": 0.38285714285714284,
+      "grad_norm": 0.025975586846470833,
+      "kl": 0.00021225214004516602,
+      "learning_rate": 3.6984293534939737e-07,
+      "loss": 0.0,
+      "reward": 1.0843083523213863,
+      "reward_std": 0.8515229849144816,
+      "rewards/cosine_scaled_reward": 0.19840414822101593,
+      "rewards/format_reward": 0.6875000037252903,
+      "step": 335
+    },
+    {
+      "completion_length": 2667.9583892822266,
+      "epoch": 0.384,
+      "grad_norm": 0.029666097834706306,
+      "kl": 0.0002618134021759033,
+      "learning_rate": 3.6696851061588994e-07,
+      "loss": 0.0,
+      "reward": 0.6127568809315562,
+      "reward_std": 0.935376051813364,
+      "rewards/cosine_scaled_reward": 0.045961756026372313,
+      "rewards/format_reward": 0.5208333469927311,
+      "step": 336
+    },
+    {
+      "completion_length": 2586.4583587646484,
+      "epoch": 0.3851428571428571,
+      "grad_norm": 0.03280950337648392,
+      "kl": 0.00030684471130371094,
+      "learning_rate": 3.641030065789562e-07,
+      "loss": 0.0,
+      "reward": 0.4611587089020759,
+      "reward_std": 0.941901870071888,
+      "rewards/cosine_scaled_reward": -0.029837319627404213,
+      "rewards/format_reward": 0.5208333488553762,
+      "step": 337
+    },
+    {
+      "completion_length": 1920.6250457763672,
+      "epoch": 0.3862857142857143,
+      "grad_norm": 0.03358282893896103,
+      "kl": 0.00031685829162597656,
+      "learning_rate": 3.612465628992203e-07,
+      "loss": 0.0,
+      "reward": 1.1313491351902485,
+      "reward_std": 1.0058028288185596,
+      "rewards/cosine_scaled_reward": 0.2115078903734684,
+      "rewards/format_reward": 0.7083333507180214,
+      "step": 338
+    },
+    {
+      "completion_length": 2708.6042098999023,
+      "epoch": 0.38742857142857146,
+      "grad_norm": 0.023695282638072968,
+      "kl": 0.00026488304138183594,
+      "learning_rate": 3.5839931879571725e-07,
+      "loss": 0.0,
+      "reward": 0.38326615560799837,
+      "reward_std": 0.6079130750149488,
+      "rewards/cosine_scaled_reward": -0.0375335980206728,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 339
+    },
+    {
+      "completion_length": 2371.625030517578,
+      "epoch": 0.38857142857142857,
+      "grad_norm": 0.025804683566093445,
+      "kl": 0.00025916099548339844,
+      "learning_rate": 3.555614130391079e-07,
+      "loss": 0.0,
+      "reward": 0.6570014208555222,
+      "reward_std": 0.9428398255258799,
+      "rewards/cosine_scaled_reward": 0.026417370419949293,
+      "rewards/format_reward": 0.6041666734963655,
+      "step": 340
+    },
+    {
+      "completion_length": 2184.333381652832,
+      "epoch": 0.38971428571428574,
+      "grad_norm": 0.018583592027425766,
+      "kl": 0.00025847554206848145,
+      "learning_rate": 3.5273298394491515e-07,
+      "loss": 0.0,
+      "reward": 1.163216508924961,
+      "reward_std": 0.36735112965106964,
+      "rewards/cosine_scaled_reward": 0.21702490840107203,
+      "rewards/format_reward": 0.7291666716337204,
+      "step": 341
+    },
+    {
+      "completion_length": 2734.500045776367,
+      "epoch": 0.39085714285714285,
+      "grad_norm": 0.030387967824935913,
+      "kl": 0.00031244754791259766,
+      "learning_rate": 3.4991416936678276e-07,
+      "loss": 0.0,
+      "reward": 0.5118438927456737,
+      "reward_std": 0.9676601774990559,
+      "rewards/cosine_scaled_reward": 0.037171950563788414,
+      "rewards/format_reward": 0.43750000186264515,
+      "step": 342
+    },
+    {
+      "completion_length": 2897.3959045410156,
+      "epoch": 0.392,
+      "grad_norm": 0.018177591264247894,
+      "kl": 0.00023752450942993164,
+      "learning_rate": 3.471051066897562e-07,
+      "loss": 0.0,
+      "reward": 0.8089198600500822,
+      "reward_std": 1.0615319572389126,
+      "rewards/cosine_scaled_reward": 0.13362658489495516,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 343
+    },
+    {
+      "completion_length": 2030.3333549499512,
+      "epoch": 0.3931428571428571,
+      "grad_norm": 0.02531350590288639,
+      "kl": 0.00022709369659423828,
+      "learning_rate": 3.4430593282358777e-07,
+      "loss": 0.0,
+      "reward": 1.0889792470261455,
+      "reward_std": 0.7364504262804985,
+      "rewards/cosine_scaled_reward": 0.20073960922309197,
+      "rewards/format_reward": 0.6875000018626451,
+      "step": 344
+    },
+    {
+      "completion_length": 2663.0833587646484,
+      "epoch": 0.3942857142857143,
+      "grad_norm": 0.0251361895352602,
+      "kl": 0.0002938807010650635,
+      "learning_rate": 3.4151678419606233e-07,
+      "loss": 0.0,
+      "reward": 0.43384309159591794,
+      "reward_std": 0.49294498562812805,
+      "rewards/cosine_scaled_reward": -0.012245113030076027,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 345
+    },
+    {
+      "completion_length": 3057.0625610351562,
+      "epoch": 0.3954285714285714,
+      "grad_norm": 0.01870022341609001,
+      "kl": 0.00028312206268310547,
+      "learning_rate": 3.387377967463493e-07,
+      "loss": 0.0,
+      "reward": 0.3562997840344906,
+      "reward_std": 0.6700815781950951,
+      "rewards/cosine_scaled_reward": -0.071850111708045,
+      "rewards/format_reward": 0.5000000018626451,
+      "step": 346
+    },
+    {
+      "completion_length": 2926.7708740234375,
+      "epoch": 0.3965714285714286,
+      "grad_norm": 0.019071895629167557,
+      "kl": 0.0002764463424682617,
+      "learning_rate": 3.359691059183761e-07,
+      "loss": 0.0,
+      "reward": 0.21276836469769478,
+      "reward_std": 0.5730974823236465,
+      "rewards/cosine_scaled_reward": -0.12278249301016331,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 347
+    },
+    {
+      "completion_length": 2656.3958435058594,
+      "epoch": 0.3977142857142857,
+      "grad_norm": 0.03188029304146767,
+      "kl": 0.0002663731575012207,
+      "learning_rate": 3.3321084665422803e-07,
+      "loss": 0.0,
+      "reward": 0.4368863687850535,
+      "reward_std": 0.4383883699774742,
+      "rewards/cosine_scaled_reward": -0.02114013209939003,
+      "rewards/format_reward": 0.4791666679084301,
+      "step": 348
+    },
+    {
+      "completion_length": 3118.750030517578,
+      "epoch": 0.39885714285714285,
+      "grad_norm": 0.020537259057164192,
+      "kl": 0.0003848075866699219,
+      "learning_rate": 3.3046315338757026e-07,
+      "loss": 0.0,
+      "reward": 0.15319344215095043,
+      "reward_std": 0.5101428851485252,
+      "rewards/cosine_scaled_reward": -0.07965327124111354,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 349
+    },
+    {
+      "completion_length": 2212.3750381469727,
+      "epoch": 0.4,
+      "grad_norm": 0.03013945370912552,
+      "kl": 0.0002713203430175781,
+      "learning_rate": 3.2772616003709616e-07,
+      "loss": 0.0,
+      "reward": 0.9151125885546207,
+      "reward_std": 0.895346611738205,
+      "rewards/cosine_scaled_reward": 0.14505626424215734,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 350
+    },
+    {
+      "completion_length": 2579.4375381469727,
+      "epoch": 0.40114285714285713,
+      "grad_norm": 0.021040596067905426,
+      "kl": 0.0002969503402709961,
+      "learning_rate": 3.250000000000001e-07,
+      "loss": 0.0,
+      "reward": 0.5091639803722501,
+      "reward_std": 0.6855394951999187,
+      "rewards/cosine_scaled_reward": 0.004581989720463753,
+      "rewards/format_reward": 0.5000000018626451,
+      "step": 351
+    },
+    {
+      "completion_length": 2121.1458740234375,
+      "epoch": 0.4022857142857143,
+      "grad_norm": 0.02947075478732586,
+      "kl": 0.0002119988203048706,
+      "learning_rate": 3.222848061454764e-07,
+      "loss": 0.0,
+      "reward": 0.7395412735641003,
+      "reward_std": 0.7547396682202816,
+      "rewards/cosine_scaled_reward": 0.046853944193571806,
+      "rewards/format_reward": 0.6458333432674408,
+      "step": 352
+    },
+    {
+      "completion_length": 2014.5833435058594,
+      "epoch": 0.4034285714285714,
+      "grad_norm": 0.03271789103746414,
+      "kl": 0.0002213120460510254,
+      "learning_rate": 3.195807108082429e-07,
+      "loss": 0.0,
+      "reward": 1.2261908203363419,
+      "reward_std": 0.7751676780171692,
+      "rewards/cosine_scaled_reward": 0.25892873853445053,
+      "rewards/format_reward": 0.7083333414047956,
+      "step": 353
+    },
+    {
+      "completion_length": 1930.916706085205,
+      "epoch": 0.4045714285714286,
+      "grad_norm": 0.03464234620332718,
+      "kl": 0.00023859739303588867,
+      "learning_rate": 3.168878457820915e-07,
+      "loss": 0.0,
+      "reward": 0.788333578966558,
+      "reward_std": 0.7727246461436152,
+      "rewards/cosine_scaled_reward": 0.060833441093564034,
+      "rewards/format_reward": 0.6666666734963655,
+      "step": 354
+    },
+    {
+      "completion_length": 2235.229217529297,
+      "epoch": 0.4057142857142857,
+      "grad_norm": 0.02619633823633194,
+      "kl": 0.0002925395965576172,
+      "learning_rate": 3.142063423134644e-07,
+      "loss": 0.0,
+      "reward": 0.9775698743760586,
+      "reward_std": 1.000068573281169,
+      "rewards/cosine_scaled_reward": 0.18670159205794334,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 355
+    },
+    {
+      "completion_length": 2525.604232788086,
+      "epoch": 0.40685714285714286,
+      "grad_norm": 0.022776400670409203,
+      "kl": 0.00028067827224731445,
+      "learning_rate": 3.115363310950578e-07,
+      "loss": 0.0,
+      "reward": 1.0638019368052483,
+      "reward_std": 0.8996341824531555,
+      "rewards/cosine_scaled_reward": 0.17773428186774254,
+      "rewards/format_reward": 0.7083333358168602,
+      "step": 356
+    },
+    {
+      "completion_length": 2803.0208854675293,
+      "epoch": 0.408,
+      "grad_norm": 0.02899840474128723,
+      "kl": 0.0002684593200683594,
+      "learning_rate": 3.0887794225945143e-07,
+      "loss": 0.0,
+      "reward": 0.3746182285249233,
+      "reward_std": 0.6245617978274822,
+      "rewards/cosine_scaled_reward": -0.03144089970737696,
+      "rewards/format_reward": 0.4375000074505806,
+      "step": 357
+    },
+    {
+      "completion_length": 2429.312530517578,
+      "epoch": 0.40914285714285714,
+      "grad_norm": 0.022451486438512802,
+      "kl": 0.00027573108673095703,
+      "learning_rate": 3.062313053727671e-07,
+      "loss": 0.0,
+      "reward": 1.352016813121736,
+      "reward_std": 0.6039175763726234,
+      "rewards/cosine_scaled_reward": 0.31142502650618553,
+      "rewards/format_reward": 0.7291666679084301,
+      "step": 358
+    },
+    {
+      "completion_length": 2191.812515258789,
+      "epoch": 0.4102857142857143,
+      "grad_norm": 0.03193458169698715,
+      "kl": 0.00021660327911376953,
+      "learning_rate": 3.0359654942835247e-07,
+      "loss": 0.0,
+      "reward": 0.7233314402401447,
+      "reward_std": 0.7219348400831223,
+      "rewards/cosine_scaled_reward": 0.059582391288131475,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 359
+    },
+    {
+      "completion_length": 2400.979217529297,
+      "epoch": 0.4114285714285714,
+      "grad_norm": 0.02467941865324974,
+      "kl": 0.00025135278701782227,
+      "learning_rate": 3.0097380284049523e-07,
+      "loss": 0.0,
+      "reward": 0.9745171531103551,
+      "reward_std": 0.8171700052917004,
+      "rewards/cosine_scaled_reward": 0.15392523724585772,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 360
+    },
+    {
+      "completion_length": 2900.041732788086,
+      "epoch": 0.4125714285714286,
+      "grad_norm": 0.02299378626048565,
+      "kl": 0.0003606081008911133,
+      "learning_rate": 2.9836319343816397e-07,
+      "loss": 0.0,
+      "reward": 0.8722579134628177,
+      "reward_std": 0.942744567990303,
+      "rewards/cosine_scaled_reward": 0.17571225319989026,
+      "rewards/format_reward": 0.520833345130086,
+      "step": 361
+    },
+    {
+      "completion_length": 1323.479190826416,
+      "epoch": 0.4137142857142857,
+      "grad_norm": 0.03528214618563652,
+      "kl": 0.000218123197555542,
+      "learning_rate": 2.9576484845877793e-07,
+      "loss": 0.0,
+      "reward": 1.2730410918593407,
+      "reward_std": 0.7479732874780893,
+      "rewards/cosine_scaled_reward": 0.19902052055113018,
+      "rewards/format_reward": 0.8750000074505806,
+      "step": 362
+    },
+    {
+      "completion_length": 1805.2916793823242,
+      "epoch": 0.41485714285714287,
+      "grad_norm": 0.029074430465698242,
+      "kl": 0.00025790929794311523,
+      "learning_rate": 2.931788945420058e-07,
+      "loss": 0.0,
+      "reward": 1.1047739461064339,
+      "reward_std": 0.6205479633063078,
+      "rewards/cosine_scaled_reward": 0.1982202921062708,
+      "rewards/format_reward": 0.7083333432674408,
+      "step": 363
+    },
+    {
+      "completion_length": 2740.7083587646484,
+      "epoch": 0.416,
+      "grad_norm": 0.021566243842244148,
+      "kl": 0.000292360782623291,
+      "learning_rate": 2.9060545772359305e-07,
+      "loss": 0.0,
+      "reward": 0.36955657228827477,
+      "reward_std": 0.7109892182052135,
+      "rewards/cosine_scaled_reward": -0.05480505700688809,
+      "rewards/format_reward": 0.479166679084301,
+      "step": 364
+    },
+    {
+      "completion_length": 2874.7500534057617,
+      "epoch": 0.41714285714285715,
+      "grad_norm": 0.023213258013129234,
+      "kl": 0.00028192996978759766,
+      "learning_rate": 2.8804466342921987e-07,
+      "loss": 0.0,
+      "reward": 0.25813389755785465,
+      "reward_std": 0.7393531780689955,
+      "rewards/cosine_scaled_reward": -0.048016379587352276,
+      "rewards/format_reward": 0.3541666828095913,
+      "step": 365
+    },
+    {
+      "completion_length": 1737.541675567627,
+      "epoch": 0.41828571428571426,
+      "grad_norm": 0.03312158212065697,
+      "kl": 0.00028526782989501953,
+      "learning_rate": 2.854966364683872e-07,
+      "loss": 0.0,
+      "reward": 1.2915470600128174,
+      "reward_std": 0.7252576723694801,
+      "rewards/cosine_scaled_reward": 0.27077352383639663,
+      "rewards/format_reward": 0.75,
+      "step": 366
+    },
+    {
+      "completion_length": 2430.625045776367,
+      "epoch": 0.41942857142857143,
+      "grad_norm": 0.023319371044635773,
+      "kl": 0.00021529197692871094,
+      "learning_rate": 2.829615010283344e-07,
+      "loss": 0.0,
+      "reward": 0.817657120525837,
+      "reward_std": 0.5943502280861139,
+      "rewards/cosine_scaled_reward": 0.09632855094969273,
+      "rewards/format_reward": 0.6250000037252903,
+      "step": 367
+    },
+    {
+      "completion_length": 3034.270835876465,
+      "epoch": 0.4205714285714286,
+      "grad_norm": 0.026619471609592438,
+      "kl": 0.0003840923309326172,
+      "learning_rate": 2.8043938066798645e-07,
+      "loss": 0.0,
+      "reward": 0.38300488237291574,
+      "reward_std": 0.5576745513826609,
+      "rewards/cosine_scaled_reward": 0.024835781194269657,
+      "rewards/format_reward": 0.3333333395421505,
+      "step": 368
+    },
+    {
+      "completion_length": 2265.8334045410156,
+      "epoch": 0.4217142857142857,
+      "grad_norm": 0.02840518392622471,
+      "kl": 0.00026166439056396484,
+      "learning_rate": 2.7793039831193133e-07,
+      "loss": 0.0,
+      "reward": 0.9877472408115864,
+      "reward_std": 0.911319050937891,
+      "rewards/cosine_scaled_reward": 0.18137363530695438,
+      "rewards/format_reward": 0.6250000111758709,
+      "step": 369
+    },
+    {
+      "completion_length": 3168.2916717529297,
+      "epoch": 0.4228571428571429,
+      "grad_norm": 0.02979353815317154,
+      "kl": 0.00041425228118896484,
+      "learning_rate": 2.7543467624442956e-07,
+      "loss": 0.0,
+      "reward": 0.3131989259272814,
+      "reward_std": 0.9070627521723509,
+      "rewards/cosine_scaled_reward": 0.03159945458173752,
+      "rewards/format_reward": 0.2500000037252903,
+      "step": 370
+    },
+    {
+      "completion_length": 1747.312515258789,
+      "epoch": 0.424,
+      "grad_norm": 0.04161190241575241,
+      "kl": 0.00025528669357299805,
+      "learning_rate": 2.729523361034538e-07,
+      "loss": 0.0,
+      "reward": 0.7982922215014696,
+      "reward_std": 0.31508788373321295,
+      "rewards/cosine_scaled_reward": 0.0762293990701437,
+      "rewards/format_reward": 0.645833333954215,
+      "step": 371
+    },
+    {
+      "completion_length": 2756.2708587646484,
+      "epoch": 0.42514285714285716,
+      "grad_norm": 0.02483726479113102,
+      "kl": 0.0003197193145751953,
+      "learning_rate": 2.7048349887476037e-07,
+      "loss": 0.0,
+      "reward": 1.0121797006577253,
+      "reward_std": 0.9594811573624611,
+      "rewards/cosine_scaled_reward": 0.2248398419469595,
+      "rewards/format_reward": 0.5625000093132257,
+      "step": 372
+    },
+    {
+      "completion_length": 1751.770866394043,
+      "epoch": 0.42628571428571427,
+      "grad_norm": 0.03236696869134903,
+      "kl": 0.00019246339797973633,
+      "learning_rate": 2.6802828488599294e-07,
+      "loss": 0.0,
+      "reward": 0.8297730721533298,
+      "reward_std": 0.8522606380283833,
+      "rewards/cosine_scaled_reward": 0.039886531041702256,
+      "rewards/format_reward": 0.7500000037252903,
+      "step": 373
+    },
+    {
+      "completion_length": 1826.958351135254,
+      "epoch": 0.42742857142857144,
+      "grad_norm": 0.038021981716156006,
+      "kl": 0.0002339780330657959,
+      "learning_rate": 2.655868138008171e-07,
+      "loss": 0.0,
+      "reward": 0.8999335905537009,
+      "reward_std": 0.7978312708437443,
+      "rewards/cosine_scaled_reward": 0.08538344036787748,
+      "rewards/format_reward": 0.729166679084301,
+      "step": 374
+    },
+    {
+      "completion_length": 2728.250015258789,
+      "epoch": 0.42857142857142855,
+      "grad_norm": 0.025368480011820793,
+      "kl": 0.0003457069396972656,
+      "learning_rate": 2.631592046130896e-07,
+      "loss": 0.0,
+      "reward": 0.6862165480852127,
+      "reward_std": 0.7109800316393375,
+      "rewards/cosine_scaled_reward": 0.12435825169086456,
+      "rewards/format_reward": 0.4375000074505806,
+      "step": 375
+    },
+    {
+      "completion_length": 2175.3333587646484,
+      "epoch": 0.4297142857142857,
+      "grad_norm": 0.0324525311589241,
+      "kl": 0.0003424808382987976,
+      "learning_rate": 2.6074557564105724e-07,
+      "loss": 0.0,
+      "reward": 0.5297784507274628,
+      "reward_std": 0.4516606852412224,
+      "rewards/cosine_scaled_reward": -0.05802745930850506,
+      "rewards/format_reward": 0.645833333954215,
+      "step": 376
+    },
+    {
+      "completion_length": 3404.8750915527344,
+      "epoch": 0.4308571428571429,
+      "grad_norm": 0.019506795331835747,
+      "kl": 0.00038254261016845703,
+      "learning_rate": 2.583460445215911e-07,
+      "loss": 0.0,
+      "reward": 0.21527918288484216,
+      "reward_std": 0.9994356147944927,
+      "rewards/cosine_scaled_reward": -0.05902708321809769,
+      "rewards/format_reward": 0.33333333767950535,
+      "step": 377
+    },
+    {
+      "completion_length": 1961.4167003631592,
+      "epoch": 0.432,
+      "grad_norm": 0.039375122636556625,
+      "kl": 0.00024241209030151367,
+      "learning_rate": 2.5596072820445254e-07,
+      "loss": 0.0,
+      "reward": 0.893386272713542,
+      "reward_std": 0.8861826229840517,
+      "rewards/cosine_scaled_reward": 0.10294315160717815,
+      "rewards/format_reward": 0.687500013038516,
+      "step": 378
+    },
+    {
+      "completion_length": 3086.8542098999023,
+      "epoch": 0.43314285714285716,
+      "grad_norm": 0.022253919392824173,
+      "kl": 0.0003380775451660156,
+      "learning_rate": 2.5358974294659373e-07,
+      "loss": 0.0,
+      "reward": 0.2040771245956421,
+      "reward_std": 0.7313600815832615,
+      "rewards/cosine_scaled_reward": -0.033378101885318756,
+      "rewards/format_reward": 0.27083333767950535,
+      "step": 379
+    },
+    {
+      "completion_length": 2130.1666870117188,
+      "epoch": 0.4342857142857143,
+      "grad_norm": 0.031271472573280334,
+      "kl": 0.0002847909927368164,
+      "learning_rate": 2.512332043064913e-07,
+      "loss": 0.0,
+      "reward": 1.0673715379089117,
+      "reward_std": 0.6758305430412292,
+      "rewards/cosine_scaled_reward": 0.16910243220627308,
+      "rewards/format_reward": 0.7291666772216558,
+      "step": 380
+    },
+    {
+      "completion_length": 2840.5209197998047,
+      "epoch": 0.43542857142857144,
+      "grad_norm": 0.024770023301243782,
+      "kl": 0.00039768218994140625,
+      "learning_rate": 2.488912271385139e-07,
+      "loss": 0.0,
+      "reward": 0.8948189206421375,
+      "reward_std": 1.0691867098212242,
+      "rewards/cosine_scaled_reward": 0.15574277006089687,
+      "rewards/format_reward": 0.5833333507180214,
+      "step": 381
+    },
+    {
+      "completion_length": 1638.2917022705078,
+      "epoch": 0.43657142857142855,
+      "grad_norm": 0.028202086687088013,
+      "kl": 0.00019669532775878906,
+      "learning_rate": 2.465639255873246e-07,
+      "loss": 0.0,
+      "reward": 0.4687234200537205,
+      "reward_std": 0.6315413638949394,
+      "rewards/cosine_scaled_reward": -0.1718882955610752,
+      "rewards/format_reward": 0.8125000074505806,
+      "step": 382
+    },
+    {
+      "completion_length": 2525.875030517578,
+      "epoch": 0.4377142857142857,
+      "grad_norm": 0.028156043961644173,
+      "kl": 0.0003873109817504883,
+      "learning_rate": 2.4425141308231765e-07,
+      "loss": 0.0,
+      "reward": 0.6626773066818714,
+      "reward_std": 0.6682372465729713,
+      "rewards/cosine_scaled_reward": 0.06050530215725303,
+      "rewards/format_reward": 0.5416666679084301,
+      "step": 383
+    },
+    {
+      "completion_length": 2035.6875305175781,
+      "epoch": 0.43885714285714283,
+      "grad_norm": 0.0351465679705143,
+      "kl": 0.0002747178077697754,
+      "learning_rate": 2.4195380233209006e-07,
+      "loss": 0.0,
+      "reward": 1.3732114508748055,
+      "reward_std": 1.1746384501457214,
+      "rewards/cosine_scaled_reward": 0.3428557086735964,
+      "rewards/format_reward": 0.6875000149011612,
+      "step": 384
+    },
+    {
+      "completion_length": 2768.8541870117188,
+      "epoch": 0.44,
+      "grad_norm": 0.020549314096570015,
+      "kl": 0.00025400519371032715,
+      "learning_rate": 2.3967120531894857e-07,
+      "loss": 0.0,
+      "reward": 0.41839180141687393,
+      "reward_std": 0.7576583102345467,
+      "rewards/cosine_scaled_reward": -0.06163745652884245,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 385
+    },
+    {
+      "completion_length": 2741.3541870117188,
+      "epoch": 0.44114285714285717,
+      "grad_norm": 0.022283930331468582,
+      "kl": 0.00032389163970947266,
+      "learning_rate": 2.374037332934512e-07,
+      "loss": 0.0,
+      "reward": 0.35035596787929535,
+      "reward_std": 0.6659238189458847,
+      "rewards/cosine_scaled_reward": -0.06440535286674276,
+      "rewards/format_reward": 0.47916667722165585,
+      "step": 386
+    },
+    {
+      "completion_length": 3046.9791984558105,
+      "epoch": 0.4422857142857143,
+      "grad_norm": 0.031019089743494987,
+      "kl": 0.0004036426544189453,
+      "learning_rate": 2.3515149676898552e-07,
+      "loss": 0.0,
+      "reward": 0.1448732865974307,
+      "reward_std": 0.7876984663307667,
+      "rewards/cosine_scaled_reward": -0.0421467050909996,
+      "rewards/format_reward": 0.2291666753590107,
+      "step": 387
+    },
+    {
+      "completion_length": 2511.062530517578,
+      "epoch": 0.44342857142857145,
+      "grad_norm": 0.025159146636724472,
+      "kl": 0.0003024935722351074,
+      "learning_rate": 2.3291460551638237e-07,
+      "loss": 0.0,
+      "reward": 0.6130173578858376,
+      "reward_std": 0.545312637463212,
+      "rewards/cosine_scaled_reward": 0.06692530773580074,
+      "rewards/format_reward": 0.47916667722165585,
+      "step": 388
+    },
+    {
+      "completion_length": 2439.0833854675293,
+      "epoch": 0.44457142857142856,
+      "grad_norm": 0.028031574562191963,
+      "kl": 0.00030285120010375977,
+      "learning_rate": 2.306931685585657e-07,
+      "loss": 0.0,
+      "reward": 0.5443721693009138,
+      "reward_std": 0.6922687292098999,
+      "rewards/cosine_scaled_reward": -0.01948060654103756,
+      "rewards/format_reward": 0.5833333507180214,
+      "step": 389
+    },
+    {
+      "completion_length": 1954.3542175292969,
+      "epoch": 0.44571428571428573,
+      "grad_norm": 0.030413255095481873,
+      "kl": 0.00020575523376464844,
+      "learning_rate": 2.2848729416523859e-07,
+      "loss": 0.0,
+      "reward": 1.0101970732212067,
+      "reward_std": 0.8248934336006641,
+      "rewards/cosine_scaled_reward": 0.14051519706845284,
+      "rewards/format_reward": 0.7291666716337204,
+      "step": 390
+    },
+    {
+      "completion_length": 2316.9583892822266,
+      "epoch": 0.44685714285714284,
+      "grad_norm": 0.035128720104694366,
+      "kl": 0.00028714537620544434,
+      "learning_rate": 2.2629708984760706e-07,
+      "loss": 0.0,
+      "reward": 1.2428898513317108,
+      "reward_std": 0.6731934603303671,
+      "rewards/cosine_scaled_reward": 0.2881115823984146,
+      "rewards/format_reward": 0.6666666679084301,
+      "step": 391
+    },
+    {
+      "completion_length": 1697.4583587646484,
+      "epoch": 0.448,
+      "grad_norm": 0.03315872699022293,
+      "kl": 0.00025475025177001953,
+      "learning_rate": 2.2412266235313973e-07,
+      "loss": 0.0,
+      "reward": 0.8411284685134888,
+      "reward_std": 0.5785612929612398,
+      "rewards/cosine_scaled_reward": 0.04556424228940159,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 392
+    },
+    {
+      "completion_length": 2015.7917022705078,
+      "epoch": 0.4491428571428571,
+      "grad_norm": 0.027014190331101418,
+      "kl": 0.00029033422470092773,
+      "learning_rate": 2.2196411766036487e-07,
+      "loss": 0.0,
+      "reward": 1.0128965936601162,
+      "reward_std": 0.9798723421990871,
+      "rewards/cosine_scaled_reward": 0.13144830497913063,
+      "rewards/format_reward": 0.7500000018626451,
+      "step": 393
+    },
+    {
+      "completion_length": 2925.187545776367,
+      "epoch": 0.4502857142857143,
+      "grad_norm": 0.02542806603014469,
+      "kl": 0.0003414154052734375,
+      "learning_rate": 2.1982156097370557e-07,
+      "loss": 0.0,
+      "reward": 0.06015077605843544,
+      "reward_std": 0.6904054302722216,
+      "rewards/cosine_scaled_reward": -0.13659127056598663,
+      "rewards/format_reward": 0.33333333767950535,
+      "step": 394
+    },
+    {
+      "completion_length": 2059.0833740234375,
+      "epoch": 0.4514285714285714,
+      "grad_norm": 0.055620644241571426,
+      "kl": 0.0002554953098297119,
+      "learning_rate": 2.1769509671835223e-07,
+      "loss": 0.0,
+      "reward": 0.24230369459837675,
+      "reward_std": 0.5080111399292946,
+      "rewards/cosine_scaled_reward": -0.139264827943407,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 395
+    },
+    {
+      "completion_length": 2738.229217529297,
+      "epoch": 0.45257142857142857,
+      "grad_norm": 0.02174789272248745,
+      "kl": 0.0003293752670288086,
+      "learning_rate": 2.1558482853517253e-07,
+      "loss": 0.0,
+      "reward": 0.5809464631602168,
+      "reward_std": 0.8570992723107338,
+      "rewards/cosine_scaled_reward": 0.03005655948072672,
+      "rewards/format_reward": 0.5208333432674408,
+      "step": 396
+    },
+    {
+      "completion_length": 2119.9375534057617,
+      "epoch": 0.45371428571428574,
+      "grad_norm": 0.02603338658809662,
+      "kl": 0.000241011381149292,
+      "learning_rate": 2.134908592756607e-07,
+      "loss": 0.0,
+      "reward": 0.9264453984797001,
+      "reward_std": 0.5849390588700771,
+      "rewards/cosine_scaled_reward": 0.0882226787507534,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 397
+    },
+    {
+      "completion_length": 2522.7709045410156,
+      "epoch": 0.45485714285714285,
+      "grad_norm": 0.023683685809373856,
+      "kl": 0.000274658203125,
+      "learning_rate": 2.1141329099692406e-07,
+      "loss": 0.0,
+      "reward": 0.8225542418658733,
+      "reward_std": 1.0746662057936192,
+      "rewards/cosine_scaled_reward": 0.07794378884136677,
+      "rewards/format_reward": 0.6666666734963655,
+      "step": 398
+    },
+    {
+      "completion_length": 1766.2083740234375,
+      "epoch": 0.456,
+      "grad_norm": 0.025745665654540062,
+      "kl": 0.000264585018157959,
+      "learning_rate": 2.0935222495670968e-07,
+      "loss": 0.0,
+      "reward": 1.4033365920186043,
+      "reward_std": 0.6987381167709827,
+      "rewards/cosine_scaled_reward": 0.26416830345988274,
+      "rewards/format_reward": 0.875,
+      "step": 399
+    },
+    {
+      "completion_length": 1470.5000305175781,
+      "epoch": 0.45714285714285713,
+      "grad_norm": 0.030597828328609467,
+      "kl": 0.00021582841873168945,
+      "learning_rate": 2.0730776160846853e-07,
+      "loss": 0.0,
+      "reward": 1.6081257872283459,
+      "reward_std": 0.7085444070398808,
+      "rewards/cosine_scaled_reward": 0.33531289361417294,
+      "rewards/format_reward": 0.9375000074505806,
+      "step": 400
+    },
+    {
+      "completion_length": 3003.7708587646484,
+      "epoch": 0.4582857142857143,
+      "grad_norm": 0.023032892495393753,
+      "kl": 0.0003560781478881836,
+      "learning_rate": 2.0528000059645995e-07,
+      "loss": 0.0,
+      "reward": 0.5061473976820707,
+      "reward_std": 0.665686160326004,
+      "rewards/cosine_scaled_reward": 0.08640703570563346,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 401
+    },
+    {
+      "completion_length": 2160.0416946411133,
+      "epoch": 0.4594285714285714,
+      "grad_norm": 0.03311454504728317,
+      "kl": 0.00029790401458740234,
+      "learning_rate": 2.032690407508949e-07,
+      "loss": 0.0,
+      "reward": 0.4731251299381256,
+      "reward_std": 0.45016609132289886,
+      "rewards/cosine_scaled_reward": -0.03427077550441027,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 402
+    },
+    {
+      "completion_length": 1579.2292251586914,
+      "epoch": 0.4605714285714286,
+      "grad_norm": 0.040304120630025864,
+      "kl": 0.00027740001678466797,
+      "learning_rate": 2.0127498008311922e-07,
+      "loss": 0.0,
+      "reward": 1.0009474894031882,
+      "reward_std": 0.58954899571836,
+      "rewards/cosine_scaled_reward": 0.08380707260221243,
+      "rewards/format_reward": 0.8333333414047956,
+      "step": 403
+    },
+    {
+      "completion_length": 2295.062557220459,
+      "epoch": 0.4617142857142857,
+      "grad_norm": 0.03594672679901123,
+      "kl": 0.0002923309803009033,
+      "learning_rate": 1.9929791578083655e-07,
+      "loss": 0.0,
+      "reward": 0.6277561672031879,
+      "reward_std": 0.576149944216013,
+      "rewards/cosine_scaled_reward": 0.04304474964737892,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 404
+    },
+    {
+      "completion_length": 2160.333339691162,
+      "epoch": 0.46285714285714286,
+      "grad_norm": 0.044510871171951294,
+      "kl": 0.00038023293018341064,
+      "learning_rate": 1.9733794420337213e-07,
+      "loss": 0.0,
+      "reward": 0.7880059815943241,
+      "reward_std": 0.9082814259454608,
+      "rewards/cosine_scaled_reward": 0.10233633872121572,
+      "rewards/format_reward": 0.5833333395421505,
+      "step": 405
+    },
+    {
+      "completion_length": 2105.3125228881836,
+      "epoch": 0.464,
+      "grad_norm": 0.026859665289521217,
+      "kl": 0.0003541707992553711,
+      "learning_rate": 1.9539516087697517e-07,
+      "loss": 0.0,
+      "reward": 1.1059295609593391,
+      "reward_std": 0.7627066448330879,
+      "rewards/cosine_scaled_reward": 0.16754809115082026,
+      "rewards/format_reward": 0.7708333432674408,
+      "step": 406
+    },
+    {
+      "completion_length": 2212.0625381469727,
+      "epoch": 0.46514285714285714,
+      "grad_norm": 0.02859460562467575,
+      "kl": 0.00026991963386535645,
+      "learning_rate": 1.934696604901642e-07,
+      "loss": 0.0,
+      "reward": 0.7443156484514475,
+      "reward_std": 0.6628005839884281,
+      "rewards/cosine_scaled_reward": 0.08049114793539047,
+      "rewards/format_reward": 0.5833333395421505,
+      "step": 407
+    },
+    {
+      "completion_length": 2292.583381652832,
+      "epoch": 0.4662857142857143,
+      "grad_norm": 0.02938149869441986,
+      "kl": 0.0002721548080444336,
+      "learning_rate": 1.915615368891117e-07,
+      "loss": 0.0,
+      "reward": 0.84524005651474,
+      "reward_std": 0.6734911154489964,
+      "rewards/cosine_scaled_reward": 0.08928670734167099,
+      "rewards/format_reward": 0.666666679084301,
+      "step": 408
+    },
+    {
+      "completion_length": 3335.9583740234375,
+      "epoch": 0.4674285714285714,
+      "grad_norm": 0.0185756403952837,
+      "kl": 0.0003679990768432617,
+      "learning_rate": 1.8967088307307e-07,
+      "loss": 0.0,
+      "reward": 0.31832231767475605,
+      "reward_std": 0.9492303058505058,
+      "rewards/cosine_scaled_reward": 0.002911144867539406,
+      "rewards/format_reward": 0.31250000558793545,
+      "step": 409
+    },
+    {
+      "completion_length": 2268.0833740234375,
+      "epoch": 0.4685714285714286,
+      "grad_norm": 0.03002990409731865,
+      "kl": 0.000295102596282959,
+      "learning_rate": 1.8779779118983867e-07,
+      "loss": 0.0,
+      "reward": 0.7843060363084078,
+      "reward_std": 1.074190242215991,
+      "rewards/cosine_scaled_reward": 0.07965302001684904,
+      "rewards/format_reward": 0.6250000093132257,
+      "step": 410
+    },
+    {
+      "completion_length": 2607.520881652832,
+      "epoch": 0.4697142857142857,
+      "grad_norm": 0.04299235716462135,
+      "kl": 0.0003064870834350586,
+      "learning_rate": 1.8594235253127372e-07,
+      "loss": 0.0,
+      "reward": 0.3397108046337962,
+      "reward_std": 0.7905894294381142,
+      "rewards/cosine_scaled_reward": -0.04889458604156971,
+      "rewards/format_reward": 0.4375000111758709,
+      "step": 411
+    },
+    {
+      "completion_length": 2604.625030517578,
+      "epoch": 0.47085714285714286,
+      "grad_norm": 0.02590368315577507,
+      "kl": 0.00034099817276000977,
+      "learning_rate": 1.8410465752883758e-07,
+      "loss": 0.0,
+      "reward": 1.047096872702241,
+      "reward_std": 0.7374865831807256,
+      "rewards/cosine_scaled_reward": 0.2631317935883999,
+      "rewards/format_reward": 0.520833333954215,
+      "step": 412
+    },
+    {
+      "completion_length": 2449.8750381469727,
+      "epoch": 0.472,
+      "grad_norm": 0.023459814488887787,
+      "kl": 0.0002518892288208008,
+      "learning_rate": 1.822847957491922e-07,
+      "loss": 0.0,
+      "reward": 0.7442583926022053,
+      "reward_std": 0.9269864112138748,
+      "rewards/cosine_scaled_reward": 0.09087921027094126,
+      "rewards/format_reward": 0.5625000111758709,
+      "step": 413
+    },
+    {
+      "completion_length": 2950.354217529297,
+      "epoch": 0.47314285714285714,
+      "grad_norm": 0.01904076710343361,
+      "kl": 0.0002581775188446045,
+      "learning_rate": 1.804828558898332e-07,
+      "loss": 0.0,
+      "reward": 0.674991624429822,
+      "reward_std": 0.8730818629264832,
+      "rewards/cosine_scaled_reward": 0.06666246941313148,
+      "rewards/format_reward": 0.541666679084301,
+      "step": 414
+    },
+    {
+      "completion_length": 3076.6666870117188,
+      "epoch": 0.4742857142857143,
+      "grad_norm": 0.023224493488669395,
+      "kl": 0.0003394484519958496,
+      "learning_rate": 1.7869892577476722e-07,
+      "loss": 0.0,
+      "reward": 0.21714868023991585,
+      "reward_std": 0.7628773404285312,
+      "rewards/cosine_scaled_reward": -0.05809233826585114,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 415
+    },
+    {
+      "completion_length": 1542.4583740234375,
+      "epoch": 0.4754285714285714,
+      "grad_norm": 0.030471056699752808,
+      "kl": 0.00021645426750183105,
+      "learning_rate": 1.7693309235023127e-07,
+      "loss": 0.0,
+      "reward": 0.9979265034198761,
+      "reward_std": 0.8310877997428179,
+      "rewards/cosine_scaled_reward": 0.0927132535725832,
+      "rewards/format_reward": 0.8125000149011612,
+      "step": 416
+    },
+    {
+      "completion_length": 3084.7083587646484,
+      "epoch": 0.4765714285714286,
+      "grad_norm": 0.021396063268184662,
+      "kl": 0.00035440921783447266,
+      "learning_rate": 1.7518544168045524e-07,
+      "loss": 0.0,
+      "reward": 0.03550974000245333,
+      "reward_std": 0.7396349869668484,
+      "rewards/cosine_scaled_reward": -0.12807846441864967,
+      "rewards/format_reward": 0.29166667722165585,
+      "step": 417
+    },
+    {
+      "completion_length": 2215.229232788086,
+      "epoch": 0.4777142857142857,
+      "grad_norm": 0.033279899507761,
+      "kl": 0.00040906667709350586,
+      "learning_rate": 1.7345605894346726e-07,
+      "loss": 0.0,
+      "reward": 0.7383007109165192,
+      "reward_std": 0.3981512626633048,
+      "rewards/cosine_scaled_reward": 0.07748369872570038,
+      "rewards/format_reward": 0.5833333358168602,
+      "step": 418
+    },
+    {
+      "completion_length": 2424.5625381469727,
+      "epoch": 0.47885714285714287,
+      "grad_norm": 0.027742447331547737,
+      "kl": 0.00024384260177612305,
+      "learning_rate": 1.7174502842694212e-07,
+      "loss": 0.0,
+      "reward": 0.6631737016141415,
+      "reward_std": 0.6519222892820835,
+      "rewards/cosine_scaled_reward": 0.07117018103599548,
+      "rewards/format_reward": 0.520833333954215,
+      "step": 419
+    },
+    {
+      "completion_length": 1580.56254196167,
+      "epoch": 0.48,
+      "grad_norm": 0.04106001928448677,
+      "kl": 0.00024759769439697266,
+      "learning_rate": 1.7005243352409333e-07,
+      "loss": 0.0,
+      "reward": 0.5669936803169549,
+      "reward_std": 0.6721935961395502,
+      "rewards/cosine_scaled_reward": -0.11233649682253599,
+      "rewards/format_reward": 0.7916666679084301,
+      "step": 420
+    },
+    {
+      "completion_length": 2925.1250228881836,
+      "epoch": 0.48114285714285715,
+      "grad_norm": 0.027175476774573326,
+      "kl": 0.0003466606140136719,
+      "learning_rate": 1.6837835672960831e-07,
+      "loss": 0.0,
+      "reward": 0.4191740155220032,
+      "reward_std": 0.6842759978026152,
+      "rewards/cosine_scaled_reward": 0.0012536547146737576,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 421
+    },
+    {
+      "completion_length": 2770.1458740234375,
+      "epoch": 0.48228571428571426,
+      "grad_norm": 0.0218949094414711,
+      "kl": 0.00034099817276000977,
+      "learning_rate": 1.6672287963562852e-07,
+      "loss": 0.0,
+      "reward": 0.4831668883562088,
+      "reward_std": 0.7019177824258804,
+      "rewards/cosine_scaled_reward": 0.012416768004186451,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 422
+    },
+    {
+      "completion_length": 2963.6667098999023,
+      "epoch": 0.48342857142857143,
+      "grad_norm": 0.024210037663578987,
+      "kl": 0.0003743171691894531,
+      "learning_rate": 1.6508608292777203e-07,
+      "loss": 0.0,
+      "reward": 0.4695848897099495,
+      "reward_std": 0.885172575712204,
+      "rewards/cosine_scaled_reward": 0.005625786259770393,
+      "rewards/format_reward": 0.4583333358168602,
+      "step": 423
+    },
+    {
+      "completion_length": 2673.93758392334,
+      "epoch": 0.4845714285714286,
+      "grad_norm": 0.028210051357746124,
+      "kl": 0.00032579898834228516,
+      "learning_rate": 1.6346804638120098e-07,
+      "loss": 0.0,
+      "reward": 0.5200405437499285,
+      "reward_std": 0.9913763739168644,
+      "rewards/cosine_scaled_reward": 0.041270275600254536,
+      "rewards/format_reward": 0.4375000074505806,
+      "step": 424
+    },
+    {
+      "completion_length": 1474.7500305175781,
+      "epoch": 0.4857142857142857,
+      "grad_norm": 0.032733287662267685,
+      "kl": 0.00016742944717407227,
+      "learning_rate": 1.6186884885673413e-07,
+      "loss": 0.0,
+      "reward": 2.216394782066345,
+      "reward_std": 0.6638942826539278,
+      "rewards/cosine_scaled_reward": 0.6186139956116676,
+      "rewards/format_reward": 0.9791666716337204,
+      "step": 425
+    },
+    {
+      "completion_length": 1995.0416946411133,
+      "epoch": 0.4868571428571429,
+      "grad_norm": 0.041653890162706375,
+      "kl": 0.00038945674896240234,
+      "learning_rate": 1.6028856829700258e-07,
+      "loss": 0.0,
+      "reward": 0.8248560577630997,
+      "reward_std": 0.8269761241972446,
+      "rewards/cosine_scaled_reward": 0.11034468945581466,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 426
+    },
+    {
+      "completion_length": 2970.2083587646484,
+      "epoch": 0.488,
+      "grad_norm": 0.022912204265594482,
+      "kl": 0.0003581047058105469,
+      "learning_rate": 1.5872728172265146e-07,
+      "loss": 0.0,
+      "reward": 0.3813675809651613,
+      "reward_std": 0.5880897492170334,
+      "rewards/cosine_scaled_reward": 0.024017130956053734,
+      "rewards/format_reward": 0.33333333395421505,
+      "step": 427
+    },
+    {
+      "completion_length": 2192.604202270508,
+      "epoch": 0.48914285714285716,
+      "grad_norm": 0.03489791974425316,
+      "kl": 0.0003153085708618164,
+      "learning_rate": 1.5718506522858572e-07,
+      "loss": 0.0,
+      "reward": 0.6708185374736786,
+      "reward_std": 1.0236610621213913,
+      "rewards/cosine_scaled_reward": 0.03332593198865652,
+      "rewards/format_reward": 0.604166679084301,
+      "step": 428
+    },
+    {
+      "completion_length": 2013.4167098999023,
+      "epoch": 0.49028571428571427,
+      "grad_norm": 0.03182235732674599,
+      "kl": 0.00034117698669433594,
+      "learning_rate": 1.5566199398026147e-07,
+      "loss": 0.0,
+      "reward": 0.7822023816406727,
+      "reward_std": 0.6942301616072655,
+      "rewards/cosine_scaled_reward": 0.01610116008669138,
+      "rewards/format_reward": 0.7500000111758709,
+      "step": 429
+    },
+    {
+      "completion_length": 2299.312511444092,
+      "epoch": 0.49142857142857144,
+      "grad_norm": 0.0236895140260458,
+      "kl": 0.0003446042537689209,
+      "learning_rate": 1.5415814221002265e-07,
+      "loss": 0.0,
+      "reward": 0.884235505014658,
+      "reward_std": 0.5636085197329521,
+      "rewards/cosine_scaled_reward": 0.129617752507329,
+      "rewards/format_reward": 0.625,
+      "step": 430
+    },
+    {
+      "completion_length": 2268.312530517578,
+      "epoch": 0.49257142857142855,
+      "grad_norm": 0.028193598613142967,
+      "kl": 0.0004367828369140625,
+      "learning_rate": 1.5267358321348285e-07,
+      "loss": 0.0,
+      "reward": 0.5578006114810705,
+      "reward_std": 0.7199101895093918,
+      "rewards/cosine_scaled_reward": -0.023183029145002365,
+      "rewards/format_reward": 0.6041666809469461,
+      "step": 431
+    },
+    {
+      "completion_length": 2806.8125610351562,
+      "epoch": 0.4937142857142857,
+      "grad_norm": 0.02204781211912632,
+      "kl": 0.00034296512603759766,
+      "learning_rate": 1.5120838934595337e-07,
+      "loss": 0.0,
+      "reward": 0.5058688074350357,
+      "reward_std": 1.0230275467038155,
+      "rewards/cosine_scaled_reward": 0.0237677285913378,
+      "rewards/format_reward": 0.45833334513008595,
+      "step": 432
+    },
+    {
+      "completion_length": 2901.750030517578,
+      "epoch": 0.4948571428571429,
+      "grad_norm": 0.023653727024793625,
+      "kl": 0.00032711029052734375,
+      "learning_rate": 1.4976263201891613e-07,
+      "loss": 0.0,
+      "reward": 0.39292088645743206,
+      "reward_std": 0.6577230170369148,
+      "rewards/cosine_scaled_reward": 0.0193770844489336,
+      "rewards/format_reward": 0.35416666977107525,
+      "step": 433
+    },
+    {
+      "completion_length": 2797.5208740234375,
+      "epoch": 0.496,
+      "grad_norm": 0.026688504964113235,
+      "kl": 0.0003750920295715332,
+      "learning_rate": 1.483363816965435e-07,
+      "loss": 0.0,
+      "reward": -0.11356328427791595,
+      "reward_std": 0.5170617178082466,
+      "rewards/cosine_scaled_reward": -0.23386498540639877,
+      "rewards/format_reward": 0.3541666679084301,
+      "step": 434
+    },
+    {
+      "completion_length": 2098.0416946411133,
+      "epoch": 0.49714285714285716,
+      "grad_norm": 0.03707343339920044,
+      "kl": 0.00035834312438964844,
+      "learning_rate": 1.469297078922642e-07,
+      "loss": 0.0,
+      "reward": 0.45782112469896674,
+      "reward_std": 0.514393039047718,
+      "rewards/cosine_scaled_reward": -0.07317277602851391,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 435
+    },
+    {
+      "completion_length": 1943.4167022705078,
+      "epoch": 0.4982857142857143,
+      "grad_norm": 0.031391605734825134,
+      "kl": 0.0002682805061340332,
+      "learning_rate": 1.4554267916537495e-07,
+      "loss": 0.0,
+      "reward": 1.0878197699785233,
+      "reward_std": 0.7217961475253105,
+      "rewards/cosine_scaled_reward": 0.15849320590496063,
+      "rewards/format_reward": 0.7708333376795053,
+      "step": 436
+    },
+    {
+      "completion_length": 2698.0000228881836,
+      "epoch": 0.49942857142857144,
+      "grad_norm": 0.02348100021481514,
+      "kl": 0.0003057122230529785,
+      "learning_rate": 1.4417536311769885e-07,
+      "loss": 0.0,
+      "reward": 0.29235613951459527,
+      "reward_std": 0.37082277424633503,
+      "rewards/cosine_scaled_reward": -0.06215526815503836,
+      "rewards/format_reward": 0.4166666679084301,
+      "step": 437
+    },
+    {
+      "completion_length": 2779.583396911621,
+      "epoch": 0.5005714285714286,
+      "grad_norm": 0.021524518728256226,
+      "kl": 0.0003088712692260742,
+      "learning_rate": 1.4282782639029128e-07,
+      "loss": 0.0,
+      "reward": 0.6523042246699333,
+      "reward_std": 0.884812381118536,
+      "rewards/cosine_scaled_reward": 0.05531877093017101,
+      "rewards/format_reward": 0.5416666772216558,
+      "step": 438
+    },
+    {
+      "completion_length": 2301.604217529297,
+      "epoch": 0.5017142857142857,
+      "grad_norm": 0.02649969980120659,
+      "kl": 0.00031888484954833984,
+      "learning_rate": 1.4150013466019114e-07,
+      "loss": 0.0,
+      "reward": 0.5825948305428028,
+      "reward_std": 0.844958171248436,
+      "rewards/cosine_scaled_reward": -0.010785935446619987,
+      "rewards/format_reward": 0.6041666734963655,
+      "step": 439
+    },
+    {
+      "completion_length": 2899.458339691162,
+      "epoch": 0.5028571428571429,
+      "grad_norm": 0.026726072654128075,
+      "kl": 0.0004227161407470703,
+      "learning_rate": 1.4019235263722034e-07,
+      "loss": 0.0,
+      "reward": -0.10262651741504669,
+      "reward_std": 0.5177539475262165,
+      "rewards/cosine_scaled_reward": -0.17631326615810394,
+      "rewards/format_reward": 0.25,
+      "step": 440
+    },
+    {
+      "completion_length": 2830.2292137145996,
+      "epoch": 0.504,
+      "grad_norm": 0.024895742535591125,
+      "kl": 0.000400543212890625,
+      "learning_rate": 1.3890454406082956e-07,
+      "loss": 0.0,
+      "reward": 0.5320205697789788,
+      "reward_std": 0.6080023515969515,
+      "rewards/cosine_scaled_reward": 0.05767695792019367,
+      "rewards/format_reward": 0.41666667349636555,
+      "step": 441
+    },
+    {
+      "completion_length": 2856.729179382324,
+      "epoch": 0.5051428571428571,
+      "grad_norm": 0.024759601801633835,
+      "kl": 0.00035691261291503906,
+      "learning_rate": 1.3763677169699217e-07,
+      "loss": 0.0,
+      "reward": 0.14839190989732742,
+      "reward_std": 0.5381195954978466,
+      "rewards/cosine_scaled_reward": -0.11330404752516188,
+      "rewards/format_reward": 0.375,
+      "step": 442
+    },
+    {
+      "completion_length": 3081.895896911621,
+      "epoch": 0.5062857142857143,
+      "grad_norm": 0.022310299798846245,
+      "kl": 0.00034332275390625,
+      "learning_rate": 1.3638909733514452e-07,
+      "loss": 0.0,
+      "reward": 0.271579097956419,
+      "reward_std": 0.840357281267643,
+      "rewards/cosine_scaled_reward": -0.02046043984591961,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 443
+    },
+    {
+      "completion_length": 2853.8333587646484,
+      "epoch": 0.5074285714285715,
+      "grad_norm": 0.024710780009627342,
+      "kl": 0.00029730796813964844,
+      "learning_rate": 1.351615817851748e-07,
+      "loss": 0.0,
+      "reward": -0.03774651139974594,
+      "reward_std": 0.6097207479178905,
+      "rewards/cosine_scaled_reward": -0.175123262219131,
+      "rewards/format_reward": 0.31250000186264515,
+      "step": 444
+    },
+    {
+      "completion_length": 2816.8125534057617,
+      "epoch": 0.5085714285714286,
+      "grad_norm": 0.023254111409187317,
+      "kl": 0.0003762245178222656,
+      "learning_rate": 1.3395428487445914e-07,
+      "loss": 0.0,
+      "reward": 0.46338549302890897,
+      "reward_std": 0.5440634004771709,
+      "rewards/cosine_scaled_reward": -0.05997392535209656,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 445
+    },
+    {
+      "completion_length": 2824.0416946411133,
+      "epoch": 0.5097142857142857,
+      "grad_norm": 0.0227351114153862,
+      "kl": 0.0003020167350769043,
+      "learning_rate": 1.3276726544494571e-07,
+      "loss": 0.0,
+      "reward": 0.1085881874896586,
+      "reward_std": 0.8252651356160641,
+      "rewards/cosine_scaled_reward": -0.13320591230876744,
+      "rewards/format_reward": 0.3750000037252903,
+      "step": 446
+    },
+    {
+      "completion_length": 1687.4791831970215,
+      "epoch": 0.5108571428571429,
+      "grad_norm": 0.03260977193713188,
+      "kl": 0.00029033422470092773,
+      "learning_rate": 1.316005813502869e-07,
+      "loss": 0.0,
+      "reward": 0.66987594217062,
+      "reward_std": 0.68526791036129,
+      "rewards/cosine_scaled_reward": -0.05047871172428131,
+      "rewards/format_reward": 0.7708333395421505,
+      "step": 447
+    },
+    {
+      "completion_length": 1887.5000305175781,
+      "epoch": 0.512,
+      "grad_norm": 0.03330305963754654,
+      "kl": 0.0002282261848449707,
+      "learning_rate": 1.3045428945301953e-07,
+      "loss": 0.0,
+      "reward": 0.6945083662867546,
+      "reward_std": 0.6089707408100367,
+      "rewards/cosine_scaled_reward": 0.013920823112130165,
+      "rewards/format_reward": 0.6666666679084301,
+      "step": 448
+    },
+    {
+      "completion_length": 2491.375026702881,
+      "epoch": 0.5131428571428571,
+      "grad_norm": 0.02716596983373165,
+      "kl": 0.00034224987030029297,
+      "learning_rate": 1.2932844562179352e-07,
+      "loss": 0.0,
+      "reward": 0.04862409457564354,
+      "reward_std": 0.44541748240590096,
+      "rewards/cosine_scaled_reward": -0.18402129039168358,
+      "rewards/format_reward": 0.4166666679084301,
+      "step": 449
+    },
+    {
+      "completion_length": 2376.3333702087402,
+      "epoch": 0.5142857142857142,
+      "grad_norm": 0.026498140767216682,
+      "kl": 0.000276029109954834,
+      "learning_rate": 1.2822310472864885e-07,
+      "loss": 0.0,
+      "reward": 0.45986196398735046,
+      "reward_std": 0.5138499777531251,
+      "rewards/cosine_scaled_reward": -0.009652364067733288,
+      "rewards/format_reward": 0.4791666679084301,
+      "step": 450
+    },
+    {
+      "completion_length": 2555.812526702881,
+      "epoch": 0.5154285714285715,
+      "grad_norm": 0.03134075179696083,
+      "kl": 0.00034427642822265625,
+      "learning_rate": 1.2713832064634125e-07,
+      "loss": 0.0,
+      "reward": 0.52835633745417,
+      "reward_std": 0.5414374638348818,
+      "rewards/cosine_scaled_reward": -0.017071835696697235,
+      "rewards/format_reward": 0.5625000055879354,
+      "step": 451
+    },
+    {
+      "completion_length": 2659.3542098999023,
+      "epoch": 0.5165714285714286,
+      "grad_norm": 0.025225255638360977,
+      "kl": 0.00030180811882019043,
+      "learning_rate": 1.260741462457165e-07,
+      "loss": 0.0,
+      "reward": 0.6744653210043907,
+      "reward_std": 0.7822801172733307,
+      "rewards/cosine_scaled_reward": 0.06639931770041585,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 452
+    },
+    {
+      "completion_length": 2583.979259490967,
+      "epoch": 0.5177142857142857,
+      "grad_norm": 0.0343073271214962,
+      "kl": 0.0003584623336791992,
+      "learning_rate": 1.2503063339313356e-07,
+      "loss": 0.0,
+      "reward": 0.6048378571867943,
+      "reward_std": 0.8740231655538082,
+      "rewards/cosine_scaled_reward": 0.05241893231868744,
+      "rewards/format_reward": 0.5000000055879354,
+      "step": 453
+    },
+    {
+      "completion_length": 2162.4167098999023,
+      "epoch": 0.5188571428571429,
+      "grad_norm": 0.024462906643748283,
+      "kl": 0.0002703070640563965,
+      "learning_rate": 1.2400783294793668e-07,
+      "loss": 0.0,
+      "reward": 0.7111623287200928,
+      "reward_std": 0.5601445361971855,
+      "rewards/cosine_scaled_reward": -0.00900217518210411,
+      "rewards/format_reward": 0.7291666716337204,
+      "step": 454
+    },
+    {
+      "completion_length": 2792.5416831970215,
+      "epoch": 0.52,
+      "grad_norm": 0.032766759395599365,
+      "kl": 0.00035133957862854004,
+      "learning_rate": 1.2300579475997657e-07,
+      "loss": 0.0,
+      "reward": -0.10949165653437376,
+      "reward_std": 0.4895745702087879,
+      "rewards/cosine_scaled_reward": -0.26307916827499866,
+      "rewards/format_reward": 0.4166666716337204,
+      "step": 455
+    },
+    {
+      "completion_length": 3024.1875228881836,
+      "epoch": 0.5211428571428571,
+      "grad_norm": 0.02690344676375389,
+      "kl": 0.00028765201568603516,
+      "learning_rate": 1.220245676671809e-07,
+      "loss": 0.0,
+      "reward": 0.023419101489707828,
+      "reward_std": 0.571003105957061,
+      "rewards/cosine_scaled_reward": -0.1549571119248867,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 456
+    },
+    {
+      "completion_length": 2981.645881652832,
+      "epoch": 0.5222857142857142,
+      "grad_norm": 0.03079189360141754,
+      "kl": 0.0004329681396484375,
+      "learning_rate": 1.2106419949317388e-07,
+      "loss": 0.0,
+      "reward": 0.26157396100461483,
+      "reward_std": 0.710116732865572,
+      "rewards/cosine_scaled_reward": -0.046296343207359314,
+      "rewards/format_reward": 0.3541666679084301,
+      "step": 457
+    },
+    {
+      "completion_length": 2113.458354949951,
+      "epoch": 0.5234285714285715,
+      "grad_norm": 0.03327843174338341,
+      "kl": 0.00027066469192504883,
+      "learning_rate": 1.2012473704494537e-07,
+      "loss": 0.0,
+      "reward": 0.6744469236582518,
+      "reward_std": 0.7916180063039064,
+      "rewards/cosine_scaled_reward": 0.024723445996642113,
+      "rewards/format_reward": 0.6250000093132257,
+      "step": 458
+    },
+    {
+      "completion_length": 1241.8541870117188,
+      "epoch": 0.5245714285714286,
+      "grad_norm": 0.03587225452065468,
+      "kl": 0.00023978948593139648,
+      "learning_rate": 1.1920622611056974e-07,
+      "loss": 0.0,
+      "reward": 1.3831844218075275,
+      "reward_std": 0.652434827759862,
+      "rewards/cosine_scaled_reward": 0.2540921785403043,
+      "rewards/format_reward": 0.875,
+      "step": 459
+    },
+    {
+      "completion_length": 2897.6250610351562,
+      "epoch": 0.5257142857142857,
+      "grad_norm": 0.023615367710590363,
+      "kl": 0.0003662109375,
+      "learning_rate": 1.1830871145697412e-07,
+      "loss": 0.0,
+      "reward": 0.6960660554468632,
+      "reward_std": 0.8808310199528933,
+      "rewards/cosine_scaled_reward": 0.10844967514276505,
+      "rewards/format_reward": 0.4791666716337204,
+      "step": 460
+    },
+    {
+      "completion_length": 2998.562545776367,
+      "epoch": 0.5268571428571428,
+      "grad_norm": 0.021689005196094513,
+      "kl": 0.0003491640090942383,
+      "learning_rate": 1.1743223682775649e-07,
+      "loss": 0.0,
+      "reward": 0.4467354565858841,
+      "reward_std": 0.8461587205529213,
+      "rewards/cosine_scaled_reward": 0.046284390380606055,
+      "rewards/format_reward": 0.3541666753590107,
+      "step": 461
+    },
+    {
+      "completion_length": 2987.208335876465,
+      "epoch": 0.528,
+      "grad_norm": 0.027593903243541718,
+      "kl": 0.0003991127014160156,
+      "learning_rate": 1.1657684494105386e-07,
+      "loss": 0.0,
+      "reward": -0.17366999201476574,
+      "reward_std": 0.5358176417648792,
+      "rewards/cosine_scaled_reward": -0.19100166484713554,
+      "rewards/format_reward": 0.2083333432674408,
+      "step": 462
+    },
+    {
+      "completion_length": 2619.4375610351562,
+      "epoch": 0.5291428571428571,
+      "grad_norm": 0.020383333787322044,
+      "kl": 0.0002516806125640869,
+      "learning_rate": 1.1574257748745986e-07,
+      "loss": 0.0,
+      "reward": 0.5960872750729322,
+      "reward_std": 0.5420881155878305,
+      "rewards/cosine_scaled_reward": 0.07929362449795008,
+      "rewards/format_reward": 0.4375000037252903,
+      "step": 463
+    },
+    {
+      "completion_length": 1739.2500076293945,
+      "epoch": 0.5302857142857142,
+      "grad_norm": 0.03312518447637558,
+      "kl": 0.0002790689468383789,
+      "learning_rate": 1.1492947512799328e-07,
+      "loss": 0.0,
+      "reward": 0.8862170621287078,
+      "reward_std": 0.4330704230815172,
+      "rewards/cosine_scaled_reward": 0.07852520421147346,
+      "rewards/format_reward": 0.7291666679084301,
+      "step": 464
+    },
+    {
+      "completion_length": 2631.979232788086,
+      "epoch": 0.5314285714285715,
+      "grad_norm": 0.028379030525684357,
+      "kl": 0.00039136409759521484,
+      "learning_rate": 1.1413757749211602e-07,
+      "loss": 0.0,
+      "reward": 0.547543827444315,
+      "reward_std": 1.127050258219242,
+      "rewards/cosine_scaled_reward": -0.017894762102514505,
+      "rewards/format_reward": 0.5833333414047956,
+      "step": 465
+    },
+    {
+      "completion_length": 3075.208335876465,
+      "epoch": 0.5325714285714286,
+      "grad_norm": 0.022313648834824562,
+      "kl": 0.00036776065826416016,
+      "learning_rate": 1.1336692317580158e-07,
+      "loss": 0.0,
+      "reward": 0.19751616939902306,
+      "reward_std": 0.7597125731408596,
+      "rewards/cosine_scaled_reward": -0.04707525676349178,
+      "rewards/format_reward": 0.2916666716337204,
+      "step": 466
+    },
+    {
+      "completion_length": 2750.958354949951,
+      "epoch": 0.5337142857142857,
+      "grad_norm": 0.02687336690723896,
+      "kl": 0.0003802776336669922,
+      "learning_rate": 1.1261754973965422e-07,
+      "loss": 0.0,
+      "reward": 0.6484562270343304,
+      "reward_std": 0.9428794495761395,
+      "rewards/cosine_scaled_reward": 0.07422811887226999,
+      "rewards/format_reward": 0.5000000074505806,
+      "step": 467
+    },
+    {
+      "completion_length": 2817.1666984558105,
+      "epoch": 0.5348571428571428,
+      "grad_norm": 0.02439255081117153,
+      "kl": 0.00035244226455688477,
+      "learning_rate": 1.1188949370707787e-07,
+      "loss": 0.0,
+      "reward": 0.07247266173362732,
+      "reward_std": 0.4720825038384646,
+      "rewards/cosine_scaled_reward": -0.12001369334757328,
+      "rewards/format_reward": 0.3125,
+      "step": 468
+    },
+    {
+      "completion_length": 2571.729202270508,
+      "epoch": 0.536,
+      "grad_norm": 0.030684776604175568,
+      "kl": 0.00042176246643066406,
+      "learning_rate": 1.1118279056249653e-07,
+      "loss": 0.0,
+      "reward": 0.2719057723879814,
+      "reward_std": 0.8036453519016504,
+      "rewards/cosine_scaled_reward": -0.05154711566865444,
+      "rewards/format_reward": 0.3750000074505806,
+      "step": 469
+    },
+    {
+      "completion_length": 2709.229217529297,
+      "epoch": 0.5371428571428571,
+      "grad_norm": 0.022495390847325325,
+      "kl": 0.00028634071350097656,
+      "learning_rate": 1.1049747474962444e-07,
+      "loss": 0.0,
+      "reward": 0.35282724909484386,
+      "reward_std": 0.6710633486509323,
+      "rewards/cosine_scaled_reward": -0.10483637941069901,
+      "rewards/format_reward": 0.5625000149011612,
+      "step": 470
+    },
+    {
+      "completion_length": 2807.5833587646484,
+      "epoch": 0.5382857142857143,
+      "grad_norm": 0.021343547850847244,
+      "kl": 0.00029522180557250977,
+      "learning_rate": 1.0983357966978745e-07,
+      "loss": 0.0,
+      "reward": 0.1727424543350935,
+      "reward_std": 0.5023133680224419,
+      "rewards/cosine_scaled_reward": -0.08029545284807682,
+      "rewards/format_reward": 0.3333333358168602,
+      "step": 471
+    },
+    {
+      "completion_length": 2692.979202270508,
+      "epoch": 0.5394285714285715,
+      "grad_norm": 0.021070998162031174,
+      "kl": 0.0003638267517089844,
+      "learning_rate": 1.0919113768029517e-07,
+      "loss": 0.0,
+      "reward": 0.23685874231159687,
+      "reward_std": 0.6326540801674128,
+      "rewards/cosine_scaled_reward": -0.12115396093577147,
+      "rewards/format_reward": 0.47916667349636555,
+      "step": 472
+    },
+    {
+      "completion_length": 2968.833366394043,
+      "epoch": 0.5405714285714286,
+      "grad_norm": 0.025542479008436203,
+      "kl": 0.0003510713577270508,
+      "learning_rate": 1.0857018009286381e-07,
+      "loss": 0.0,
+      "reward": 0.5230726413428783,
+      "reward_std": 0.677385538816452,
+      "rewards/cosine_scaled_reward": 0.06361966766417027,
+      "rewards/format_reward": 0.3958333395421505,
+      "step": 473
+    },
+    {
+      "completion_length": 2142.437545776367,
+      "epoch": 0.5417142857142857,
+      "grad_norm": 0.026798170059919357,
+      "kl": 0.00038617849349975586,
+      "learning_rate": 1.0797073717209013e-07,
+      "loss": 0.0,
+      "reward": 1.0524125322699547,
+      "reward_std": 0.6080759838223457,
+      "rewards/cosine_scaled_reward": 0.23453958705067635,
+      "rewards/format_reward": 0.5833333395421505,
+      "step": 474
+    },
+    {
+      "completion_length": 2133.979179382324,
+      "epoch": 0.5428571428571428,
+      "grad_norm": 0.026474615558981895,
+      "kl": 0.0002849102020263672,
+      "learning_rate": 1.0739283813397639e-07,
+      "loss": 0.0,
+      "reward": 1.4010358676314354,
+      "reward_std": 0.9220043309032917,
+      "rewards/cosine_scaled_reward": 0.2838512365706265,
+      "rewards/format_reward": 0.8333333358168602,
+      "step": 475
+    },
+    {
+      "completion_length": 2235.916702270508,
+      "epoch": 0.544,
+      "grad_norm": 0.02376784197986126,
+      "kl": 0.0003908872604370117,
+      "learning_rate": 1.068365111445064e-07,
+      "loss": 0.0,
+      "reward": 1.4387648329138756,
+      "reward_std": 1.0878624990582466,
+      "rewards/cosine_scaled_reward": 0.3339657662436366,
+      "rewards/format_reward": 0.7708333507180214,
+      "step": 476
+    },
+    {
+      "completion_length": 1504.1667022705078,
+      "epoch": 0.5451428571428572,
+      "grad_norm": 0.031137563288211823,
+      "kl": 0.00024572014808654785,
+      "learning_rate": 1.063017833182728e-07,
+      "loss": 0.0,
+      "reward": 1.313527725636959,
+      "reward_std": 0.6671716086566448,
+      "rewards/cosine_scaled_reward": 0.22968053398653865,
+      "rewards/format_reward": 0.8541666716337204,
+      "step": 477
+    },
+    {
+      "completion_length": 2789.708366394043,
+      "epoch": 0.5462857142857143,
+      "grad_norm": 0.028890853747725487,
+      "kl": 0.00031247735023498535,
+      "learning_rate": 1.0578868071715544e-07,
+      "loss": 0.0,
+      "reward": 0.7100160159170628,
+      "reward_std": 0.7924649901688099,
+      "rewards/cosine_scaled_reward": 0.08417466329410672,
+      "rewards/format_reward": 0.5416666716337204,
+      "step": 478
+    },
+    {
+      "completion_length": 2903.416702270508,
+      "epoch": 0.5474285714285714,
+      "grad_norm": 0.025586150586605072,
+      "kl": 0.00043582916259765625,
+      "learning_rate": 1.0529722834905125e-07,
+      "loss": 0.0,
+      "reward": 0.32888744259253144,
+      "reward_std": 0.7015850022435188,
+      "rewards/cosine_scaled_reward": -0.06472294870764017,
+      "rewards/format_reward": 0.45833334513008595,
+      "step": 479
+    },
+    {
+      "completion_length": 2397.000045776367,
+      "epoch": 0.5485714285714286,
+      "grad_norm": 0.029168743640184402,
+      "kl": 0.00034499168395996094,
+      "learning_rate": 1.0482745016665526e-07,
+      "loss": 0.0,
+      "reward": 0.3047554672230035,
+      "reward_std": 0.6797374170273542,
+      "rewards/cosine_scaled_reward": -0.11845559580251575,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 480
+    },
+    {
+      "completion_length": 2620.979217529297,
+      "epoch": 0.5497142857142857,
+      "grad_norm": 0.018642256036400795,
+      "kl": 0.00029289722442626953,
+      "learning_rate": 1.0437936906629334e-07,
+      "loss": 0.0,
+      "reward": 0.34190886095166206,
+      "reward_std": 0.49963630735874176,
+      "rewards/cosine_scaled_reward": -0.17279557418078184,
+      "rewards/format_reward": 0.6875000055879354,
+      "step": 481
+    },
+    {
+      "completion_length": 2566.8333587646484,
+      "epoch": 0.5508571428571428,
+      "grad_norm": 0.03212100639939308,
+      "kl": 0.00039565563201904297,
+      "learning_rate": 1.0395300688680625e-07,
+      "loss": 0.0,
+      "reward": 0.8112224619835615,
+      "reward_std": 0.8934317454695702,
+      "rewards/cosine_scaled_reward": 0.1556112226098776,
+      "rewards/format_reward": 0.5000000037252903,
+      "step": 482
+    },
+    {
+      "completion_length": 2690.5833892822266,
+      "epoch": 0.552,
+      "grad_norm": 0.02576862834393978,
+      "kl": 0.00036197900772094727,
+      "learning_rate": 1.0354838440848501e-07,
+      "loss": 0.0,
+      "reward": 0.5433402359485626,
+      "reward_std": 0.7418918535113335,
+      "rewards/cosine_scaled_reward": -0.04082988388836384,
+      "rewards/format_reward": 0.625,
+      "step": 483
+    },
+    {
+      "completion_length": 2239.270881652832,
+      "epoch": 0.5531428571428572,
+      "grad_norm": 0.02764633297920227,
+      "kl": 0.0002956390380859375,
+      "learning_rate": 1.0316552135205837e-07,
+      "loss": 0.0,
+      "reward": 0.9058556277304888,
+      "reward_std": 0.6842145500704646,
+      "rewards/cosine_scaled_reward": 0.16126114130020142,
+      "rewards/format_reward": 0.5833333432674408,
+      "step": 484
+    },
+    {
+      "completion_length": 1729.1667022705078,
+      "epoch": 0.5542857142857143,
+      "grad_norm": 0.032674845308065414,
+      "kl": 0.00025451183319091797,
+      "learning_rate": 1.0280443637773163e-07,
+      "loss": 0.0,
+      "reward": 0.7097102329134941,
+      "reward_std": 0.5658861342817545,
+      "rewards/cosine_scaled_reward": -0.020144885405898094,
+      "rewards/format_reward": 0.7500000055879354,
+      "step": 485
+    },
+    {
+      "completion_length": 1834.5833435058594,
+      "epoch": 0.5554285714285714,
+      "grad_norm": 0.035480376332998276,
+      "kl": 0.0004056096076965332,
+      "learning_rate": 1.0246514708427701e-07,
+      "loss": 0.0,
+      "reward": 0.5227991119027138,
+      "reward_std": 0.5087759112939239,
+      "rewards/cosine_scaled_reward": -0.040683780796825886,
+      "rewards/format_reward": 0.6041666716337204,
+      "step": 486
+    },
+    {
+      "completion_length": 1802.9166851043701,
+      "epoch": 0.5565714285714286,
+      "grad_norm": 0.042596880346536636,
+      "kl": 0.0003460049629211426,
+      "learning_rate": 1.0214767000817596e-07,
+      "loss": 0.0,
+      "reward": 1.4236529916524887,
+      "reward_std": 0.7221514284610748,
+      "rewards/cosine_scaled_reward": 0.3472431628033519,
+      "rewards/format_reward": 0.7291666679084301,
+      "step": 487
+    },
+    {
+      "completion_length": 2004.6666679382324,
+      "epoch": 0.5577142857142857,
+      "grad_norm": 0.026572274044156075,
+      "kl": 0.0002899765968322754,
+      "learning_rate": 1.0185202062281336e-07,
+      "loss": 0.0,
+      "reward": 0.3370821550488472,
+      "reward_std": 0.49599855951964855,
+      "rewards/cosine_scaled_reward": -0.14395895414054394,
+      "rewards/format_reward": 0.6250000074505806,
+      "step": 488
+    },
+    {
+      "completion_length": 3076.958354949951,
+      "epoch": 0.5588571428571428,
+      "grad_norm": 0.02607194148004055,
+      "kl": 0.0003669261932373047,
+      "learning_rate": 1.0157821333772304e-07,
+      "loss": 0.0,
+      "reward": 0.014304319396615028,
+      "reward_std": 0.7019931301474571,
+      "rewards/cosine_scaled_reward": -0.11784783285111189,
+      "rewards/format_reward": 0.2500000037252903,
+      "step": 489
+    },
+    {
+      "completion_length": 1909.958366394043,
+      "epoch": 0.56,
+      "grad_norm": 0.03012062795460224,
+      "kl": 0.00032451748847961426,
+      "learning_rate": 1.013262614978859e-07,
+      "loss": 0.0,
+      "reward": 0.8168525598011911,
+      "reward_std": 0.5254552476108074,
+      "rewards/cosine_scaled_reward": 0.012592926621437073,
+      "rewards/format_reward": 0.7916666716337204,
+      "step": 490
+    },
+    {
+      "completion_length": 2243.708351135254,
+      "epoch": 0.5611428571428572,
+      "grad_norm": 0.027276068925857544,
+      "kl": 0.0002563595771789551,
+      "learning_rate": 1.0109617738307911e-07,
+      "loss": 0.0,
+      "reward": 1.0440415889024734,
+      "reward_std": 0.801654189825058,
+      "rewards/cosine_scaled_reward": 0.17827080003917217,
+      "rewards/format_reward": 0.6875000055879354,
+      "step": 491
+    },
+    {
+      "completion_length": 2608.958396911621,
+      "epoch": 0.5622857142857143,
+      "grad_norm": 0.03424741327762604,
+      "kl": 0.00032341480255126953,
+      "learning_rate": 1.0088797220727779e-07,
+      "loss": 0.0,
+      "reward": 0.48804343002848327,
+      "reward_std": 0.7422212455421686,
+      "rewards/cosine_scaled_reward": 0.014855045825242996,
+      "rewards/format_reward": 0.4583333395421505,
+      "step": 492
+    },
+    {
+      "completion_length": 2090.6250381469727,
+      "epoch": 0.5634285714285714,
+      "grad_norm": 0.027274034917354584,
+      "kl": 0.00032576918601989746,
+      "learning_rate": 1.0070165611810855e-07,
+      "loss": 0.0,
+      "reward": 0.814801562577486,
+      "reward_std": 0.9762416146695614,
+      "rewards/cosine_scaled_reward": 0.021984107792377472,
+      "rewards/format_reward": 0.7708333414047956,
+      "step": 493
+    },
+    {
+      "completion_length": 1355.145866394043,
+      "epoch": 0.5645714285714286,
+      "grad_norm": 0.0340682789683342,
+      "kl": 0.0002550482749938965,
+      "learning_rate": 1.005372381963547e-07,
+      "loss": 0.0,
+      "reward": 1.5873138904571533,
+      "reward_std": 0.8576665632426739,
+      "rewards/cosine_scaled_reward": 0.30407360196113586,
+      "rewards/format_reward": 0.9791666716337204,
+      "step": 494
+    },
+    {
+      "completion_length": 2332.6041946411133,
+      "epoch": 0.5657142857142857,
+      "grad_norm": 0.027515675872564316,
+      "kl": 0.0003358125686645508,
+      "learning_rate": 1.0039472645551372e-07,
+      "loss": 0.0,
+      "reward": 0.4851825125515461,
+      "reward_std": 0.766112394630909,
+      "rewards/cosine_scaled_reward": -0.017825426533818245,
+      "rewards/format_reward": 0.5208333395421505,
+      "step": 495
+    },
+    {
+      "completion_length": 1843.9792137145996,
+      "epoch": 0.5668571428571428,
+      "grad_norm": 0.03946586698293686,
+      "kl": 0.00033032894134521484,
+      "learning_rate": 1.002741278414069e-07,
+      "loss": 0.0,
+      "reward": 1.2787955980747938,
+      "reward_std": 0.9207109902054071,
+      "rewards/cosine_scaled_reward": 0.3060644338838756,
+      "rewards/format_reward": 0.6666666679084301,
+      "step": 496
+    },
+    {
+      "completion_length": 2187.2917137145996,
+      "epoch": 0.568,
+      "grad_norm": 0.029607679694890976,
+      "kl": 0.00023612380027770996,
+      "learning_rate": 1.0017544823184055e-07,
+      "loss": 0.0,
+      "reward": 1.211494023911655,
+      "reward_std": 0.6968455417081714,
+      "rewards/cosine_scaled_reward": 0.2619969863444567,
+      "rewards/format_reward": 0.6875000111758709,
+      "step": 497
+    },
+    {
+      "completion_length": 2317.7917098999023,
+      "epoch": 0.5691428571428572,
+      "grad_norm": 0.025722017511725426,
+      "kl": 0.0003026127815246582,
+      "learning_rate": 1.0009869243631952e-07,
+      "loss": 0.0,
+      "reward": 0.9396341107785702,
+      "reward_std": 0.9828709922730923,
+      "rewards/cosine_scaled_reward": 0.1469003539532423,
+      "rewards/format_reward": 0.6458333414047956,
+      "step": 498
+    },
+    {
+      "completion_length": 2263.812545776367,
+      "epoch": 0.5702857142857143,
+      "grad_norm": 0.0247795432806015,
+      "kl": 0.00023448467254638672,
+      "learning_rate": 1.000438641958131e-07,
+      "loss": 0.0,
+      "reward": 0.7084082160145044,
+      "reward_std": 0.8164577055722475,
+      "rewards/cosine_scaled_reward": 0.041704108007252216,
+      "rewards/format_reward": 0.6250000111758709,
+      "step": 499
+    },
+    {
+      "completion_length": 2721.2292404174805,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.02151358313858509,
+      "kl": 0.000295102596282959,
+      "learning_rate": 1.0001096618257236e-07,
+      "loss": 0.0,
+      "reward": 0.7634283769875765,
+      "reward_std": 0.8250323310494423,
+      "rewards/cosine_scaled_reward": 0.11088085174560547,
+      "rewards/format_reward": 0.5416666734963655,
+      "step": 500
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "step": 500,
+      "total_flos": 0.0,
+      "train_loss": 8.274834573693823e-06,
+      "train_runtime": 144683.3506,
+      "train_samples_per_second": 0.166,
+      "train_steps_per_second": 0.003
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 6,
+  "trial_name": null,
+  "trial_params": null
+}