{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5714285714285714,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 2571.2083587646484,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.19727857410907745,
      "kl": 0.0,
      "learning_rate": 2e-08,
      "loss": -0.0,
      "reward": 0.1723687592893839,
      "reward_std": 0.7976016625761986,
      "rewards/cosine_scaled_reward": -0.015534311532974243,
      "rewards/format_reward": 0.5208333488553762,
      "step": 1
    },
    {
      "completion_length": 2804.395881652832,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.18166053295135498,
      "kl": 0.0,
      "learning_rate": 4e-08,
      "loss": 0.0,
      "reward": -0.018269629566930234,
      "reward_std": 0.44402940198779106,
      "rewards/cosine_scaled_reward": -0.04980122856795788,
      "rewards/format_reward": 0.37500000558793545,
      "step": 2
    },
    {
      "completion_length": 3291.9583587646484,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.20012830197811127,
      "kl": 4.538148641586304e-05,
      "learning_rate": 6e-08,
      "loss": 0.0,
      "reward": -0.4293696880340576,
      "reward_std": 0.42283543944358826,
      "rewards/cosine_scaled_reward": -0.20520474947988987,
      "rewards/format_reward": 0.1250000037252903,
      "step": 3
    },
    {
      "completion_length": 2115.8542098999023,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.25242456793785095,
      "kl": 3.884732723236084e-05,
      "learning_rate": 8e-08,
      "loss": 0.0,
      "reward": 0.2737832348793745,
      "reward_std": 0.9233334362506866,
      "rewards/cosine_scaled_reward": -0.041339562041684985,
      "rewards/format_reward": 0.6875000055879354,
      "step": 4
    },
    {
      "completion_length": 3488.5416870117188,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.2051168829202652,
      "kl": 4.386343061923981e-05,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "reward": -0.37347570061683655,
      "reward_std": 0.6771278530359268,
      "rewards/cosine_scaled_reward": -0.20584097504615784,
      "rewards/format_reward": 0.1666666716337204,
      "step": 5
    },
    {
      "completion_length": 3050.5417404174805,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.2063540816307068,
      "kl": 4.404783248901367e-05,
      "learning_rate": 1.2e-07,
      "loss": 0.0,
      "reward": -0.21201085951179266,
      "reward_std": 0.8398218862712383,
      "rewards/cosine_scaled_reward": -0.17123706359416246,
      "rewards/format_reward": 0.29166666977107525,
      "step": 6
    },
    {
      "completion_length": 3078.5000610351562,
      "epoch": 0.008,
      "grad_norm": 0.1487358957529068,
      "kl": 2.9146671295166016e-05,
      "learning_rate": 1.4e-07,
      "loss": 0.0,
      "reward": -0.06183330807834864,
      "reward_std": 0.7480270601809025,
      "rewards/cosine_scaled_reward": -0.12833543797023594,
      "rewards/format_reward": 0.4375000149011612,
      "step": 7
    },
    {
      "completion_length": 2689.9375381469727,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.18276464939117432,
      "kl": 1.888629049062729e-05,
      "learning_rate": 1.6e-07,
      "loss": 0.0,
      "reward": 0.2490266114473343,
      "reward_std": 0.8049478307366371,
      "rewards/cosine_scaled_reward": 0.06792760454118252,
      "rewards/format_reward": 0.4583333395421505,
      "step": 8
    },
    {
      "completion_length": 3336.5000610351562,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.2246587723493576,
      "kl": 4.10228967666626e-05,
      "learning_rate": 1.8e-07,
      "loss": 0.0,
      "reward": -0.2021113825030625,
      "reward_std": 0.6383066512644291,
      "rewards/cosine_scaled_reward": -0.1338323038071394,
      "rewards/format_reward": 0.27083333395421505,
      "step": 9
    },
    {
      "completion_length": 2547.7916831970215,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.2385585457086563,
      "kl": 3.205146640539169e-05,
      "learning_rate": 2e-07,
      "loss": 0.0,
      "reward": -0.010044756345450878,
      "reward_std": 0.6000867374241352,
      "rewards/cosine_scaled_reward": -0.08963373815640807,
      "rewards/format_reward": 0.43750000186264515,
      "step": 10
    },
    {
      "completion_length": 3325.1041870117188,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.22885829210281372,
      "kl": 3.9190053939819336e-05,
      "learning_rate": 2.1999999999999998e-07,
      "loss": 0.0,
      "reward": -0.3854191079735756,
      "reward_std": 0.6930013746023178,
      "rewards/cosine_scaled_reward": -0.21312777569983155,
      "rewards/format_reward": 0.1666666716337204,
      "step": 11
    },
    {
      "completion_length": 2436.750045776367,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.2692394554615021,
      "kl": 4.0858983993530273e-05,
      "learning_rate": 2.4e-07,
      "loss": 0.0,
      "reward": 0.05584145151078701,
      "reward_std": 0.7531977295875549,
      "rewards/cosine_scaled_reward": -0.14725533686578274,
      "rewards/format_reward": 0.6250000018626451,
      "step": 12
    },
    {
      "completion_length": 2905.625030517578,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.18586185574531555,
      "kl": 3.486126661300659e-05,
      "learning_rate": 2.6e-07,
      "loss": 0.0,
      "reward": -0.06436450174078345,
      "reward_std": 0.5915969423949718,
      "rewards/cosine_scaled_reward": -0.10991157777607441,
      "rewards/format_reward": 0.416666679084301,
      "step": 13
    },
    {
      "completion_length": 2978.0208892822266,
      "epoch": 0.016,
      "grad_norm": 0.21407712996006012,
      "kl": 2.9122806154191494e-05,
      "learning_rate": 2.8e-07,
      "loss": 0.0,
      "reward": 0.008545992895960808,
      "reward_std": 0.8296682890504599,
      "rewards/cosine_scaled_reward": -0.05597527138888836,
      "rewards/format_reward": 0.3750000074505806,
      "step": 14
    },
    {
      "completion_length": 2821.291679382324,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.18756185472011566,
      "kl": 2.925284206867218e-05,
      "learning_rate": 3e-07,
      "loss": 0.0,
      "reward": 0.09492451697587967,
      "reward_std": 0.5804186388850212,
      "rewards/cosine_scaled_reward": -0.005186626687645912,
      "rewards/format_reward": 0.41666667349636555,
      "step": 15
    },
    {
      "completion_length": 3463.1458435058594,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.19236689805984497,
      "kl": 3.2588839530944824e-05,
      "learning_rate": 3.2e-07,
      "loss": 0.0,
      "reward": -0.41888961754739285,
      "reward_std": 0.5801856927573681,
      "rewards/cosine_scaled_reward": -0.19731347833294421,
      "rewards/format_reward": 0.10416666977107525,
      "step": 16
    },
    {
      "completion_length": 2245.083366394043,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.24209940433502197,
      "kl": 4.054419696331024e-05,
      "learning_rate": 3.4000000000000003e-07,
      "loss": 0.0,
      "reward": 0.233390836045146,
      "reward_std": 0.9472852721810341,
      "rewards/cosine_scaled_reward": -0.0257677553454414,
      "rewards/format_reward": 0.6041666716337204,
      "step": 17
    },
    {
      "completion_length": 2929.062530517578,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.14878323674201965,
      "kl": 2.1327286958694458e-05,
      "learning_rate": 3.6e-07,
      "loss": 0.0,
      "reward": -0.019776458386331797,
      "reward_std": 0.5773040689527988,
      "rewards/cosine_scaled_reward": -0.08265479793772101,
      "rewards/format_reward": 0.4166666716337204,
      "step": 18
    },
    {
      "completion_length": 2807.437545776367,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.16009841859340668,
      "kl": 2.41696834564209e-05,
      "learning_rate": 3.7999999999999996e-07,
      "loss": 0.0,
      "reward": 0.3507382436655462,
      "reward_std": 0.921161625534296,
      "rewards/cosine_scaled_reward": 0.12314888671971858,
      "rewards/format_reward": 0.4583333395421505,
      "step": 19
    },
    {
      "completion_length": 2367.8958892822266,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.18880178034305573,
      "kl": 1.8640421330928802e-05,
      "learning_rate": 4e-07,
      "loss": 0.0,
      "reward": 0.5233698049560189,
      "reward_std": 0.9742525108158588,
      "rewards/cosine_scaled_reward": 0.14227081835269928,
      "rewards/format_reward": 0.666666679084301,
      "step": 20
    },
    {
      "completion_length": 2770.4583740234375,
      "epoch": 0.024,
      "grad_norm": 0.24240563809871674,
      "kl": 3.499537706375122e-05,
      "learning_rate": 4.1999999999999995e-07,
      "loss": 0.0,
      "reward": -0.16247276589274406,
      "reward_std": 0.5624231658875942,
      "rewards/cosine_scaled_reward": -0.1772752869874239,
      "rewards/format_reward": 0.4166666716337204,
      "step": 21
    },
    {
      "completion_length": 1995.1875305175781,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.43609920144081116,
      "kl": 3.091990947723389e-05,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0,
      "reward": 0.36330048087984324,
      "reward_std": 0.6434031687676907,
      "rewards/cosine_scaled_reward": 0.004692776128649712,
      "rewards/format_reward": 0.750000013038516,
      "step": 22
    },
    {
      "completion_length": 2544.666717529297,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.23585595190525055,
      "kl": 3.0001625418663025e-05,
      "learning_rate": 4.6e-07,
      "loss": 0.0,
      "reward": -0.11523125227540731,
      "reward_std": 0.6521053463220596,
      "rewards/cosine_scaled_reward": -0.16176861617714167,
      "rewards/format_reward": 0.4375000111758709,
      "step": 23
    },
    {
      "completion_length": 2566.333335876465,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.18629349768161774,
      "kl": 2.0964653231203556e-05,
      "learning_rate": 4.8e-07,
      "loss": 0.0,
      "reward": 0.23491991590708494,
      "reward_std": 0.7349754758179188,
      "rewards/cosine_scaled_reward": -0.0008563240990042686,
      "rewards/format_reward": 0.5833333469927311,
      "step": 24
    },
    {
      "completion_length": 2641.145881652832,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.1711462438106537,
      "kl": 3.1113624572753906e-05,
      "learning_rate": 5e-07,
      "loss": 0.0,
      "reward": 0.03685340657830238,
      "reward_std": 0.7854471541941166,
      "rewards/cosine_scaled_reward": -0.06554372794926167,
      "rewards/format_reward": 0.43750001303851604,
      "step": 25
    },
    {
      "completion_length": 3136.604217529297,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.19311213493347168,
      "kl": 4.175305366516113e-05,
      "learning_rate": 5.2e-07,
      "loss": 0.0,
      "reward": -0.036069474183022976,
      "reward_std": 0.7284176610410213,
      "rewards/cosine_scaled_reward": -0.06703814025968313,
      "rewards/format_reward": 0.3541666716337204,
      "step": 26
    },
    {
      "completion_length": 2979.4583740234375,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.2027575522661209,
      "kl": 3.305543214082718e-05,
      "learning_rate": 5.4e-07,
      "loss": 0.0,
      "reward": 0.02418377436697483,
      "reward_std": 0.8362919054925442,
      "rewards/cosine_scaled_reward": -0.07142194919288158,
      "rewards/format_reward": 0.4375000111758709,
      "step": 27
    },
    {
      "completion_length": 2897.979202270508,
      "epoch": 0.032,
      "grad_norm": 0.19330666959285736,
      "kl": 3.600865602493286e-05,
      "learning_rate": 5.6e-07,
      "loss": 0.0,
      "reward": 0.09552552457898855,
      "reward_std": 0.6331899762153625,
      "rewards/cosine_scaled_reward": -0.015744205564260483,
      "rewards/format_reward": 0.43750001303851604,
      "step": 28
    },
    {
      "completion_length": 3302.8958740234375,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.16206476092338562,
      "kl": 2.98917293548584e-05,
      "learning_rate": 5.8e-07,
      "loss": 0.0,
      "reward": -0.2337268814444542,
      "reward_std": 0.5517648197710514,
      "rewards/cosine_scaled_reward": -0.12987937778234482,
      "rewards/format_reward": 0.2291666753590107,
      "step": 29
    },
    {
      "completion_length": 2989.604232788086,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.15634141862392426,
      "kl": 2.584606409072876e-05,
      "learning_rate": 6e-07,
      "loss": 0.0,
      "reward": 0.2867018459364772,
      "reward_std": 0.8898419290781021,
      "rewards/cosine_scaled_reward": 0.0461630261852406,
      "rewards/format_reward": 0.541666679084301,
      "step": 30
    },
    {
      "completion_length": 2971.3750228881836,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.17451131343841553,
      "kl": 2.5779008865356445e-05,
      "learning_rate": 6.2e-07,
      "loss": 0.0,
      "reward": -0.1252674162387848,
      "reward_std": 0.6488835532218218,
      "rewards/cosine_scaled_reward": -0.11415221774950624,
      "rewards/format_reward": 0.3333333395421505,
      "step": 31
    },
    {
      "completion_length": 3259.375030517578,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.1704079955816269,
      "kl": 2.6818830519914627e-05,
      "learning_rate": 6.4e-07,
      "loss": 0.0,
      "reward": 0.015448510646820068,
      "reward_std": 0.6844744682312012,
      "rewards/cosine_scaled_reward": -0.022644946351647377,
      "rewards/format_reward": 0.3333333395421505,
      "step": 32
    },
    {
      "completion_length": 3343.5208435058594,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.1502685844898224,
      "kl": 2.950429916381836e-05,
      "learning_rate": 6.6e-07,
      "loss": 0.0,
      "reward": -0.3123129680752754,
      "reward_std": 0.4822962246835232,
      "rewards/cosine_scaled_reward": -0.17962717823684216,
      "rewards/format_reward": 0.22916666977107525,
      "step": 33
    },
    {
      "completion_length": 2390.2083435058594,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.32976019382476807,
      "kl": 3.445148468017578e-05,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0,
      "reward": 0.5364574566483498,
      "reward_std": 0.7043890058994293,
      "rewards/cosine_scaled_reward": 0.2165452465415001,
      "rewards/format_reward": 0.5625,
      "step": 34
    },
    {
      "completion_length": 3077.145866394043,
      "epoch": 0.04,
      "grad_norm": 0.21368089318275452,
      "kl": 4.202127456665039e-05,
      "learning_rate": 7e-07,
      "loss": 0.0,
      "reward": -0.110074105206877,
      "reward_std": 0.9502048678696156,
      "rewards/cosine_scaled_reward": -0.08587458729743958,
      "rewards/format_reward": 0.2500000037252903,
      "step": 35
    },
    {
      "completion_length": 3360.687530517578,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.1669422686100006,
      "kl": 3.0137598514556885e-05,
      "learning_rate": 7.2e-07,
      "loss": 0.0,
      "reward": -0.4919657548889518,
      "reward_std": 0.4837723895907402,
      "rewards/cosine_scaled_reward": -0.28421421349048615,
      "rewards/format_reward": 0.18750000558793545,
      "step": 36
    },
    {
      "completion_length": 3358.7291870117188,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.1673896312713623,
      "kl": 1.73947773873806e-05,
      "learning_rate": 7.4e-07,
      "loss": 0.0,
      "reward": -0.35696901264600456,
      "reward_std": 0.4194028750061989,
      "rewards/cosine_scaled_reward": -0.20523495320230722,
      "rewards/format_reward": 0.22916666977107525,
      "step": 37
    },
    {
      "completion_length": 3273.2708435058594,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.17379561066627502,
      "kl": 3.8963742554187775e-05,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0,
      "reward": -0.2825834956020117,
      "reward_std": 0.6573631279170513,
      "rewards/cosine_scaled_reward": -0.14204833284020424,
      "rewards/format_reward": 0.1666666679084301,
      "step": 38
    },
    {
      "completion_length": 2792.2500228881836,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.2008347511291504,
      "kl": 1.8621794879436493e-05,
      "learning_rate": 7.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.01606392953544855,
      "reward_std": 0.6000814624130726,
      "rewards/cosine_scaled_reward": -0.07735766470432281,
      "rewards/format_reward": 0.4583333469927311,
      "step": 39
    },
    {
      "completion_length": 2641.7084045410156,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.21129944920539856,
      "kl": 6.28940761089325e-05,
      "learning_rate": 8e-07,
      "loss": 0.0,
      "reward": 0.01600947417318821,
      "reward_std": 0.6365399248898029,
      "rewards/cosine_scaled_reward": -0.11234160404001159,
      "rewards/format_reward": 0.5208333469927311,
      "step": 40
    },
    {
      "completion_length": 2874.9791870117188,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.17788062989711761,
      "kl": 3.086775541305542e-05,
      "learning_rate": 8.199999999999999e-07,
      "loss": 0.0,
      "reward": -0.050673360005021095,
      "reward_std": 0.9452972374856472,
      "rewards/cosine_scaled_reward": -0.14972026087343693,
      "rewards/format_reward": 0.4583333469927311,
      "step": 41
    },
    {
      "completion_length": 2774.2916984558105,
      "epoch": 0.048,
      "grad_norm": 0.3309200704097748,
      "kl": 6.945431232452393e-05,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.0,
      "reward": -0.2789352214895189,
      "reward_std": 0.5039437636733055,
      "rewards/cosine_scaled_reward": -0.24113959958776832,
      "rewards/format_reward": 0.3958333358168602,
      "step": 42
    },
    {
      "completion_length": 2986.1667098999023,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.19303752481937408,
      "kl": 3.374367952346802e-05,
      "learning_rate": 8.599999999999999e-07,
      "loss": 0.0,
      "reward": -0.041154950857162476,
      "reward_std": 0.6381138861179352,
      "rewards/cosine_scaled_reward": -0.029581679904367775,
      "rewards/format_reward": 0.27083333395421505,
      "step": 43
    },
    {
      "completion_length": 2853.937530517578,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.24794019758701324,
      "kl": 0.00017474591732025146,
      "learning_rate": 8.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.08132272958755493,
      "reward_std": 0.7235868386924267,
      "rewards/cosine_scaled_reward": -0.0631332267075777,
      "rewards/format_reward": 0.5000000111758709,
      "step": 44
    },
    {
      "completion_length": 3266.812530517578,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.15977022051811218,
      "kl": 5.7324767112731934e-05,
      "learning_rate": 9e-07,
      "loss": 0.0,
      "reward": -0.18004001304507256,
      "reward_std": 0.7050215303897858,
      "rewards/cosine_scaled_reward": -0.12542735040187836,
      "rewards/format_reward": 0.2708333358168602,
      "step": 45
    },
    {
      "completion_length": 3250.2083435058594,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.19012750685214996,
      "kl": 8.171796798706055e-05,
      "learning_rate": 9.2e-07,
      "loss": 0.0,
      "reward": -0.32018135441467166,
      "reward_std": 0.6548310741782188,
      "rewards/cosine_scaled_reward": -0.17957252322230488,
      "rewards/format_reward": 0.18750000186264515,
      "step": 46
    },
    {
      "completion_length": 2832.333351135254,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.2044163942337036,
      "kl": 3.51807102560997e-05,
      "learning_rate": 9.399999999999999e-07,
      "loss": 0.0,
      "reward": 0.25673263892531395,
      "reward_std": 0.8521627373993397,
      "rewards/cosine_scaled_reward": 0.061620082706213,
      "rewards/format_reward": 0.47916667349636555,
      "step": 47
    },
    {
      "completion_length": 2832.937530517578,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.20488843321800232,
      "kl": 0.00012751109898090363,
      "learning_rate": 9.6e-07,
      "loss": 0.0,
      "reward": -0.01403064839541912,
      "reward_std": 0.847560465335846,
      "rewards/cosine_scaled_reward": -0.08664842648431659,
      "rewards/format_reward": 0.3958333358168602,
      "step": 48
    },
    {
      "completion_length": 2368.645896911621,
      "epoch": 0.056,
      "grad_norm": 0.21413730084896088,
      "kl": 6.522238254547119e-05,
      "learning_rate": 9.8e-07,
      "loss": 0.0,
      "reward": 0.28362084983382374,
      "reward_std": 0.9133072569966316,
      "rewards/cosine_scaled_reward": 0.0036178361624479294,
      "rewards/format_reward": 0.6250000037252903,
      "step": 49
    },
    {
      "completion_length": 2862.520835876465,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.184512659907341,
      "kl": 0.00011056661605834961,
      "learning_rate": 1e-06,
      "loss": 0.0,
      "reward": 0.04502496123313904,
      "reward_std": 0.570039439946413,
      "rewards/cosine_scaled_reward": -0.0024489806964993477,
      "rewards/format_reward": 0.3541666679084301,
      "step": 50
    },
    {
      "completion_length": 2245.750015258789,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.21354830265045166,
      "kl": 0.00020595639944076538,
      "learning_rate": 9.999890338174275e-07,
      "loss": 0.0,
      "reward": 0.049862007377669215,
      "reward_std": 0.5920056514441967,
      "rewards/cosine_scaled_reward": -0.11850284365937114,
      "rewards/format_reward": 0.5833333358168602,
      "step": 51
    },
    {
      "completion_length": 2897.93754196167,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.222417950630188,
      "kl": 9.300559759140015e-05,
      "learning_rate": 9.999561358041868e-07,
      "loss": 0.0,
      "reward": 0.30677789729088545,
      "reward_std": 0.9850648231804371,
      "rewards/cosine_scaled_reward": 0.08488897839561105,
      "rewards/format_reward": 0.47916667722165585,
      "step": 52
    },
    {
      "completion_length": 2835.979202270508,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.2359960377216339,
      "kl": 0.00013843923807144165,
      "learning_rate": 9.999013075636804e-07,
      "loss": 0.0,
      "reward": 0.12393231969326735,
      "reward_std": 0.8365762289613485,
      "rewards/cosine_scaled_reward": -0.017082044621929526,
      "rewards/format_reward": 0.45833334513008595,
      "step": 53
    },
    {
      "completion_length": 2773.250030517578,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.16520988941192627,
      "kl": 6.0267746448516846e-05,
      "learning_rate": 9.998245517681593e-07,
      "loss": 0.0,
      "reward": 0.4487614845857024,
      "reward_std": 0.9010818786919117,
      "rewards/cosine_scaled_reward": 0.14949666894972324,
      "rewards/format_reward": 0.5625000186264515,
      "step": 54
    },
    {
      "completion_length": 2909.0000762939453,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.17167004942893982,
      "kl": 8.597038686275482e-05,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0,
      "reward": 0.11020261491648853,
      "reward_std": 0.8988244608044624,
      "rewards/cosine_scaled_reward": 0.003133175428956747,
      "rewards/format_reward": 0.3750000074505806,
      "step": 55
    },
    {
      "completion_length": 2952.333366394043,
      "epoch": 0.064,
      "grad_norm": 0.18516214191913605,
      "kl": 4.446879029273987e-05,
      "learning_rate": 9.996052735444862e-07,
      "loss": 0.0,
      "reward": -0.027613874524831772,
      "reward_std": 0.6530672702938318,
      "rewards/cosine_scaled_reward": -0.09866334870457649,
      "rewards/format_reward": 0.4375000111758709,
      "step": 56
    },
    {
      "completion_length": 3412.1041870117188,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.12485022842884064,
      "kl": 1.195073127746582e-05,
      "learning_rate": 9.994627618036452e-07,
      "loss": 0.0,
      "reward": -0.1474010832607746,
      "reward_std": 0.7124437093734741,
      "rewards/cosine_scaled_reward": -0.10700338426977396,
      "rewards/format_reward": 0.2708333358168602,
      "step": 57
    },
    {
      "completion_length": 2150.3750534057617,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.1975310742855072,
      "kl": 0.0003490075469017029,
      "learning_rate": 9.992983438818915e-07,
      "loss": 0.0,
      "reward": 0.4446102287620306,
      "reward_std": 0.8766555078327656,
      "rewards/cosine_scaled_reward": 0.0772969089448452,
      "rewards/format_reward": 0.6875000186264515,
      "step": 58
    },
    {
      "completion_length": 2856.041717529297,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.1683138906955719,
      "kl": 3.864988684654236e-05,
      "learning_rate": 9.991120277927223e-07,
      "loss": 0.0,
      "reward": -0.09963385201990604,
      "reward_std": 0.6718280278146267,
      "rewards/cosine_scaled_reward": -0.10808059107512236,
      "rewards/format_reward": 0.35416668094694614,
      "step": 59
    },
    {
      "completion_length": 3047.6041870117188,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.16972136497497559,
      "kl": 4.49158251285553e-05,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0,
      "reward": -0.3548573371954262,
      "reward_std": 0.5078517571091652,
      "rewards/cosine_scaled_reward": -0.25396787002682686,
      "rewards/format_reward": 0.31250000186264515,
      "step": 60
    },
    {
      "completion_length": 3066.0000610351562,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.16273915767669678,
      "kl": 0.00016046315431594849,
      "learning_rate": 9.98673738502114e-07,
      "loss": 0.0,
      "reward": 0.08701966446824372,
      "reward_std": 0.6688249669969082,
      "rewards/cosine_scaled_reward": -0.06528997980058193,
      "rewards/format_reward": 0.5208333395421505,
      "step": 61
    },
    {
      "completion_length": 2751.4583740234375,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.28360143303871155,
      "kl": 0.0005411431193351746,
      "learning_rate": 9.98421786662277e-07,
      "loss": 0.0,
      "reward": 0.1395623767748475,
      "reward_std": 0.8662982396781445,
      "rewards/cosine_scaled_reward": -0.021802921197377145,
      "rewards/format_reward": 0.479166679084301,
      "step": 62
    },
    {
      "completion_length": 2314.854217529297,
      "epoch": 0.072,
      "grad_norm": 0.21262036263942719,
      "kl": 0.0005500763654708862,
      "learning_rate": 9.981479793771866e-07,
      "loss": 0.0,
      "reward": 0.5263404976576567,
      "reward_std": 0.880022443830967,
      "rewards/cosine_scaled_reward": 0.14735051710158587,
      "rewards/format_reward": 0.6666666716337204,
      "step": 63
    },
    {
      "completion_length": 2922.4583892822266,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.17176398634910583,
      "kl": 0.00019650161266326904,
      "learning_rate": 9.97852329991824e-07,
      "loss": 0.0,
      "reward": 0.010981407947838306,
      "reward_std": 0.8544860817492008,
      "rewards/cosine_scaled_reward": -0.04802468419075012,
      "rewards/format_reward": 0.35416667349636555,
      "step": 64
    },
    {
      "completion_length": 2751.0625381469727,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.19784730672836304,
      "kl": 0.00014169886708259583,
      "learning_rate": 9.975348529157229e-07,
      "loss": 0.0,
      "reward": -0.14385659247636795,
      "reward_std": 0.6372686810791492,
      "rewards/cosine_scaled_reward": -0.16209950670599937,
      "rewards/format_reward": 0.3958333358168602,
      "step": 65
    },
    {
      "completion_length": 2123.6666717529297,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.2654637396335602,
      "kl": 0.00044383853673934937,
      "learning_rate": 9.971955636222684e-07,
      "loss": 0.0,
      "reward": 0.21715208888053894,
      "reward_std": 0.5327535588294268,
      "rewards/cosine_scaled_reward": 0.03916000574827194,
      "rewards/format_reward": 0.5,
      "step": 66
    },
    {
      "completion_length": 3416.0625,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.14119970798492432,
      "kl": 0.0003875941038131714,
      "learning_rate": 9.968344786479415e-07,
      "loss": 0.0,
      "reward": -0.544161144644022,
      "reward_std": 0.4329412467777729,
      "rewards/cosine_scaled_reward": -0.28361437405692413,
      "rewards/format_reward": 0.1250000037252903,
      "step": 67
    },
    {
      "completion_length": 2162.645881652832,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.25515133142471313,
      "kl": 0.0016131103038787842,
      "learning_rate": 9.964516155915151e-07,
      "loss": 0.0001,
      "reward": 0.18700658343732357,
      "reward_std": 0.9016527011990547,
      "rewards/cosine_scaled_reward": -0.053536335937678814,
      "rewards/format_reward": 0.6041666716337204,
      "step": 68
    },
    {
      "completion_length": 2560.0625534057617,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.2858283817768097,
      "kl": 0.0012285411357879639,
      "learning_rate": 9.960469931131936e-07,
      "loss": 0.0,
      "reward": -0.2131683579646051,
      "reward_std": 0.6651364080607891,
      "rewards/cosine_scaled_reward": -0.23473481088876724,
      "rewards/format_reward": 0.43750000558793545,
      "step": 69
    },
    {
      "completion_length": 3074.875015258789,
      "epoch": 0.08,
      "grad_norm": 0.18202482163906097,
      "kl": 0.0006042998284101486,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0,
      "reward": -0.12620488926768303,
      "reward_std": 0.6856267843395472,
      "rewards/cosine_scaled_reward": -0.13051698391791433,
      "rewards/format_reward": 0.3541666679084301,
      "step": 70
    },
    {
      "completion_length": 2711.4375228881836,
      "epoch": 0.08114285714285714,
      "grad_norm": 0.37329548597335815,
      "kl": 0.0009671822190284729,
      "learning_rate": 9.951725498333448e-07,
      "loss": 0.0,
      "reward": 0.09207316488027573,
      "reward_std": 0.7490551918745041,
      "rewards/cosine_scaled_reward": 0.00596673134714365,
      "rewards/format_reward": 0.37500000558793545,
      "step": 71
    },
    {
      "completion_length": 2840.895866394043,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.2226312905550003,
      "kl": 0.0008899271488189697,
      "learning_rate": 9.947027716509488e-07,
      "loss": 0.0,
      "reward": -0.16130333952605724,
      "reward_std": 0.48493045195937157,
      "rewards/cosine_scaled_reward": -0.13195209205150604,
      "rewards/format_reward": 0.33333334140479565,
      "step": 72
    },
    {
      "completion_length": 3428.4583740234375,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.18076418340206146,
      "kl": 0.00024478137493133545,
      "learning_rate": 9.942113192828444e-07,
      "loss": 0.0,
      "reward": -0.3564223051071167,
      "reward_std": 0.7073610313236713,
      "rewards/cosine_scaled_reward": -0.18848324241116643,
      "rewards/format_reward": 0.14583333767950535,
      "step": 73
    },
    {
      "completion_length": 3174.479248046875,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.19092188775539398,
      "kl": 0.0007887572282925248,
      "learning_rate": 9.93698216681727e-07,
      "loss": 0.0,
      "reward": -0.04798390786163509,
      "reward_std": 0.9048562906682491,
      "rewards/cosine_scaled_reward": -0.06373507156968117,
      "rewards/format_reward": 0.2916666716337204,
      "step": 74
    },
    {
      "completion_length": 3045.8958587646484,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.1473468691110611,
      "kl": 0.0011369436979293823,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0,
      "reward": 0.051474731182679534,
      "reward_std": 0.6175347343087196,
      "rewards/cosine_scaled_reward": 0.00460137240588665,
      "rewards/format_reward": 0.3333333358168602,
      "step": 75
    },
    {
      "completion_length": 2876.5208740234375,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.22771689295768738,
      "kl": 0.00022584199905395508,
      "learning_rate": 9.926071618660237e-07,
      "loss": 0.0,
      "reward": -0.2855505235493183,
      "reward_std": 0.3848421312868595,
      "rewards/cosine_scaled_reward": -0.23991595953702927,
      "rewards/format_reward": 0.39583333767950535,
      "step": 76
    },
    {
      "completion_length": 2967.3958435058594,
      "epoch": 0.088,
      "grad_norm": 0.15564849972724915,
      "kl": 0.00028374046087265015,
      "learning_rate": 9.9202926282791e-07,
      "loss": 0.0,
      "reward": -0.11336795706301928,
      "reward_std": 0.42253825441002846,
      "rewards/cosine_scaled_reward": -0.09275355748832226,
      "rewards/format_reward": 0.33333334140479565,
      "step": 77
    },
    {
      "completion_length": 3223.9583892822266,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.1456020325422287,
      "kl": 0.00031157582998275757,
      "learning_rate": 9.91429819907136e-07,
      "loss": 0.0,
      "reward": -0.039307162165641785,
      "reward_std": 0.6996998488903046,
      "rewards/cosine_scaled_reward": -0.04336683638393879,
      "rewards/format_reward": 0.3125000074505806,
      "step": 78
    },
    {
      "completion_length": 2270.0417098999023,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.20563355088233948,
      "kl": 0.0013409852981567383,
      "learning_rate": 9.908088623197048e-07,
      "loss": 0.0001,
      "reward": 0.09923344664275646,
      "reward_std": 0.7259157933294773,
      "rewards/cosine_scaled_reward": -0.10380276478827,
      "rewards/format_reward": 0.6041666679084301,
      "step": 79
    },
    {
      "completion_length": 3190.3333587646484,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.21478085219860077,
      "kl": 0.0009518936276435852,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.0,
      "reward": -0.2797470228979364,
      "reward_std": 0.6308383457362652,
      "rewards/cosine_scaled_reward": -0.18668191879987717,
      "rewards/format_reward": 0.27083333767950535,
      "step": 80
    },
    {
      "completion_length": 2895.583354949951,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.2533528804779053,
      "kl": 0.002989828586578369,
      "learning_rate": 9.895025252503755e-07,
      "loss": 0.0001,
      "reward": -0.1824733428657055,
      "reward_std": 0.5710629485547543,
      "rewards/cosine_scaled_reward": -0.15205951500684023,
      "rewards/format_reward": 0.33333333395421505,
      "step": 81
    },
    {
      "completion_length": 2906.6458740234375,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.19861973822116852,
      "kl": 0.0019384026527404785,
      "learning_rate": 9.888172094375033e-07,
      "loss": 0.0001,
      "reward": 0.002590768039226532,
      "reward_std": 0.6845040954649448,
      "rewards/cosine_scaled_reward": -0.050318428009632044,
      "rewards/format_reward": 0.37500000931322575,
      "step": 82
    },
    {
      "completion_length": 2862.9583587646484,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.5050163865089417,
      "kl": 0.0011357255280017853,
      "learning_rate": 9.881105062929221e-07,
      "loss": 0.0,
      "reward": -0.2405467852950096,
      "reward_std": 0.5951391458511353,
      "rewards/cosine_scaled_reward": -0.16155940247699618,
      "rewards/format_reward": 0.27083333395421505,
      "step": 83
    },
    {
      "completion_length": 3058.375045776367,
      "epoch": 0.096,
      "grad_norm": 0.1727391928434372,
      "kl": 0.0007171034812927246,
      "learning_rate": 9.873824502603459e-07,
      "loss": 0.0,
      "reward": 0.07485563680529594,
      "reward_std": 0.8334123902022839,
      "rewards/cosine_scaled_reward": -0.0007621082477271557,
      "rewards/format_reward": 0.35416667349636555,
      "step": 84
    },
    {
      "completion_length": 2977.7709045410156,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.14627371728420258,
      "kl": 0.0006437301635742188,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0,
      "reward": -0.01777968415990472,
      "reward_std": 0.8515256457030773,
      "rewards/cosine_scaled_reward": -0.09596842993050814,
      "rewards/format_reward": 0.4166666753590107,
      "step": 85
    },
    {
      "completion_length": 2917.0625228881836,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.20686082541942596,
      "kl": 0.001413583755493164,
      "learning_rate": 9.85862422507884e-07,
      "loss": 0.0001,
      "reward": 0.06965514738112688,
      "reward_std": 0.6772139333188534,
      "rewards/cosine_scaled_reward": -0.03397463448345661,
      "rewards/format_reward": 0.4375000074505806,
      "step": 86
    },
    {
      "completion_length": 2726.0833740234375,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.22425347566604614,
      "kl": 0.001748785376548767,
      "learning_rate": 9.850705248720068e-07,
      "loss": 0.0001,
      "reward": 0.15189366973936558,
      "reward_std": 0.8081401586532593,
      "rewards/cosine_scaled_reward": -0.0596234705299139,
      "rewards/format_reward": 0.5833333469927311,
      "step": 87
    },
    {
      "completion_length": 2768.354202270508,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.1945296823978424,
      "kl": 0.002580702304840088,
      "learning_rate": 9.8425742251254e-07,
      "loss": 0.0001,
      "reward": -0.015582697466015816,
      "reward_std": 0.7642906121909618,
      "rewards/cosine_scaled_reward": -0.13294149283319712,
      "rewards/format_reward": 0.5,
      "step": 88
    },
    {
      "completion_length": 3245.0833740234375,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.1559896320104599,
      "kl": 0.0016347765922546387,
      "learning_rate": 9.83423155058946e-07,
      "loss": 0.0001,
      "reward": 0.022897440940141678,
      "reward_std": 0.6935332976281643,
      "rewards/cosine_scaled_reward": -0.016921459697186947,
      "rewards/format_reward": 0.3333333395421505,
      "step": 89
    },
    {
      "completion_length": 2285.729179382324,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.3084622621536255,
      "kl": 0.0027695298194885254,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0001,
      "reward": -0.18626173213124275,
      "reward_std": 0.48771001771092415,
      "rewards/cosine_scaled_reward": -0.2529527278384194,
      "rewards/format_reward": 0.5416666679084301,
      "step": 90
    },
    {
      "completion_length": 2982.937530517578,
      "epoch": 0.104,
      "grad_norm": 0.2009773999452591,
      "kl": 0.0014831870794296265,
      "learning_rate": 9.816912885430258e-07,
      "loss": 0.0001,
      "reward": -0.15441028028726578,
      "reward_std": 0.5955647341907024,
      "rewards/cosine_scaled_reward": -0.11808557622134686,
      "rewards/format_reward": 0.3125000074505806,
      "step": 91
    },
    {
      "completion_length": 2740.3125228881836,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.22146174311637878,
      "kl": 0.003424704074859619,
      "learning_rate": 9.807937738894303e-07,
      "loss": 0.0001,
      "reward": -0.21460794005542994,
      "reward_std": 0.6763530150055885,
      "rewards/cosine_scaled_reward": -0.24455153848975897,
      "rewards/format_reward": 0.4583333469927311,
      "step": 92
    },
    {
      "completion_length": 3584.0,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.15897655487060547,
      "kl": 0.0015739202499389648,
      "learning_rate": 9.798752629550546e-07,
      "loss": 0.0001,
      "reward": -0.5496739558875561,
      "reward_std": 0.33528773859143257,
      "rewards/cosine_scaled_reward": -0.22698437981307507,
      "rewards/format_reward": 0.02083333395421505,
      "step": 93
    },
    {
      "completion_length": 3051.958366394043,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.17778827250003815,
      "kl": 0.0024556964635849,
      "learning_rate": 9.78935800506826e-07,
      "loss": 0.0001,
      "reward": -0.12991986190900207,
      "reward_std": 0.6625968441367149,
      "rewards/cosine_scaled_reward": -0.10003662994131446,
      "rewards/format_reward": 0.2916666716337204,
      "step": 94
    },
    {
      "completion_length": 3445.2500610351562,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.1542736142873764,
      "kl": 0.0006103254854679108,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.0,
      "reward": -0.22923543583601713,
      "reward_std": 0.6721294671297073,
      "rewards/cosine_scaled_reward": -0.14534958777949214,
      "rewards/format_reward": 0.25000000931322575,
      "step": 95
    },
    {
      "completion_length": 2762.812545776367,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.1754182130098343,
      "kl": 0.0021101534366607666,
      "learning_rate": 9.769942052400235e-07,
      "loss": 0.0001,
      "reward": -0.0632266215980053,
      "reward_std": 0.49075169675052166,
      "rewards/cosine_scaled_reward": -0.09756997041404247,
      "rewards/format_reward": 0.39583333395421505,
      "step": 96
    },
    {
      "completion_length": 3206.5208740234375,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.175271138548851,
      "kl": 0.0009508877992630005,
      "learning_rate": 9.759921670520634e-07,
      "loss": 0.0,
      "reward": 0.05997042031958699,
      "reward_std": 0.8464921619743109,
      "rewards/cosine_scaled_reward": -0.04258055402897298,
      "rewards/format_reward": 0.41666668094694614,
      "step": 97
    },
    {
      "completion_length": 2683.291702270508,
      "epoch": 0.112,
      "grad_norm": 0.1968272626399994,
      "kl": 0.00044539570808410645,
      "learning_rate": 9.749693666068663e-07,
      "loss": 0.0,
      "reward": 0.1016645822674036,
      "reward_std": 0.7722124308347702,
      "rewards/cosine_scaled_reward": -0.06791386939585209,
      "rewards/format_reward": 0.520833345130086,
      "step": 98
    },
    {
      "completion_length": 2768.7500076293945,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.22673000395298004,
      "kl": 0.0013333559036254883,
      "learning_rate": 9.739258537542835e-07,
      "loss": 0.0001,
      "reward": -0.15783914551138878,
      "reward_std": 0.5159456543624401,
      "rewards/cosine_scaled_reward": -0.11131853237748146,
      "rewards/format_reward": 0.2916666679084301,
      "step": 99
    },
    {
      "completion_length": 2545.4792098999023,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.18069936335086823,
      "kl": 0.002035856246948242,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0001,
      "reward": 0.4198254104703665,
      "reward_std": 0.8630654141306877,
      "rewards/cosine_scaled_reward": 0.12847260013222694,
      "rewards/format_reward": 0.5625000111758709,
      "step": 100
    },
    {
      "completion_length": 2497.000030517578,
      "epoch": 0.11542857142857142,
      "grad_norm": 0.22976230084896088,
      "kl": 0.0024021267890930176,
      "learning_rate": 9.717768952713511e-07,
      "loss": 0.0001,
      "reward": 0.06796598061919212,
      "reward_std": 0.587594460695982,
      "rewards/cosine_scaled_reward": -0.033663152426015586,
      "rewards/format_reward": 0.4375,
      "step": 101
    },
    {
      "completion_length": 2184.916717529297,
      "epoch": 0.11657142857142858,
      "grad_norm": 0.24666009843349457,
      "kl": 0.003571033477783203,
      "learning_rate": 9.706715543782064e-07,
      "loss": 0.0001,
      "reward": 0.22202930855564773,
      "reward_std": 0.7155356109142303,
      "rewards/cosine_scaled_reward": -0.06345116719603539,
      "rewards/format_reward": 0.6875000186264515,
      "step": 102
    },
    {
      "completion_length": 2856.0000610351562,
      "epoch": 0.11771428571428572,
      "grad_norm": 0.2828216850757599,
      "kl": 0.0014602243900299072,
      "learning_rate": 9.695457105469804e-07,
      "loss": 0.0001,
      "reward": 0.047393606044352055,
      "reward_std": 0.709520248696208,
      "rewards/cosine_scaled_reward": -0.06169603951275349,
      "rewards/format_reward": 0.45833334513008595,
      "step": 103
    },
    {
      "completion_length": 2754.2083587646484,
      "epoch": 0.11885714285714286,
      "grad_norm": 0.6580407023429871,
      "kl": 0.02184271812438965,
      "learning_rate": 9.683994186497132e-07,
      "loss": 0.0009,
      "reward": -0.11339985858649015,
      "reward_std": 0.5312070623040199,
      "rewards/cosine_scaled_reward": -0.11011355556547642,
      "rewards/format_reward": 0.35416666977107525,
      "step": 104
    },
    {
      "completion_length": 2917.104232788086,
      "epoch": 0.12,
      "grad_norm": 0.2027282565832138,
      "kl": 0.0013439655303955078,
      "learning_rate": 9.672327345550543e-07,
      "loss": 0.0001,
      "reward": -0.16319485567510128,
      "reward_std": 0.8842182755470276,
      "rewards/cosine_scaled_reward": -0.14117630943655968,
      "rewards/format_reward": 0.291666679084301,
      "step": 105
    },
    {
      "completion_length": 2187.4166870117188,
      "epoch": 0.12114285714285715,
      "grad_norm": 0.23987863957881927,
      "kl": 0.0010932087898254395,
      "learning_rate": 9.66045715125541e-07,
      "loss": 0.0,
      "reward": 0.5684326654300094,
      "reward_std": 0.8030262924730778,
      "rewards/cosine_scaled_reward": 0.16187836416065693,
      "rewards/format_reward": 0.7083333414047956,
      "step": 106
    },
    {
      "completion_length": 2847.750030517578,
      "epoch": 0.12228571428571429,
      "grad_norm": 0.2214292734861374,
      "kl": 0.001035928726196289,
      "learning_rate": 9.648384182148252e-07,
      "loss": 0.0,
      "reward": 0.11794419679790735,
      "reward_std": 0.49015025421977043,
      "rewards/cosine_scaled_reward": -0.0253047663718462,
      "rewards/format_reward": 0.5000000037252903,
      "step": 107
    },
    {
      "completion_length": 2861.7916870117188,
      "epoch": 0.12342857142857143,
      "grad_norm": 0.21802686154842377,
      "kl": 0.001120924949645996,
      "learning_rate": 9.636109026648554e-07,
      "loss": 0.0,
      "reward": 0.12055499106645584,
      "reward_std": 0.8332936242222786,
      "rewards/cosine_scaled_reward": -0.0109476950019598,
      "rewards/format_reward": 0.43750000558793545,
      "step": 108
    },
    {
      "completion_length": 3038.4583587646484,
      "epoch": 0.12457142857142857,
      "grad_norm": 0.17514924705028534,
      "kl": 0.0005573034286499023,
      "learning_rate": 9.623632283030077e-07,
      "loss": 0.0,
      "reward": -0.07062571635469794,
      "reward_std": 0.6698406562209129,
      "rewards/cosine_scaled_reward": -0.08270325418561697,
      "rewards/format_reward": 0.3333333395421505,
      "step": 109
    },
    {
      "completion_length": 2691.1250228881836,
      "epoch": 0.12571428571428572,
      "grad_norm": 0.2149229645729065,
      "kl": 0.000943649560213089,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0,
      "reward": -0.06578723713755608,
      "reward_std": 0.8007752932608128,
      "rewards/cosine_scaled_reward": -0.16101466468535364,
      "rewards/format_reward": 0.4791666716337204,
      "step": 110
    },
    {
      "completion_length": 3023.562545776367,
      "epoch": 0.12685714285714286,
      "grad_norm": 0.1930156946182251,
      "kl": 0.0017292499542236328,
      "learning_rate": 9.598076473627796e-07,
      "loss": 0.0001,
      "reward": 0.05401752423495054,
      "reward_std": 0.6248798258602619,
      "rewards/cosine_scaled_reward": -0.012754572555422783,
      "rewards/format_reward": 0.3750000149011612,
      "step": 111
    },
    {
      "completion_length": 3057.2083740234375,
      "epoch": 0.128,
      "grad_norm": 0.17147748172283173,
      "kl": 0.000888526439666748,
      "learning_rate": 9.58499865339809e-07,
      "loss": 0.0,
      "reward": 0.12023577280342579,
      "reward_std": 1.0201664790511131,
      "rewards/cosine_scaled_reward": -0.015141034498810768,
      "rewards/format_reward": 0.41666668094694614,
      "step": 112
    },
    {
      "completion_length": 2712.000030517578,
      "epoch": 0.12914285714285714,
      "grad_norm": 0.2709903419017792,
      "kl": 0.0015894174575805664,
      "learning_rate": 9.571721736097088e-07,
      "loss": 0.0001,
      "reward": 0.06785004865378141,
      "reward_std": 0.7393594160676003,
      "rewards/cosine_scaled_reward": -0.029724635183811188,
      "rewards/format_reward": 0.41666667349636555,
      "step": 113
    },
    {
      "completion_length": 2551.7500228881836,
      "epoch": 0.13028571428571428,
      "grad_norm": 0.23112726211547852,
      "kl": 0.002204298973083496,
      "learning_rate": 9.55824636882301e-07,
      "loss": 0.0001,
      "reward": -0.08507447713054717,
      "reward_std": 0.6186108030378819,
      "rewards/cosine_scaled_reward": -0.19081172411097214,
      "rewards/format_reward": 0.5416666846722364,
      "step": 114
    },
    {
      "completion_length": 2909.4583587646484,
      "epoch": 0.13142857142857142,
      "grad_norm": 0.20251956582069397,
      "kl": 0.002556443214416504,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.0001,
      "reward": 0.06335902085993439,
      "reward_std": 0.6296372078359127,
      "rewards/cosine_scaled_reward": -0.025603776797652245,
      "rewards/format_reward": 0.41666666977107525,
      "step": 115
    },
    {
      "completion_length": 3241.979179382324,
      "epoch": 0.13257142857142856,
      "grad_norm": 0.19871990382671356,
      "kl": 0.0013921260833740234,
      "learning_rate": 9.530702921077358e-07,
      "loss": 0.0001,
      "reward": -0.24349494744092226,
      "reward_std": 0.7026082035154104,
      "rewards/cosine_scaled_reward": -0.13611777569167316,
      "rewards/format_reward": 0.2083333358168602,
      "step": 116
    },
    {
      "completion_length": 2976.6041870117188,
      "epoch": 0.1337142857142857,
      "grad_norm": 0.20164498686790466,
      "kl": 0.0022377967834472656,
      "learning_rate": 9.516636183034564e-07,
      "loss": 0.0001,
      "reward": -0.2154933400452137,
      "reward_std": 0.7364092990756035,
      "rewards/cosine_scaled_reward": -0.16451909206807613,
      "rewards/format_reward": 0.2916666679084301,
      "step": 117
    },
    {
      "completion_length": 2854.416732788086,
      "epoch": 0.13485714285714287,
      "grad_norm": 0.18143188953399658,
      "kl": 0.001271367073059082,
      "learning_rate": 9.502373679810839e-07,
      "loss": 0.0001,
      "reward": 0.48517339397221804,
      "reward_std": 1.1588403210043907,
      "rewards/cosine_scaled_reward": 0.1642640804639086,
      "rewards/format_reward": 0.5416666753590107,
      "step": 118
    },
    {
      "completion_length": 2527.312530517578,
      "epoch": 0.136,
      "grad_norm": 0.2451496571302414,
      "kl": 0.004798531532287598,
      "learning_rate": 9.487916106540465e-07,
      "loss": 0.0002,
      "reward": 0.31419147294946015,
      "reward_std": 0.5915131606161594,
      "rewards/cosine_scaled_reward": 0.07133831456303596,
      "rewards/format_reward": 0.5625000037252903,
      "step": 119
    },
    {
      "completion_length": 2332.937526702881,
      "epoch": 0.13714285714285715,
      "grad_norm": 0.23065590858459473,
      "kl": 0.0033817291259765625,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0001,
      "reward": 0.3343121665529907,
      "reward_std": 0.7079905085265636,
      "rewards/cosine_scaled_reward": 0.032319008372724056,
      "rewards/format_reward": 0.6458333395421505,
      "step": 120
    },
    {
      "completion_length": 1775.125015258789,
      "epoch": 0.1382857142857143,
      "grad_norm": 0.2634807825088501,
      "kl": 0.004510760307312012,
      "learning_rate": 9.458418577899774e-07,
      "loss": 0.0002,
      "reward": 0.29028519266285,
      "reward_std": 0.6599521674215794,
      "rewards/cosine_scaled_reward": -0.04384784400463104,
      "rewards/format_reward": 0.7500000149011612,
      "step": 121
    },
    {
      "completion_length": 2823.0208740234375,
      "epoch": 0.13942857142857143,
      "grad_norm": 0.18922610580921173,
      "kl": 0.0015087127685546875,
      "learning_rate": 9.443380060197385e-07,
      "loss": 0.0001,
      "reward": 0.09716521622613072,
      "reward_std": 0.784220265224576,
      "rewards/cosine_scaled_reward": -0.04663149267435074,
      "rewards/format_reward": 0.4791666716337204,
      "step": 122
    },
    {
      "completion_length": 2748.3125610351562,
      "epoch": 0.14057142857142857,
      "grad_norm": 0.18109041452407837,
      "kl": 0.001852273941040039,
      "learning_rate": 9.428149347714143e-07,
      "loss": 0.0001,
      "reward": 0.029360684799030423,
      "reward_std": 0.6506396494805813,
      "rewards/cosine_scaled_reward": -0.08873580838553607,
      "rewards/format_reward": 0.47916666977107525,
      "step": 123
    },
    {
      "completion_length": 2168.9583740234375,
      "epoch": 0.1417142857142857,
      "grad_norm": 0.21630355715751648,
      "kl": 0.004555702209472656,
      "learning_rate": 9.412727182773486e-07,
      "loss": 0.0002,
      "reward": 0.06176229752600193,
      "reward_std": 0.7377404235303402,
      "rewards/cosine_scaled_reward": -0.12269194982945919,
      "rewards/format_reward": 0.5833333395421505,
      "step": 124
    },
    {
      "completion_length": 2883.666702270508,
      "epoch": 0.14285714285714285,
      "grad_norm": 0.21693550050258636,
      "kl": 0.001661539077758789,
      "learning_rate": 9.397114317029974e-07,
      "loss": 0.0001,
      "reward": -0.012577313929796219,
      "reward_std": 0.5609125196933746,
      "rewards/cosine_scaled_reward": -0.0033534724498167634,
      "rewards/format_reward": 0.27083333395421505,
      "step": 125
    },
    {
      "completion_length": 2796.4584045410156,
      "epoch": 0.144,
      "grad_norm": 0.16647285223007202,
      "kl": 0.0015259981155395508,
      "learning_rate": 9.381311511432658e-07,
      "loss": 0.0001,
      "reward": 0.02252695895731449,
      "reward_std": 0.5781615749001503,
      "rewards/cosine_scaled_reward": -0.0809040479362011,
      "rewards/format_reward": 0.479166679084301,
      "step": 126
    },
    {
      "completion_length": 2886.479202270508,
      "epoch": 0.14514285714285713,
      "grad_norm": 0.2508350610733032,
      "kl": 0.0021495819091796875,
      "learning_rate": 9.36531953618799e-07,
      "loss": 0.0001,
      "reward": -0.1614384746644646,
      "reward_std": 0.5212756060063839,
      "rewards/cosine_scaled_reward": -0.16197758680209517,
      "rewards/format_reward": 0.3958333432674408,
      "step": 127
    },
    {
      "completion_length": 2887.708366394043,
      "epoch": 0.1462857142857143,
      "grad_norm": 0.20055890083312988,
      "kl": 0.0028142929077148438,
      "learning_rate": 9.34913917072228e-07,
      "loss": 0.0001,
      "reward": 0.24019910395145416,
      "reward_std": 0.7546846177428961,
      "rewards/cosine_scaled_reward": 0.09899781085550785,
      "rewards/format_reward": 0.3958333358168602,
      "step": 128
    },
    {
      "completion_length": 3410.8333740234375,
      "epoch": 0.14742857142857144,
      "grad_norm": 0.18904834985733032,
      "kl": 0.002665996551513672,
      "learning_rate": 9.332771203643714e-07,
      "loss": 0.0001,
      "reward": -0.25656406581401825,
      "reward_std": 0.6125860698521137,
      "rewards/cosine_scaled_reward": -0.12984570860862732,
      "rewards/format_reward": 0.18750000186264515,
      "step": 129
    },
    {
      "completion_length": 2878.7500228881836,
      "epoch": 0.14857142857142858,
      "grad_norm": 0.18152809143066406,
      "kl": 0.0019643306732177734,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0001,
      "reward": -0.2177959904074669,
      "reward_std": 0.5922619923949242,
      "rewards/cosine_scaled_reward": -0.16636610962450504,
      "rewards/format_reward": 0.31250000186264515,
      "step": 130
    },
    {
      "completion_length": 2894.3125228881836,
      "epoch": 0.14971428571428572,
      "grad_norm": 0.19060392677783966,
      "kl": 0.0039920806884765625,
      "learning_rate": 9.299475664759068e-07,
      "loss": 0.0002,
      "reward": 0.15484683774411678,
      "reward_std": 0.8720951899886131,
      "rewards/cosine_scaled_reward": 0.04138324782252312,
      "rewards/format_reward": 0.3750000037252903,
      "step": 131
    },
    {
      "completion_length": 2647.1250762939453,
      "epoch": 0.15085714285714286,
      "grad_norm": 0.2062901258468628,
      "kl": 0.002198457717895508,
      "learning_rate": 9.282549715730579e-07,
      "loss": 0.0001,
      "reward": 0.21524347737431526,
      "reward_std": 0.9688759073615074,
      "rewards/cosine_scaled_reward": 0.02236761897802353,
      "rewards/format_reward": 0.4791666753590107,
      "step": 132
    },
    {
      "completion_length": 3305.375030517578,
      "epoch": 0.152,
      "grad_norm": 0.19573596119880676,
      "kl": 0.002875089645385742,
      "learning_rate": 9.265439410565328e-07,
      "loss": 0.0001,
      "reward": -0.3930224981158972,
      "reward_std": 0.40591937862336636,
      "rewards/cosine_scaled_reward": -0.22959117405116558,
      "rewards/format_reward": 0.2291666716337204,
      "step": 133
    },
    {
      "completion_length": 2513.2291870117188,
      "epoch": 0.15314285714285714,
      "grad_norm": 0.22160597145557404,
      "kl": 0.0038213729858398438,
      "learning_rate": 9.248145583195447e-07,
      "loss": 0.0002,
      "reward": 0.14458692615153268,
      "reward_std": 0.6939279437065125,
      "rewards/cosine_scaled_reward": -0.046293994411826134,
      "rewards/format_reward": 0.562500013038516,
      "step": 134
    },
    {
      "completion_length": 2181.979217529297,
      "epoch": 0.15428571428571428,
      "grad_norm": 0.2493942677974701,
      "kl": 0.0032486915588378906,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0001,
      "reward": 0.49393612146377563,
      "reward_std": 0.9125401228666306,
      "rewards/cosine_scaled_reward": 0.16448672115802765,
      "rewards/format_reward": 0.5833333395421505,
      "step": 135
    },
    {
      "completion_length": 2919.604248046875,
      "epoch": 0.15542857142857142,
      "grad_norm": 0.16584192216396332,
      "kl": 0.0039215087890625,
      "learning_rate": 9.213010742252327e-07,
      "loss": 0.0002,
      "reward": 0.26563066989183426,
      "reward_std": 0.9960127659142017,
      "rewards/cosine_scaled_reward": 0.06982969990349375,
      "rewards/format_reward": 0.4583333432674408,
      "step": 136
    },
    {
      "completion_length": 2992.1458892822266,
      "epoch": 0.15657142857142858,
      "grad_norm": 0.16871950030326843,
      "kl": 0.0024118423461914062,
      "learning_rate": 9.195171441101668e-07,
      "loss": 0.0001,
      "reward": -0.12757205590605736,
      "reward_std": 0.6657307185232639,
      "rewards/cosine_scaled_reward": -0.10885505130863748,
      "rewards/format_reward": 0.31250000186264515,
      "step": 137
    },
    {
      "completion_length": 2352.166748046875,
      "epoch": 0.15771428571428572,
      "grad_norm": 0.23502451181411743,
      "kl": 0.0022039413452148438,
      "learning_rate": 9.177152042508077e-07,
      "loss": 0.0001,
      "reward": 0.05814027041196823,
      "reward_std": 0.5509532168507576,
      "rewards/cosine_scaled_reward": -0.12178020738065243,
      "rewards/format_reward": 0.6041666828095913,
      "step": 138
    },
    {
      "completion_length": 3276.6041870117188,
      "epoch": 0.15885714285714286,
      "grad_norm": 0.18198812007904053,
      "kl": 0.004057884216308594,
      "learning_rate": 9.158953424711624e-07,
      "loss": 0.0002,
      "reward": -0.3338587903417647,
      "reward_std": 0.5466504357755184,
      "rewards/cosine_scaled_reward": -0.21947086788713932,
      "rewards/format_reward": 0.27083334513008595,
      "step": 139
    },
    {
      "completion_length": 2934.854202270508,
      "epoch": 0.16,
      "grad_norm": 0.230448380112648,
      "kl": 0.004750251770019531,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0002,
      "reward": -0.08668341906741261,
      "reward_std": 0.6167776882648468,
      "rewards/cosine_scaled_reward": -0.07037415914237499,
      "rewards/format_reward": 0.29166667722165585,
      "step": 140
    },
    {
      "completion_length": 2843.229248046875,
      "epoch": 0.16114285714285714,
      "grad_norm": 0.22269631922245026,
      "kl": 0.004426002502441406,
      "learning_rate": 9.122022088101613e-07,
      "loss": 0.0002,
      "reward": 0.011526300571858883,
      "reward_std": 0.9608168490231037,
      "rewards/cosine_scaled_reward": -0.12784388910222333,
      "rewards/format_reward": 0.500000013038516,
      "step": 141
    },
    {
      "completion_length": 2842.2084197998047,
      "epoch": 0.16228571428571428,
      "grad_norm": 0.201444610953331,
      "kl": 0.003329753875732422,
      "learning_rate": 9.103291169269299e-07,
      "loss": 0.0001,
      "reward": 0.07312965765595436,
      "reward_std": 0.7833281680941582,
      "rewards/cosine_scaled_reward": -0.09690700098872185,
      "rewards/format_reward": 0.5625000111758709,
      "step": 142
    },
    {
      "completion_length": 2778.687530517578,
      "epoch": 0.16342857142857142,
      "grad_norm": 0.2938580811023712,
      "kl": 0.0045318603515625,
      "learning_rate": 9.084384631108882e-07,
      "loss": 0.0002,
      "reward": -0.14028875157237053,
      "reward_std": 0.5204294696450233,
      "rewards/cosine_scaled_reward": -0.14672685600817204,
      "rewards/format_reward": 0.3958333395421505,
      "step": 143
    },
    {
      "completion_length": 2723.229217529297,
      "epoch": 0.16457142857142856,
      "grad_norm": 0.2279835343360901,
      "kl": 0.0032444000244140625,
      "learning_rate": 9.065303395098358e-07,
      "loss": 0.0001,
      "reward": 0.12287670839577913,
      "reward_std": 0.7171412445604801,
      "rewards/cosine_scaled_reward": 0.005602353252470493,
      "rewards/format_reward": 0.4166666753590107,
      "step": 144
    },
    {
      "completion_length": 2038.9584007263184,
      "epoch": 0.1657142857142857,
      "grad_norm": 0.3204021155834198,
      "kl": 0.004198551177978516,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.0002,
      "reward": 0.3942429169546813,
      "reward_std": 0.6445030942559242,
      "rewards/cosine_scaled_reward": 0.07814209163188934,
      "rewards/format_reward": 0.6458333395421505,
      "step": 145
    },
    {
      "completion_length": 2138.0209045410156,
      "epoch": 0.16685714285714287,
      "grad_norm": 0.20871621370315552,
      "kl": 0.0025124549865722656,
      "learning_rate": 9.026620557966279e-07,
      "loss": 0.0001,
      "reward": 0.16610028222203255,
      "reward_std": 0.7146016918122768,
      "rewards/cosine_scaled_reward": -0.11062343697994947,
      "rewards/format_reward": 0.7083333469927311,
      "step": 146
    },
    {
      "completion_length": 2778.729179382324,
      "epoch": 0.168,
      "grad_norm": 0.20433104038238525,
      "kl": 0.004992485046386719,
      "learning_rate": 9.007020842191634e-07,
      "loss": 0.0002,
      "reward": -0.17755376175045967,
      "reward_std": 0.5958786718547344,
      "rewards/cosine_scaled_reward": -0.16063082218170166,
      "rewards/format_reward": 0.3541666716337204,
      "step": 147
    },
    {
      "completion_length": 2253.812545776367,
      "epoch": 0.16914285714285715,
      "grad_norm": 0.25221163034439087,
      "kl": 0.0036783218383789062,
      "learning_rate": 8.987250199168808e-07,
      "loss": 0.0001,
      "reward": 0.21890676906332374,
      "reward_std": 0.684816125780344,
      "rewards/cosine_scaled_reward": -0.01959683746099472,
      "rewards/format_reward": 0.604166679084301,
      "step": 148
    },
    {
      "completion_length": 2701.104232788086,
      "epoch": 0.1702857142857143,
      "grad_norm": 0.19634747505187988,
      "kl": 0.002753734588623047,
      "learning_rate": 8.967309592491052e-07,
      "loss": 0.0001,
      "reward": 0.22702318988740444,
      "reward_std": 0.8613481521606445,
      "rewards/cosine_scaled_reward": 0.003744029439985752,
      "rewards/format_reward": 0.541666679084301,
      "step": 149
    },
    {
      "completion_length": 2753.479202270508,
      "epoch": 0.17142857142857143,
      "grad_norm": 0.22827665507793427,
      "kl": 0.0054454803466796875,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0002,
      "reward": 0.10332798771560192,
      "reward_std": 0.8186976052820683,
      "rewards/cosine_scaled_reward": -0.04311029799282551,
      "rewards/format_reward": 0.4791666753590107,
      "step": 150
    },
    {
      "completion_length": 2541.979202270508,
      "epoch": 0.17257142857142857,
      "grad_norm": 0.18608002364635468,
      "kl": 0.004375934600830078,
      "learning_rate": 8.926922383915315e-07,
      "loss": 0.0002,
      "reward": 0.2210833989083767,
      "reward_std": 0.9012398943305016,
      "rewards/cosine_scaled_reward": -0.0220514964312315,
      "rewards/format_reward": 0.5833333432674408,
      "step": 151
    },
    {
      "completion_length": 2626.9791870117188,
      "epoch": 0.1737142857142857,
      "grad_norm": 0.232888326048851,
      "kl": 0.003198862075805664,
      "learning_rate": 8.906477750432903e-07,
      "loss": 0.0001,
      "reward": -0.07092434912919998,
      "reward_std": 0.635146826505661,
      "rewards/cosine_scaled_reward": -0.11896505579352379,
      "rewards/format_reward": 0.4166666716337204,
      "step": 152
    },
    {
      "completion_length": 2805.5625610351562,
      "epoch": 0.17485714285714285,
      "grad_norm": 0.27297356724739075,
      "kl": 0.00728607177734375,
      "learning_rate": 8.88586709003076e-07,
      "loss": 0.0003,
      "reward": -0.17417170573025942,
      "reward_std": 0.6839433200657368,
      "rewards/cosine_scaled_reward": -0.18635874427855015,
      "rewards/format_reward": 0.39583334140479565,
      "step": 153
    },
    {
      "completion_length": 3413.7709045410156,
      "epoch": 0.176,
      "grad_norm": 0.14731697738170624,
      "kl": 0.0031638145446777344,
      "learning_rate": 8.865091407243394e-07,
      "loss": 0.0001,
      "reward": 0.05796261690557003,
      "reward_std": 1.0252362191677094,
      "rewards/cosine_scaled_reward": -0.010732692433521152,
      "rewards/format_reward": 0.3333333395421505,
      "step": 154
    },
    {
      "completion_length": 2530.604217529297,
      "epoch": 0.17714285714285713,
      "grad_norm": 0.1957230567932129,
      "kl": 0.0033855438232421875,
      "learning_rate": 8.844151714648274e-07,
      "loss": 0.0001,
      "reward": 0.001878822222352028,
      "reward_std": 0.5804712846875191,
      "rewards/cosine_scaled_reward": -0.07664009183645248,
      "rewards/format_reward": 0.43750000186264515,
      "step": 155
    },
    {
      "completion_length": 2793.583351135254,
      "epoch": 0.1782857142857143,
      "grad_norm": 0.17240341007709503,
      "kl": 0.0031371116638183594,
      "learning_rate": 8.823049032816478e-07,
      "loss": 0.0001,
      "reward": 0.20785324275493622,
      "reward_std": 0.9210717603564262,
      "rewards/cosine_scaled_reward": 0.044585417956113815,
      "rewards/format_reward": 0.4375000074505806,
      "step": 156
    },
    {
      "completion_length": 2746.2917098999023,
      "epoch": 0.17942857142857144,
      "grad_norm": 0.2299441546201706,
      "kl": 0.0045833587646484375,
      "learning_rate": 8.801784390262943e-07,
      "loss": 0.0002,
      "reward": -0.09681704035028815,
      "reward_std": 0.5040780827403069,
      "rewards/cosine_scaled_reward": -0.1591328363865614,
      "rewards/format_reward": 0.4791666753590107,
      "step": 157
    },
    {
      "completion_length": 3003.0209350585938,
      "epoch": 0.18057142857142858,
      "grad_norm": 0.20773041248321533,
      "kl": 0.003810882568359375,
      "learning_rate": 8.780358823396352e-07,
      "loss": 0.0002,
      "reward": 0.33219840307720006,
      "reward_std": 0.9787469133734703,
      "rewards/cosine_scaled_reward": 0.11911612004041672,
      "rewards/format_reward": 0.43750001303851604,
      "step": 158
    },
    {
      "completion_length": 2543.3958740234375,
      "epoch": 0.18171428571428572,
      "grad_norm": 0.17278896272182465,
      "kl": 0.003597259521484375,
      "learning_rate": 8.758773376468604e-07,
      "loss": 0.0001,
      "reward": -0.10545414686203003,
      "reward_std": 0.690795011818409,
      "rewards/cosine_scaled_reward": -0.1695910869166255,
      "rewards/format_reward": 0.45833334140479565,
      "step": 159
    },
    {
      "completion_length": 2490.2083740234375,
      "epoch": 0.18285714285714286,
      "grad_norm": 0.21473944187164307,
      "kl": 0.005862236022949219,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0002,
      "reward": 0.2439700961112976,
      "reward_std": 0.8647127598524094,
      "rewards/cosine_scaled_reward": 0.016008037142455578,
      "rewards/format_reward": 0.5416666734963655,
      "step": 160
    },
    {
      "completion_length": 2415.604202270508,
      "epoch": 0.184,
      "grad_norm": 0.21809473633766174,
      "kl": 0.004559516906738281,
      "learning_rate": 8.715127058347614e-07,
      "loss": 0.0002,
      "reward": 0.08977567870169878,
      "reward_std": 0.6846508830785751,
      "rewards/cosine_scaled_reward": -0.12287470698356628,
      "rewards/format_reward": 0.6250000111758709,
      "step": 161
    },
    {
      "completion_length": 3056.791702270508,
      "epoch": 0.18514285714285714,
      "grad_norm": 0.20735777914524078,
      "kl": 0.006768226623535156,
      "learning_rate": 8.693068314414344e-07,
      "loss": 0.0003,
      "reward": -0.18869051523506641,
      "reward_std": 0.6717521250247955,
      "rewards/cosine_scaled_reward": -0.13688376732170582,
      "rewards/format_reward": 0.29166667349636555,
      "step": 162
    },
    {
      "completion_length": 2427.437545776367,
      "epoch": 0.18628571428571428,
      "grad_norm": 0.24365104734897614,
      "kl": 0.004721641540527344,
      "learning_rate": 8.670853944836176e-07,
      "loss": 0.0002,
      "reward": 0.4920094236731529,
      "reward_std": 0.755946584045887,
      "rewards/cosine_scaled_reward": 0.16996393306180835,
      "rewards/format_reward": 0.5833333414047956,
      "step": 163
    },
    {
      "completion_length": 2403.041732788086,
      "epoch": 0.18742857142857142,
      "grad_norm": 0.19330641627311707,
      "kl": 0.0048122406005859375,
      "learning_rate": 8.648485032310144e-07,
      "loss": 0.0002,
      "reward": 0.08846932090818882,
      "reward_std": 0.7613176181912422,
      "rewards/cosine_scaled_reward": -0.06676248833537102,
      "rewards/format_reward": 0.5000000037252903,
      "step": 164
    },
    {
      "completion_length": 2565.2083435058594,
      "epoch": 0.18857142857142858,
      "grad_norm": 0.22222936153411865,
      "kl": 0.0052661895751953125,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.0002,
      "reward": -0.24459386244416237,
      "reward_std": 0.4413093514740467,
      "rewards/cosine_scaled_reward": -0.2269913526251912,
      "rewards/format_reward": 0.4166666716337204,
      "step": 165
    },
    {
      "completion_length": 2543.312515258789,
      "epoch": 0.18971428571428572,
      "grad_norm": 0.18792009353637695,
      "kl": 0.003421306610107422,
      "learning_rate": 8.603287946810513e-07,
      "loss": 0.0001,
      "reward": 0.20099905133247375,
      "reward_std": 0.7041075564920902,
      "rewards/cosine_scaled_reward": 0.01856833230704069,
      "rewards/format_reward": 0.5000000037252903,
      "step": 166
    },
    {
      "completion_length": 2143.854232788086,
      "epoch": 0.19085714285714286,
      "grad_norm": 0.24791951477527618,
      "kl": 0.003101825714111328,
      "learning_rate": 8.580461976679099e-07,
      "loss": 0.0001,
      "reward": 0.2092342609539628,
      "reward_std": 0.6548982411623001,
      "rewards/cosine_scaled_reward": -0.09624562226235867,
      "rewards/format_reward": 0.7500000186264515,
      "step": 167
    },
    {
      "completion_length": 2693.3333740234375,
      "epoch": 0.192,
      "grad_norm": 0.19963470101356506,
      "kl": 0.0033502578735351562,
      "learning_rate": 8.557485869176825e-07,
      "loss": 0.0001,
      "reward": 0.2024577334523201,
      "reward_std": 0.8647420592606068,
      "rewards/cosine_scaled_reward": 0.00969130964949727,
      "rewards/format_reward": 0.5000000111758709,
      "step": 168
    },
    {
      "completion_length": 1926.437515258789,
      "epoch": 0.19314285714285714,
      "grad_norm": 0.20806600153446198,
      "kl": 0.003902435302734375,
      "learning_rate": 8.534360744126753e-07,
      "loss": 0.0002,
      "reward": 0.7901292243041098,
      "reward_std": 0.9031771421432495,
      "rewards/cosine_scaled_reward": 0.24018706334754825,
      "rewards/format_reward": 0.8333333432674408,
      "step": 169
    },
    {
      "completion_length": 2477.1042098999023,
      "epoch": 0.19428571428571428,
      "grad_norm": 0.19945000112056732,
      "kl": 0.0035648345947265625,
      "learning_rate": 8.511087728614862e-07,
      "loss": 0.0001,
      "reward": 0.14062727987766266,
      "reward_std": 0.6545419208705425,
      "rewards/cosine_scaled_reward": -0.010002967901527882,
      "rewards/format_reward": 0.4791666753590107,
      "step": 170
    },
    {
      "completion_length": 2406.479202270508,
      "epoch": 0.19542857142857142,
      "grad_norm": 0.19067376852035522,
      "kl": 0.0029478073120117188,
      "learning_rate": 8.487667956935087e-07,
      "loss": 0.0001,
      "reward": 0.1707976944744587,
      "reward_std": 0.7575587891042233,
      "rewards/cosine_scaled_reward": 0.005063103046268225,
      "rewards/format_reward": 0.4791666828095913,
      "step": 171
    },
    {
      "completion_length": 2873.8750228881836,
      "epoch": 0.19657142857142856,
      "grad_norm": 0.2227620929479599,
      "kl": 0.006169319152832031,
      "learning_rate": 8.464102570534061e-07,
      "loss": 0.0002,
      "reward": 0.2180697526782751,
      "reward_std": 0.8882052935659885,
      "rewards/cosine_scaled_reward": 0.07928902423009276,
      "rewards/format_reward": 0.37500000186264515,
      "step": 172
    },
    {
      "completion_length": 1677.6875076293945,
      "epoch": 0.1977142857142857,
      "grad_norm": 0.2608514428138733,
      "kl": 0.0040760040283203125,
      "learning_rate": 8.440392717955475e-07,
      "loss": 0.0002,
      "reward": 0.11270578391849995,
      "reward_std": 0.7172344215214252,
      "rewards/cosine_scaled_reward": -0.1393726442474872,
      "rewards/format_reward": 0.6875,
      "step": 173
    },
    {
      "completion_length": 2115.6458740234375,
      "epoch": 0.19885714285714284,
      "grad_norm": 0.21290229260921478,
      "kl": 0.006714820861816406,
      "learning_rate": 8.416539554784089e-07,
      "loss": 0.0003,
      "reward": 0.4762940816581249,
      "reward_std": 0.8587245345115662,
      "rewards/cosine_scaled_reward": 0.08201689831912518,
      "rewards/format_reward": 0.7291666734963655,
      "step": 174
    },
    {
      "completion_length": 2647.6041870117188,
      "epoch": 0.2,
      "grad_norm": 0.15685485303401947,
      "kl": 0.004894256591796875,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0002,
      "reward": 0.2985331416130066,
      "reward_std": 0.5968715883791447,
      "rewards/cosine_scaled_reward": 0.10642770305275917,
      "rewards/format_reward": 0.4791666716337204,
      "step": 175
    },
    {
      "completion_length": 1974.5625686645508,
      "epoch": 0.20114285714285715,
      "grad_norm": 0.28263431787490845,
      "kl": 0.0044193267822265625,
      "learning_rate": 8.368407953869103e-07,
      "loss": 0.0002,
      "reward": 0.1971809258684516,
      "reward_std": 0.8409126959741116,
      "rewards/cosine_scaled_reward": -0.0763177121989429,
      "rewards/format_reward": 0.6666666697710752,
      "step": 176
    },
    {
      "completion_length": 2667.4583740234375,
      "epoch": 0.2022857142857143,
      "grad_norm": 0.2523181140422821,
      "kl": 0.005799293518066406,
      "learning_rate": 8.344131861991828e-07,
      "loss": 0.0002,
      "reward": 0.11005790415219963,
      "reward_std": 0.6982481181621552,
      "rewards/cosine_scaled_reward": -0.059121566824615,
      "rewards/format_reward": 0.541666679084301,
      "step": 177
    },
    {
      "completion_length": 2512.770881652832,
      "epoch": 0.20342857142857143,
      "grad_norm": 0.24519820511341095,
      "kl": 0.0081329345703125,
      "learning_rate": 8.319717151140072e-07,
      "loss": 0.0003,
      "reward": 0.2254039328545332,
      "reward_std": 0.8539987355470657,
      "rewards/cosine_scaled_reward": -0.00572938984259963,
      "rewards/format_reward": 0.5625000018626451,
      "step": 178
    },
    {
      "completion_length": 2653.083335876465,
      "epoch": 0.20457142857142857,
      "grad_norm": 0.3077125549316406,
      "kl": 0.004204273223876953,
      "learning_rate": 8.295165011252396e-07,
      "loss": 0.0002,
      "reward": 0.029737239703536034,
      "reward_std": 0.7201298326253891,
      "rewards/cosine_scaled_reward": -0.08826536871492863,
      "rewards/format_reward": 0.4791666716337204,
      "step": 179
    },
    {
      "completion_length": 1912.3333740234375,
      "epoch": 0.2057142857142857,
      "grad_norm": 0.2891172468662262,
      "kl": 0.006420135498046875,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0003,
      "reward": 0.3585444991476834,
      "reward_std": 0.922141257673502,
      "rewards/cosine_scaled_reward": 0.023937703692354262,
      "rewards/format_reward": 0.6666666679084301,
      "step": 180
    },
    {
      "completion_length": 2936.9583435058594,
      "epoch": 0.20685714285714285,
      "grad_norm": 0.21322257816791534,
      "kl": 0.005759239196777344,
      "learning_rate": 8.245653237555705e-07,
      "loss": 0.0002,
      "reward": -0.0014306185767054558,
      "reward_std": 0.7736919745802879,
      "rewards/cosine_scaled_reward": -0.05711588263511658,
      "rewards/format_reward": 0.3541666753590107,
      "step": 181
    },
    {
      "completion_length": 2112.875030517578,
      "epoch": 0.208,
      "grad_norm": 0.18484172224998474,
      "kl": 0.0020971298217773438,
      "learning_rate": 8.220696016880687e-07,
      "loss": 0.0001,
      "reward": 0.22709017619490623,
      "reward_std": 0.7963708490133286,
      "rewards/cosine_scaled_reward": -0.04323145607486367,
      "rewards/format_reward": 0.6458333432674408,
      "step": 182
    },
    {
      "completion_length": 1799.1042175292969,
      "epoch": 0.20914285714285713,
      "grad_norm": 0.2422093003988266,
      "kl": 0.007781982421875,
      "learning_rate": 8.195606193320136e-07,
      "loss": 0.0003,
      "reward": 0.4508074652403593,
      "reward_std": 0.8603553548455238,
      "rewards/cosine_scaled_reward": 0.010005924385040998,
      "rewards/format_reward": 0.8333333469927311,
      "step": 183
    },
    {
      "completion_length": 2360.8541870117188,
      "epoch": 0.2102857142857143,
      "grad_norm": 0.26070070266723633,
      "kl": 0.00511932373046875,
      "learning_rate": 8.170384989716657e-07,
      "loss": 0.0002,
      "reward": -0.12935980968177319,
      "reward_std": 0.5107476897537708,
      "rewards/cosine_scaled_reward": -0.21540038008242846,
      "rewards/format_reward": 0.5416666734963655,
      "step": 184
    },
    {
      "completion_length": 2236.8542098999023,
      "epoch": 0.21142857142857144,
      "grad_norm": 0.22640132904052734,
      "kl": 0.004220008850097656,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.0002,
      "reward": -0.22387782018631697,
      "reward_std": 0.3759702183306217,
      "rewards/cosine_scaled_reward": -0.2601937036961317,
      "rewards/format_reward": 0.5208333358168602,
      "step": 185
    },
    {
      "completion_length": 2573.1667098999023,
      "epoch": 0.21257142857142858,
      "grad_norm": 0.18543511629104614,
      "kl": 0.005625724792480469,
      "learning_rate": 8.119553365707802e-07,
      "loss": 0.0002,
      "reward": -0.0019482946954667568,
      "reward_std": 0.6483882665634155,
      "rewards/cosine_scaled_reward": -0.08200034685432911,
      "rewards/format_reward": 0.43750000558793545,
      "step": 186
    },
    {
      "completion_length": 1852.8958587646484,
      "epoch": 0.21371428571428572,
      "grad_norm": 0.2344105988740921,
      "kl": 0.005881786346435547,
      "learning_rate": 8.093945422764069e-07,
      "loss": 0.0002,
      "reward": 0.061095200944691896,
      "reward_std": 0.5442762821912766,
      "rewards/cosine_scaled_reward": -0.1909130923449993,
      "rewards/format_reward": 0.7500000055879354,
      "step": 187
    },
    {
      "completion_length": 2904.625030517578,
      "epoch": 0.21485714285714286,
      "grad_norm": 0.17529600858688354,
      "kl": 0.0055294036865234375,
      "learning_rate": 8.068211054579943e-07,
      "loss": 0.0002,
      "reward": -0.289316936628893,
      "reward_std": 0.5605541579425335,
      "rewards/cosine_scaled_reward": -0.2442341128771659,
      "rewards/format_reward": 0.3750000111758709,
      "step": 188
    },
    {
      "completion_length": 2011.3750457763672,
      "epoch": 0.216,
      "grad_norm": 0.5071147084236145,
      "kl": 0.0047817230224609375,
      "learning_rate": 8.04235151541222e-07,
      "loss": 0.0002,
      "reward": 0.18212716095149517,
      "reward_std": 0.6630742475390434,
      "rewards/cosine_scaled_reward": -0.06412508455105126,
      "rewards/format_reward": 0.6458333395421505,
      "step": 189
    },
    {
      "completion_length": 2108.0000610351562,
      "epoch": 0.21714285714285714,
      "grad_norm": 0.20937705039978027,
      "kl": 0.005312919616699219,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0002,
      "reward": 0.4989317310974002,
      "reward_std": 0.8167940117418766,
      "rewards/cosine_scaled_reward": 0.11576890759170055,
      "rewards/format_reward": 0.6875000055879354,
      "step": 190
    },
    {
      "completion_length": 1798.833381652832,
      "epoch": 0.21828571428571428,
      "grad_norm": 0.24339546263217926,
      "kl": 0.005016326904296875,
      "learning_rate": 7.990261971595048e-07,
      "loss": 0.0002,
      "reward": 0.3590816780924797,
      "reward_std": 0.886138778179884,
      "rewards/cosine_scaled_reward": 0.0189095304813236,
      "rewards/format_reward": 0.6875000055879354,
      "step": 191
    },
    {
      "completion_length": 2366.5000762939453,
      "epoch": 0.21942857142857142,
      "grad_norm": 0.20198509097099304,
      "kl": 0.0047245025634765625,
      "learning_rate": 7.964034505716476e-07,
      "loss": 0.0002,
      "reward": -0.017504574730992317,
      "reward_std": 0.6682247072458267,
      "rewards/cosine_scaled_reward": -0.1821177799720317,
      "rewards/format_reward": 0.6041666734963655,
      "step": 192
    },
    {
      "completion_length": 2782.4792098999023,
      "epoch": 0.22057142857142858,
      "grad_norm": 0.33190837502479553,
      "kl": 0.0082855224609375,
      "learning_rate": 7.93768694627233e-07,
      "loss": 0.0003,
      "reward": -0.09848117176443338,
      "reward_std": 0.7263169325888157,
      "rewards/cosine_scaled_reward": -0.16758773289620876,
      "rewards/format_reward": 0.45833334513008595,
      "step": 193
    },
    {
      "completion_length": 2497.958396911621,
      "epoch": 0.22171428571428572,
      "grad_norm": 0.20004087686538696,
      "kl": 0.005490303039550781,
      "learning_rate": 7.911220577405484e-07,
      "loss": 0.0002,
      "reward": 0.4655712964013219,
      "reward_std": 0.936052493751049,
      "rewards/cosine_scaled_reward": 0.11659512436017394,
      "rewards/format_reward": 0.6458333469927311,
      "step": 194
    },
    {
      "completion_length": 2314.0000915527344,
      "epoch": 0.22285714285714286,
      "grad_norm": 0.23152968287467957,
      "kl": 0.0047397613525390625,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0002,
      "reward": 0.33897530660033226,
      "reward_std": 0.9344584122300148,
      "rewards/cosine_scaled_reward": 0.003916108049452305,
      "rewards/format_reward": 0.6875000167638063,
      "step": 195
    },
    {
      "completion_length": 3108.625030517578,
      "epoch": 0.224,
      "grad_norm": 0.17755372822284698,
      "kl": 0.006099700927734375,
      "learning_rate": 7.857936576865356e-07,
      "loss": 0.0002,
      "reward": 0.04909700155258179,
      "reward_std": 0.5633429922163486,
      "rewards/cosine_scaled_reward": -0.03188931196928024,
      "rewards/format_reward": 0.41666667349636555,
      "step": 196
    },
    {
      "completion_length": 1294.9583549499512,
      "epoch": 0.22514285714285714,
      "grad_norm": 0.26458728313446045,
      "kl": 0.0051975250244140625,
      "learning_rate": 7.831121542179086e-07,
      "loss": 0.0002,
      "reward": 0.580374225974083,
      "reward_std": 0.8097280263900757,
      "rewards/cosine_scaled_reward": 0.0928502269089222,
      "rewards/format_reward": 0.8541666716337204,
      "step": 197
    },
    {
      "completion_length": 2055.2500381469727,
      "epoch": 0.22628571428571428,
      "grad_norm": 0.2084938883781433,
      "kl": 0.0077152252197265625,
      "learning_rate": 7.804192891917571e-07,
      "loss": 0.0003,
      "reward": 0.2619430273771286,
      "reward_std": 0.6607060171663761,
      "rewards/cosine_scaled_reward": -0.01983140129595995,
      "rewards/format_reward": 0.666666679084301,
      "step": 198
    },
    {
      "completion_length": 2145.4584045410156,
      "epoch": 0.22742857142857142,
      "grad_norm": 0.24516309797763824,
      "kl": 0.007678985595703125,
      "learning_rate": 7.777151938545235e-07,
      "loss": 0.0003,
      "reward": 0.267922455444932,
      "reward_std": 0.8017127625644207,
      "rewards/cosine_scaled_reward": -0.057456295005977154,
      "rewards/format_reward": 0.7291666846722364,
      "step": 199
    },
    {
      "completion_length": 1690.3958892822266,
      "epoch": 0.22857142857142856,
      "grad_norm": 0.25067099928855896,
      "kl": 0.004558563232421875,
      "learning_rate": 7.75e-07,
      "loss": 0.0002,
      "reward": 0.4661361053586006,
      "reward_std": 0.831662617623806,
      "rewards/cosine_scaled_reward": 0.043831199407577515,
      "rewards/format_reward": 0.7916666753590107,
      "step": 200
    },
    {
      "completion_length": 1972.2083854675293,
      "epoch": 0.2297142857142857,
      "grad_norm": 0.30941376090049744,
      "kl": 0.0045299530029296875,
      "learning_rate": 7.72273839962904e-07,
      "loss": 0.0002,
      "reward": 0.9293034709990025,
      "reward_std": 0.8486984223127365,
      "rewards/cosine_scaled_reward": 0.35864230850711465,
      "rewards/format_reward": 0.7916666716337204,
      "step": 201
    },
    {
      "completion_length": 1596.5625305175781,
      "epoch": 0.23085714285714284,
      "grad_norm": 0.20324425399303436,
      "kl": 0.0039119720458984375,
      "learning_rate": 7.695368466124296e-07,
      "loss": 0.0002,
      "reward": 0.5681986985728145,
      "reward_std": 0.5084148272871971,
      "rewards/cosine_scaled_reward": 0.16256619803607464,
      "rewards/format_reward": 0.7291666716337204,
      "step": 202
    },
    {
      "completion_length": 2294.5208435058594,
      "epoch": 0.232,
      "grad_norm": 0.2826099097728729,
      "kl": 0.006962776184082031,
      "learning_rate": 7.667891533457718e-07,
      "loss": 0.0003,
      "reward": 0.20878975582309067,
      "reward_std": 0.8034153431653976,
      "rewards/cosine_scaled_reward": -0.03315589763224125,
      "rewards/format_reward": 0.6041666734963655,
      "step": 203
    },
    {
      "completion_length": 1520.5000381469727,
      "epoch": 0.23314285714285715,
      "grad_norm": 0.3326990306377411,
      "kl": 0.005408287048339844,
      "learning_rate": 7.640308940816239e-07,
      "loss": 0.0002,
      "reward": 0.5824265358969569,
      "reward_std": 0.7743073143064976,
      "rewards/cosine_scaled_reward": 0.06728124246001244,
      "rewards/format_reward": 0.916666679084301,
      "step": 204
    },
    {
      "completion_length": 1976.6250534057617,
      "epoch": 0.2342857142857143,
      "grad_norm": 0.2846101224422455,
      "kl": 0.004940032958984375,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0002,
      "reward": 0.47044914215803146,
      "reward_std": 0.9910449758172035,
      "rewards/cosine_scaled_reward": 0.09328299947082996,
      "rewards/format_reward": 0.6875000093132257,
      "step": 205
    },
    {
      "completion_length": 2566.541702270508,
      "epoch": 0.23542857142857143,
      "grad_norm": 0.22673514485359192,
      "kl": 0.0060405731201171875,
      "learning_rate": 7.584832158039378e-07,
      "loss": 0.0002,
      "reward": -0.0411979085765779,
      "reward_std": 0.6954502202570438,
      "rewards/cosine_scaled_reward": -0.14637301303446293,
      "rewards/format_reward": 0.5000000111758709,
      "step": 206
    },
    {
      "completion_length": 1979.3125457763672,
      "epoch": 0.23657142857142857,
      "grad_norm": 0.2924785315990448,
      "kl": 0.007213592529296875,
      "learning_rate": 7.556940671764124e-07,
      "loss": 0.0003,
      "reward": 0.3043129206635058,
      "reward_std": 0.830761406570673,
      "rewards/cosine_scaled_reward": -0.0667324224486947,
      "rewards/format_reward": 0.7916666716337204,
      "step": 207
    },
    {
      "completion_length": 1684.020851135254,
      "epoch": 0.2377142857142857,
      "grad_norm": 0.21258480846881866,
      "kl": 0.0056591033935546875,
      "learning_rate": 7.528948933102438e-07,
      "loss": 0.0002,
      "reward": 0.4881124454550445,
      "reward_std": 0.6019946355372667,
      "rewards/cosine_scaled_reward": 0.062041960656642914,
      "rewards/format_reward": 0.8125000074505806,
      "step": 208
    },
    {
      "completion_length": 1827.8125457763672,
      "epoch": 0.23885714285714285,
      "grad_norm": 0.26896902918815613,
      "kl": 0.00604248046875,
      "learning_rate": 7.500858306332172e-07,
      "loss": 0.0002,
      "reward": 0.4111206401139498,
      "reward_std": 0.7256677895784378,
      "rewards/cosine_scaled_reward": 0.03580674855038524,
      "rewards/format_reward": 0.7500000037252903,
      "step": 209
    },
    {
      "completion_length": 1963.2917175292969,
      "epoch": 0.24,
      "grad_norm": 0.18104591965675354,
      "kl": 0.0045299530029296875,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.0002,
      "reward": 0.3350107278674841,
      "reward_std": 0.733224093914032,
      "rewards/cosine_scaled_reward": -0.016452430087156245,
      "rewards/format_reward": 0.7500000074505806,
      "step": 210
    },
    {
      "completion_length": 1673.8750228881836,
      "epoch": 0.24114285714285713,
      "grad_norm": 0.2317248284816742,
      "kl": 0.005611419677734375,
      "learning_rate": 7.444385869608921e-07,
      "loss": 0.0002,
      "reward": 0.4623087588697672,
      "reward_std": 0.633669501170516,
      "rewards/cosine_scaled_reward": 0.09324477170594037,
      "rewards/format_reward": 0.7083333432674408,
      "step": 211
    },
    {
      "completion_length": 1575.3125457763672,
      "epoch": 0.2422857142857143,
      "grad_norm": 0.26243215799331665,
      "kl": 0.0056781768798828125,
      "learning_rate": 7.416006812042827e-07,
      "loss": 0.0002,
      "reward": 0.524188793846406,
      "reward_std": 0.9563748612999916,
      "rewards/cosine_scaled_reward": 0.07078039133921266,
      "rewards/format_reward": 0.7916666716337204,
      "step": 212
    },
    {
      "completion_length": 1991.3750686645508,
      "epoch": 0.24342857142857144,
      "grad_norm": 0.2917014956474304,
      "kl": 0.009765625,
      "learning_rate": 7.387534371007797e-07,
      "loss": 0.0004,
      "reward": 0.44908707588911057,
      "reward_std": 0.7470048144459724,
      "rewards/cosine_scaled_reward": 0.06995298992842436,
      "rewards/format_reward": 0.7291666716337204,
      "step": 213
    },
    {
      "completion_length": 2065.70841217041,
      "epoch": 0.24457142857142858,
      "grad_norm": 0.2539423108100891,
      "kl": 0.00661468505859375,
      "learning_rate": 7.358969934210438e-07,
      "loss": 0.0003,
      "reward": 0.48666782677173615,
      "reward_std": 0.8372413367033005,
      "rewards/cosine_scaled_reward": 0.07745558395981789,
      "rewards/format_reward": 0.7500000149011612,
      "step": 214
    },
    {
      "completion_length": 1527.125015258789,
      "epoch": 0.24571428571428572,
      "grad_norm": 0.22233973443508148,
      "kl": 0.0035753250122070312,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0001,
      "reward": 0.27787265577353537,
      "reward_std": 0.5949838422238827,
      "rewards/cosine_scaled_reward": -0.09243576228618622,
      "rewards/format_reward": 0.8333333432674408,
      "step": 215
    },
    {
      "completion_length": 1395.00004196167,
      "epoch": 0.24685714285714286,
      "grad_norm": 0.35132068395614624,
      "kl": 0.00598907470703125,
      "learning_rate": 7.301570646506027e-07,
      "loss": 0.0002,
      "reward": 0.70725142583251,
      "reward_std": 0.6296214163303375,
      "rewards/cosine_scaled_reward": 0.16727344412356615,
      "rewards/format_reward": 0.8958333395421505,
      "step": 216
    },
    {
      "completion_length": 1762.5416946411133,
      "epoch": 0.248,
      "grad_norm": 0.18891237676143646,
      "kl": 0.005519866943359375,
      "learning_rate": 7.27273859315928e-07,
      "loss": 0.0002,
      "reward": 0.4034770044963807,
      "reward_std": 0.6970177218317986,
      "rewards/cosine_scaled_reward": 0.04039741773158312,
      "rewards/format_reward": 0.7291666716337204,
      "step": 217
    },
    {
      "completion_length": 1985.6875305175781,
      "epoch": 0.24914285714285714,
      "grad_norm": 0.2248099148273468,
      "kl": 0.005970954895019531,
      "learning_rate": 7.243820139034464e-07,
      "loss": 0.0002,
      "reward": 0.15873102471232414,
      "reward_std": 0.6960118226706982,
      "rewards/cosine_scaled_reward": -0.12544306740164757,
      "rewards/format_reward": 0.7291666697710752,
      "step": 218
    },
    {
      "completion_length": 1378.6666946411133,
      "epoch": 0.2502857142857143,
      "grad_norm": 0.26156318187713623,
      "kl": 0.0061511993408203125,
      "learning_rate": 7.214816693576234e-07,
      "loss": 0.0002,
      "reward": 0.537579414434731,
      "reward_std": 0.7653206884860992,
      "rewards/cosine_scaled_reward": 0.07624521851539612,
      "rewards/format_reward": 0.8333333432674408,
      "step": 219
    },
    {
      "completion_length": 1764.0000228881836,
      "epoch": 0.25142857142857145,
      "grad_norm": 0.2845953404903412,
      "kl": 0.006932258605957031,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0003,
      "reward": 0.02811681106686592,
      "reward_std": 0.4322117939591408,
      "rewards/cosine_scaled_reward": -0.2192363552749157,
      "rewards/format_reward": 0.7708333432674408,
      "step": 220
    },
    {
      "completion_length": 1456.3958740234375,
      "epoch": 0.25257142857142856,
      "grad_norm": 0.22674359381198883,
      "kl": 0.0051517486572265625,
      "learning_rate": 7.156560487081051e-07,
      "loss": 0.0002,
      "reward": 0.47192759346216917,
      "reward_std": 0.6820324845612049,
      "rewards/cosine_scaled_reward": 0.045280902180820704,
      "rewards/format_reward": 0.8125000074505806,
      "step": 221
    },
    {
      "completion_length": 1963.2917175292969,
      "epoch": 0.2537142857142857,
      "grad_norm": 0.21307772397994995,
      "kl": 0.005227088928222656,
      "learning_rate": 7.127310565369415e-07,
      "loss": 0.0002,
      "reward": 0.42855058796703815,
      "reward_std": 0.7586027830839157,
      "rewards/cosine_scaled_reward": 0.055503834038972855,
      "rewards/format_reward": 0.7291666828095913,
      "step": 222
    },
    {
      "completion_length": 1937.0208740234375,
      "epoch": 0.25485714285714284,
      "grad_norm": 0.26616960763931274,
      "kl": 0.0052585601806640625,
      "learning_rate": 7.097981330836616e-07,
      "loss": 0.0002,
      "reward": 0.2477805533562787,
      "reward_std": 0.5524286925792694,
      "rewards/cosine_scaled_reward": -0.013783978298306465,
      "rewards/format_reward": 0.6458333376795053,
      "step": 223
    },
    {
      "completion_length": 1899.812515258789,
      "epoch": 0.256,
      "grad_norm": 0.18020717799663544,
      "kl": 0.00457000732421875,
      "learning_rate": 7.068574212948169e-07,
      "loss": 0.0002,
      "reward": 0.4362371205352247,
      "reward_std": 0.8643104657530785,
      "rewards/cosine_scaled_reward": -0.009671762585639954,
      "rewards/format_reward": 0.8541666716337204,
      "step": 224
    },
    {
      "completion_length": 2635.125045776367,
      "epoch": 0.2571428571428571,
      "grad_norm": 0.2361401468515396,
      "kl": 0.011203765869140625,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0004,
      "reward": 0.1398951131850481,
      "reward_std": 0.7698212433606386,
      "rewards/cosine_scaled_reward": -0.0572519232518971,
      "rewards/format_reward": 0.5625,
      "step": 225
    },
    {
      "completion_length": 1900.4375534057617,
      "epoch": 0.2582857142857143,
      "grad_norm": 0.21450620889663696,
      "kl": 0.0057201385498046875,
      "learning_rate": 7.009532063876148e-07,
      "loss": 0.0002,
      "reward": 0.5856498526409268,
      "reward_std": 0.7293794807046652,
      "rewards/cosine_scaled_reward": 0.13076626230031252,
      "rewards/format_reward": 0.7916666734963655,
      "step": 226
    },
    {
      "completion_length": 1369.208366394043,
      "epoch": 0.25942857142857145,
      "grad_norm": 0.323088675737381,
      "kl": 0.008134841918945312,
      "learning_rate": 6.979899910323624e-07,
      "loss": 0.0003,
      "reward": 0.4275068351998925,
      "reward_std": 0.8954740911722183,
      "rewards/cosine_scaled_reward": -0.019443090073764324,
      "rewards/format_reward": 0.8541666679084301,
      "step": 227
    },
    {
      "completion_length": 1602.8750343322754,
      "epoch": 0.26057142857142856,
      "grad_norm": 0.2612713873386383,
      "kl": 0.0060291290283203125,
      "learning_rate": 6.950195628537299e-07,
      "loss": 0.0002,
      "reward": 0.46579291112720966,
      "reward_std": 0.7513364851474762,
      "rewards/cosine_scaled_reward": 0.05927048996090889,
      "rewards/format_reward": 0.7708333432674408,
      "step": 228
    },
    {
      "completion_length": 1817.6041870117188,
      "epoch": 0.26171428571428573,
      "grad_norm": 0.2555846869945526,
      "kl": 0.0076618194580078125,
      "learning_rate": 6.920420666261961e-07,
      "loss": 0.0003,
      "reward": 0.22703039785847068,
      "reward_std": 0.6141838692128658,
      "rewards/cosine_scaled_reward": -0.08688993845134974,
      "rewards/format_reward": 0.750000013038516,
      "step": 229
    },
    {
      "completion_length": 1947.083396911621,
      "epoch": 0.26285714285714284,
      "grad_norm": 0.22352339327335358,
      "kl": 0.0059909820556640625,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.0002,
      "reward": 0.030418872833251953,
      "reward_std": 0.5589127205312252,
      "rewards/cosine_scaled_reward": -0.2263335685711354,
      "rewards/format_reward": 0.7708333432674408,
      "step": 230
    },
    {
      "completion_length": 1575.8125381469727,
      "epoch": 0.264,
      "grad_norm": 0.24202732741832733,
      "kl": 0.0063457489013671875,
      "learning_rate": 6.860664508377001e-07,
      "loss": 0.0003,
      "reward": 0.5021250182762742,
      "reward_std": 0.6612259335815907,
      "rewards/cosine_scaled_reward": 0.03393824491649866,
      "rewards/format_reward": 0.8750000111758709,
      "step": 231
    },
    {
      "completion_length": 2361.145881652832,
      "epoch": 0.2651428571428571,
      "grad_norm": 0.28824689984321594,
      "kl": 0.008266448974609375,
      "learning_rate": 6.83068622519821e-07,
      "loss": 0.0003,
      "reward": -0.13501371257007122,
      "reward_std": 0.600494496524334,
      "rewards/cosine_scaled_reward": -0.24766290560364723,
      "rewards/format_reward": 0.5833333376795053,
      "step": 232
    },
    {
      "completion_length": 1480.9167022705078,
      "epoch": 0.2662857142857143,
      "grad_norm": 0.32725366950035095,
      "kl": 0.006622314453125,
      "learning_rate": 6.800643086250121e-07,
      "loss": 0.0003,
      "reward": 0.22992298612371087,
      "reward_std": 0.7725229002535343,
      "rewards/cosine_scaled_reward": -0.15780717965390068,
      "rewards/format_reward": 0.8750000149011612,
      "step": 233
    },
    {
      "completion_length": 1913.5208854675293,
      "epoch": 0.2674285714285714,
      "grad_norm": 0.27367836236953735,
      "kl": 0.008426666259765625,
      "learning_rate": 6.770536555792944e-07,
      "loss": 0.0003,
      "reward": 0.17465345282107592,
      "reward_std": 0.7128382474184036,
      "rewards/cosine_scaled_reward": -0.08376146724913269,
      "rewards/format_reward": 0.6666666716337204,
      "step": 234
    },
    {
      "completion_length": 1466.6458892822266,
      "epoch": 0.26857142857142857,
      "grad_norm": 0.25469592213630676,
      "kl": 0.00775909423828125,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0003,
      "reward": 0.6011685915291309,
      "reward_std": 0.7442786023020744,
      "rewards/cosine_scaled_reward": 0.11913689319044352,
      "rewards/format_reward": 0.833333333954215,
      "step": 235
    },
    {
      "completion_length": 1939.0417175292969,
      "epoch": 0.26971428571428574,
      "grad_norm": 0.23096546530723572,
      "kl": 0.0063419342041015625,
      "learning_rate": 6.710139192768694e-07,
      "loss": 0.0003,
      "reward": 0.33170187287032604,
      "reward_std": 0.7268010787665844,
      "rewards/cosine_scaled_reward": -0.018086417112499475,
      "rewards/format_reward": 0.7500000018626451,
      "step": 236
    },
    {
      "completion_length": 1496.2083740234375,
      "epoch": 0.27085714285714285,
      "grad_norm": 0.25223881006240845,
      "kl": 0.0052013397216796875,
      "learning_rate": 6.679851303883891e-07,
      "loss": 0.0002,
      "reward": 0.5692060198634863,
      "reward_std": 0.5697116628289223,
      "rewards/cosine_scaled_reward": 0.09943875670433044,
      "rewards/format_reward": 0.8541666716337204,
      "step": 237
    },
    {
      "completion_length": 1303.5625228881836,
      "epoch": 0.272,
      "grad_norm": 0.23987644910812378,
      "kl": 0.00640106201171875,
      "learning_rate": 6.649505910711058e-07,
      "loss": 0.0003,
      "reward": 0.6454257536679506,
      "reward_std": 0.8430968932807446,
      "rewards/cosine_scaled_reward": 0.09218539297580719,
      "rewards/format_reward": 0.9375000074505806,
      "step": 238
    },
    {
      "completion_length": 1533.0833473205566,
      "epoch": 0.27314285714285713,
      "grad_norm": 0.22352874279022217,
      "kl": 0.004787445068359375,
      "learning_rate": 6.619104492241847e-07,
      "loss": 0.0002,
      "reward": 0.7365382118150592,
      "reward_std": 0.6625222954899073,
      "rewards/cosine_scaled_reward": 0.24717165902256966,
      "rewards/format_reward": 0.770833333954215,
      "step": 239
    },
    {
      "completion_length": 1882.8125381469727,
      "epoch": 0.2742857142857143,
      "grad_norm": 0.35037240386009216,
      "kl": 0.010135650634765625,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0004,
      "reward": 0.04069505538791418,
      "reward_std": 0.6465389877557755,
      "rewards/cosine_scaled_reward": -0.16988872209913097,
      "rewards/format_reward": 0.6666666772216558,
      "step": 240
    },
    {
      "completion_length": 2043.1667175292969,
      "epoch": 0.2754285714285714,
      "grad_norm": 0.2375987321138382,
      "kl": 0.009412765502929688,
      "learning_rate": 6.558139508961654e-07,
      "loss": 0.0004,
      "reward": -0.024178337305784225,
      "reward_std": 0.5009024143218994,
      "rewards/cosine_scaled_reward": -0.20676567568443716,
      "rewards/format_reward": 0.6666666753590107,
      "step": 241
    },
    {
      "completion_length": 1260.9792022705078,
      "epoch": 0.2765714285714286,
      "grad_norm": 0.29835185408592224,
      "kl": 0.010770797729492188,
      "learning_rate": 6.527578915497951e-07,
      "loss": 0.0004,
      "reward": 0.3275939063169062,
      "reward_std": 0.5498746670782566,
      "rewards/cosine_scaled_reward": -0.09544416703283787,
      "rewards/format_reward": 0.9166666716337204,
      "step": 242
    },
    {
      "completion_length": 1774.5000534057617,
      "epoch": 0.2777142857142857,
      "grad_norm": 0.19600969552993774,
      "kl": 0.0063343048095703125,
      "learning_rate": 6.496968239287603e-07,
      "loss": 0.0003,
      "reward": 0.43449581041932106,
      "reward_std": 0.7577872760593891,
      "rewards/cosine_scaled_reward": 0.026275813579559326,
      "rewards/format_reward": 0.7916666716337204,
      "step": 243
    },
    {
      "completion_length": 1623.8750457763672,
      "epoch": 0.27885714285714286,
      "grad_norm": 0.22528576850891113,
      "kl": 0.0068035125732421875,
      "learning_rate": 6.466308972251785e-07,
      "loss": 0.0003,
      "reward": 0.7132167363015469,
      "reward_std": 0.7599900439381599,
      "rewards/cosine_scaled_reward": 0.18282007612287998,
      "rewards/format_reward": 0.8541666679084301,
      "step": 244
    },
    {
      "completion_length": 1842.708381652832,
      "epoch": 0.28,
      "grad_norm": 0.23914141952991486,
      "kl": 0.006076812744140625,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0002,
      "reward": 0.589666100917384,
      "reward_std": 0.911857221275568,
      "rewards/cosine_scaled_reward": 0.11452391929924488,
      "rewards/format_reward": 0.8125000111758709,
      "step": 245
    },
    {
      "completion_length": 1442.0625457763672,
      "epoch": 0.28114285714285714,
      "grad_norm": 0.22671173512935638,
      "kl": 0.007045745849609375,
      "learning_rate": 6.404850645156841e-07,
      "loss": 0.0003,
      "reward": 0.3358262628316879,
      "reward_std": 0.5915343575179577,
      "rewards/cosine_scaled_reward": -0.07317247241735458,
      "rewards/format_reward": 0.8750000037252903,
      "step": 246
    },
    {
      "completion_length": 2208.770866394043,
      "epoch": 0.2822857142857143,
      "grad_norm": 0.29410502314567566,
      "kl": 0.010019302368164062,
      "learning_rate": 6.374054580489873e-07,
      "loss": 0.0004,
      "reward": -0.014087029732763767,
      "reward_std": 0.6953110322356224,
      "rewards/cosine_scaled_reward": -0.19972090609371662,
      "rewards/format_reward": 0.6458333469927311,
      "step": 247
    },
    {
      "completion_length": 1561.9583740234375,
      "epoch": 0.2834285714285714,
      "grad_norm": 0.2551148533821106,
      "kl": 0.007082939147949219,
      "learning_rate": 6.343215915635761e-07,
      "loss": 0.0003,
      "reward": 0.7867583259940147,
      "reward_std": 0.7739764004945755,
      "rewards/cosine_scaled_reward": 0.28318586223758757,
      "rewards/format_reward": 0.7500000055879354,
      "step": 248
    },
    {
      "completion_length": 1342.2917022705078,
      "epoch": 0.2845714285714286,
      "grad_norm": 0.2254449427127838,
      "kl": 0.00762939453125,
      "learning_rate": 6.31233615362752e-07,
      "loss": 0.0003,
      "reward": 0.7728391233831644,
      "reward_std": 0.6931521892547607,
      "rewards/cosine_scaled_reward": 0.20786779932677746,
      "rewards/format_reward": 0.8958333358168602,
      "step": 249
    },
    {
      "completion_length": 1144.3750305175781,
      "epoch": 0.2857142857142857,
      "grad_norm": 0.4075692892074585,
      "kl": 0.008514404296875,
      "learning_rate": 6.281416799501187e-07,
      "loss": 0.0003,
      "reward": 0.4015044257976115,
      "reward_std": 0.511387325823307,
      "rewards/cosine_scaled_reward": -0.07808645971817896,
      "rewards/format_reward": 0.9791666716337204,
      "step": 250
    },
    {
      "completion_length": 1381.1459045410156,
      "epoch": 0.28685714285714287,
      "grad_norm": 0.32429951429367065,
      "kl": 0.0106201171875,
      "learning_rate": 6.25045936022246e-07,
      "loss": 0.0004,
      "reward": 0.26284962613135576,
      "reward_std": 0.8155036717653275,
      "rewards/cosine_scaled_reward": -0.09845332545228302,
      "rewards/format_reward": 0.7916666679084301,
      "step": 251
    },
    {
      "completion_length": 1654.3750457763672,
      "epoch": 0.288,
      "grad_norm": 0.2845689356327057,
      "kl": 0.009103775024414062,
      "learning_rate": 6.219465344613258e-07,
      "loss": 0.0004,
      "reward": 0.05027025658637285,
      "reward_std": 0.4697149991989136,
      "rewards/cosine_scaled_reward": -0.1850506253540516,
      "rewards/format_reward": 0.7291666753590107,
      "step": 252
    },
    {
      "completion_length": 1660.5208740234375,
      "epoch": 0.28914285714285715,
      "grad_norm": 0.2830834984779358,
      "kl": 0.010486602783203125,
      "learning_rate": 6.188436263278172e-07,
      "loss": 0.0004,
      "reward": 0.18601116666104645,
      "reward_std": 0.66238809004426,
      "rewards/cosine_scaled_reward": -0.1478569945320487,
      "rewards/format_reward": 0.8125000074505806,
      "step": 253
    },
    {
      "completion_length": 1486.208339691162,
      "epoch": 0.29028571428571426,
      "grad_norm": 0.2770550847053528,
      "kl": 0.008548736572265625,
      "learning_rate": 6.157373628530852e-07,
      "loss": 0.0003,
      "reward": 0.3256164574995637,
      "reward_std": 0.72544976323843,
      "rewards/cosine_scaled_reward": -0.07894822582602501,
      "rewards/format_reward": 0.8541666753590107,
      "step": 254
    },
    {
      "completion_length": 2314.5416946411133,
      "epoch": 0.2914285714285714,
      "grad_norm": 0.32598185539245605,
      "kl": 0.01126861572265625,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0005,
      "reward": -0.04141565319150686,
      "reward_std": 0.7138533964753151,
      "rewards/cosine_scaled_reward": -0.21306942123919725,
      "rewards/format_reward": 0.6250000149011612,
      "step": 255
    },
    {
      "completion_length": 1553.5208740234375,
      "epoch": 0.2925714285714286,
      "grad_norm": 0.2590584456920624,
      "kl": 0.008270263671875,
      "learning_rate": 6.095153756157051e-07,
      "loss": 0.0003,
      "reward": 0.5183277567848563,
      "reward_std": 0.539212403818965,
      "rewards/cosine_scaled_reward": 0.05446232855319977,
      "rewards/format_reward": 0.8750000111758709,
      "step": 256
    },
    {
      "completion_length": 1969.1042175292969,
      "epoch": 0.2937142857142857,
      "grad_norm": 0.2547340989112854,
      "kl": 0.007389068603515625,
      "learning_rate": 6.06399955103937e-07,
      "loss": 0.0003,
      "reward": 0.6981238089501858,
      "reward_std": 0.957792617380619,
      "rewards/cosine_scaled_reward": 0.20680510997772217,
      "rewards/format_reward": 0.7708333414047956,
      "step": 257
    },
    {
      "completion_length": 1860.1042175292969,
      "epoch": 0.2948571428571429,
      "grad_norm": 0.2526947855949402,
      "kl": 0.006866455078125,
      "learning_rate": 6.032817857379256e-07,
      "loss": 0.0003,
      "reward": 0.3261422934010625,
      "reward_std": 0.8735288791358471,
      "rewards/cosine_scaled_reward": -0.07447653356939554,
      "rewards/format_reward": 0.8333333395421505,
      "step": 258
    },
    {
      "completion_length": 1497.6667137145996,
      "epoch": 0.296,
      "grad_norm": 0.2954126298427582,
      "kl": 0.009616851806640625,
      "learning_rate": 6.001610194928464e-07,
      "loss": 0.0004,
      "reward": 0.4608106706291437,
      "reward_std": 0.6903665885329247,
      "rewards/cosine_scaled_reward": 0.035531939938664436,
      "rewards/format_reward": 0.8125000186264515,
      "step": 259
    },
    {
      "completion_length": 1280.8958778381348,
      "epoch": 0.29714285714285715,
      "grad_norm": 0.2724701166152954,
      "kl": 0.0075778961181640625,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0003,
      "reward": 0.6839517981279641,
      "reward_std": 0.8328232653439045,
      "rewards/cosine_scaled_reward": 0.13848848675843328,
      "rewards/format_reward": 0.8958333358168602,
      "step": 260
    },
    {
      "completion_length": 2193.687515258789,
      "epoch": 0.29828571428571427,
      "grad_norm": 0.18936224281787872,
      "kl": 0.00774383544921875,
      "learning_rate": 5.939123048916173e-07,
      "loss": 0.0003,
      "reward": -0.03597256541252136,
      "reward_std": 0.5031098667532206,
      "rewards/cosine_scaled_reward": -0.18370476551353931,
      "rewards/format_reward": 0.6041666716337204,
      "step": 261
    },
    {
      "completion_length": 1862.2083740234375,
      "epoch": 0.29942857142857143,
      "grad_norm": 0.2641579806804657,
      "kl": 0.0114593505859375,
      "learning_rate": 5.907846610890011e-07,
      "loss": 0.0005,
      "reward": 0.04227437451481819,
      "reward_std": 0.5424713045358658,
      "rewards/cosine_scaled_reward": -0.1736736847087741,
      "rewards/format_reward": 0.6875000037252903,
      "step": 262
    },
    {
      "completion_length": 1353.3125228881836,
      "epoch": 0.30057142857142854,
      "grad_norm": 0.2091461718082428,
      "kl": 0.00551605224609375,
      "learning_rate": 5.87655029499542e-07,
      "loss": 0.0002,
      "reward": 0.29918272816576064,
      "reward_std": 0.7759961858391762,
      "rewards/cosine_scaled_reward": -0.13049082271754742,
      "rewards/format_reward": 0.9166666716337204,
      "step": 263
    },
    {
      "completion_length": 1441.7083740234375,
      "epoch": 0.3017142857142857,
      "grad_norm": 0.2494620829820633,
      "kl": 0.0072498321533203125,
      "learning_rate": 5.845235626570683e-07,
      "loss": 0.0003,
      "reward": 0.2639927687123418,
      "reward_std": 0.689430944621563,
      "rewards/cosine_scaled_reward": -0.11652671941556036,
      "rewards/format_reward": 0.8541666772216558,
      "step": 264
    },
    {
      "completion_length": 1564.6250457763672,
      "epoch": 0.3028571428571429,
      "grad_norm": 0.2662133276462555,
      "kl": 0.010656356811523438,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.0004,
      "reward": 0.44831580482423306,
      "reward_std": 0.7113944664597511,
      "rewards/cosine_scaled_reward": -0.0021858818363398314,
      "rewards/format_reward": 0.8750000055879354,
      "step": 265
    },
    {
      "completion_length": 1647.4375381469727,
      "epoch": 0.304,
      "grad_norm": 0.2215126007795334,
      "kl": 0.00804901123046875,
      "learning_rate": 5.78255733788191e-07,
      "loss": 0.0003,
      "reward": 0.1729487591655925,
      "reward_std": 0.6001881808042526,
      "rewards/cosine_scaled_reward": -0.10871189273893833,
      "rewards/format_reward": 0.7291666865348816,
      "step": 266
    },
    {
      "completion_length": 2309.1458740234375,
      "epoch": 0.30514285714285716,
      "grad_norm": 0.33464720845222473,
      "kl": 0.013841629028320312,
      "learning_rate": 5.751196772469237e-07,
      "loss": 0.0006,
      "reward": -0.04994155094027519,
      "reward_std": 0.63846031203866,
      "rewards/cosine_scaled_reward": -0.18117934837937355,
      "rewards/format_reward": 0.5625000111758709,
      "step": 267
    },
    {
      "completion_length": 1347.291732788086,
      "epoch": 0.3062857142857143,
      "grad_norm": 0.2725054919719696,
      "kl": 0.011320114135742188,
      "learning_rate": 5.71982396408026e-07,
      "loss": 0.0005,
      "reward": 0.3638640786521137,
      "reward_std": 0.856957983225584,
      "rewards/cosine_scaled_reward": -0.07128076790831983,
      "rewards/format_reward": 0.8750000149011612,
      "step": 268
    },
    {
      "completion_length": 1536.8541870117188,
      "epoch": 0.30742857142857144,
      "grad_norm": 0.242116779088974,
      "kl": 0.008541107177734375,
      "learning_rate": 5.688440441781398e-07,
      "loss": 0.0003,
      "reward": 0.2962089798747911,
      "reward_std": 0.652573972940445,
      "rewards/cosine_scaled_reward": -0.07353247702121735,
      "rewards/format_reward": 0.8125000149011612,
      "step": 269
    },
    {
      "completion_length": 1723.6875305175781,
      "epoch": 0.30857142857142855,
      "grad_norm": 0.20673726499080658,
      "kl": 0.00959014892578125,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0004,
      "reward": 0.6585689373314381,
      "reward_std": 0.8569062799215317,
      "rewards/cosine_scaled_reward": 0.13866457249969244,
      "rewards/format_reward": 0.854166679084301,
      "step": 270
    },
    {
      "completion_length": 1226.4375534057617,
      "epoch": 0.3097142857142857,
      "grad_norm": 0.3247756063938141,
      "kl": 0.009336471557617188,
      "learning_rate": 5.625647374256061e-07,
      "loss": 0.0004,
      "reward": 0.6650833152234554,
      "reward_std": 0.7789544351398945,
      "rewards/cosine_scaled_reward": 0.11849892261670902,
      "rewards/format_reward": 0.9166666679084301,
      "step": 271
    },
    {
      "completion_length": 1959.395896911621,
      "epoch": 0.31085714285714283,
      "grad_norm": 0.2230900079011917,
      "kl": 0.0106964111328125,
      "learning_rate": 5.594240889475106e-07,
      "loss": 0.0004,
      "reward": 0.23697172570973635,
      "reward_std": 0.7450396865606308,
      "rewards/cosine_scaled_reward": -0.0959045309573412,
      "rewards/format_reward": 0.7708333507180214,
      "step": 272
    },
    {
      "completion_length": 1628.1042175292969,
      "epoch": 0.312,
      "grad_norm": 0.3747721314430237,
      "kl": 0.01097869873046875,
      "learning_rate": 5.562829811526154e-07,
      "loss": 0.0004,
      "reward": 0.4147872976027429,
      "reward_std": 0.6467026993632317,
      "rewards/cosine_scaled_reward": 0.0440314169973135,
      "rewards/format_reward": 0.7500000149011612,
      "step": 273
    },
    {
      "completion_length": 1172.4375457763672,
      "epoch": 0.31314285714285717,
      "grad_norm": 0.2670040726661682,
      "kl": 0.008930206298828125,
      "learning_rate": 5.531415671340826e-07,
      "loss": 0.0004,
      "reward": 0.6682712404581252,
      "reward_std": 0.8026308417320251,
      "rewards/cosine_scaled_reward": 0.0984476669691503,
      "rewards/format_reward": 0.9583333432674408,
      "step": 274
    },
    {
      "completion_length": 1716.9375534057617,
      "epoch": 0.3142857142857143,
      "grad_norm": 0.2296976000070572,
      "kl": 0.00942230224609375,
      "learning_rate": 5.5e-07,
      "loss": 0.0004,
      "reward": 0.6355814579874277,
      "reward_std": 0.902884915471077,
      "rewards/cosine_scaled_reward": 0.1562671698629856,
      "rewards/format_reward": 0.791666679084301,
      "step": 275
    },
    {
      "completion_length": 1406.5625457763672,
      "epoch": 0.31542857142857145,
      "grad_norm": 0.27804121375083923,
      "kl": 0.012622833251953125,
      "learning_rate": 5.468584328659172e-07,
      "loss": 0.0005,
      "reward": 0.47063780203461647,
      "reward_std": 0.826964907348156,
      "rewards/cosine_scaled_reward": 0.035029259510338306,
      "rewards/format_reward": 0.8125000055879354,
      "step": 276
    },
    {
      "completion_length": 1531.7708892822266,
      "epoch": 0.31657142857142856,
      "grad_norm": 0.3786030411720276,
      "kl": 0.0130615234375,
      "learning_rate": 5.437170188473847e-07,
      "loss": 0.0005,
      "reward": 0.4646074064075947,
      "reward_std": 0.7622004933655262,
      "rewards/cosine_scaled_reward": 0.04781521949917078,
      "rewards/format_reward": 0.7916666734963655,
      "step": 277
    },
    {
      "completion_length": 1472.6458740234375,
      "epoch": 0.3177142857142857,
      "grad_norm": 0.29159483313560486,
      "kl": 0.012044906616210938,
      "learning_rate": 5.405759110524894e-07,
      "loss": 0.0005,
      "reward": 0.5979791600257158,
      "reward_std": 0.6016073673963547,
      "rewards/cosine_scaled_reward": 0.13445703126490116,
      "rewards/format_reward": 0.8125000186264515,
      "step": 278
    },
    {
      "completion_length": 1808.8750457763672,
      "epoch": 0.31885714285714284,
      "grad_norm": 0.45572957396507263,
      "kl": 0.0137481689453125,
      "learning_rate": 5.37435262574394e-07,
      "loss": 0.0006,
      "reward": 0.3207678751787171,
      "reward_std": 0.7218646891415119,
      "rewards/cosine_scaled_reward": -0.07327653095126152,
      "rewards/format_reward": 0.854166679084301,
      "step": 279
    },
    {
      "completion_length": 1779.0000457763672,
      "epoch": 0.32,
      "grad_norm": 0.3021228611469269,
      "kl": 0.011760711669921875,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0005,
      "reward": 0.8133369982242584,
      "reward_std": 0.8129135742783546,
      "rewards/cosine_scaled_reward": 0.2262397282756865,
      "rewards/format_reward": 0.8958333395421505,
      "step": 280
    },
    {
      "completion_length": 2400.2708740234375,
      "epoch": 0.3211428571428571,
      "grad_norm": 0.2547665238380432,
      "kl": 0.012500762939453125,
      "learning_rate": 5.311559558218603e-07,
      "loss": 0.0005,
      "reward": -0.10774591006338596,
      "reward_std": 0.6432830318808556,
      "rewards/cosine_scaled_reward": -0.22082971967756748,
      "rewards/format_reward": 0.5625000055879354,
      "step": 281
    },
    {
      "completion_length": 1696.4792251586914,
      "epoch": 0.3222857142857143,
      "grad_norm": 0.3483729362487793,
      "kl": 0.013248443603515625,
      "learning_rate": 5.28017603591974e-07,
      "loss": 0.0005,
      "reward": 0.4830823950469494,
      "reward_std": 0.720955528318882,
      "rewards/cosine_scaled_reward": 0.053989187348634005,
      "rewards/format_reward": 0.8125000149011612,
      "step": 282
    },
    {
      "completion_length": 2184.6250534057617,
      "epoch": 0.32342857142857145,
      "grad_norm": 0.26255351305007935,
      "kl": 0.01146697998046875,
      "learning_rate": 5.248803227530763e-07,
      "loss": 0.0005,
      "reward": 0.6261483291164041,
      "reward_std": 0.8373686634004116,
      "rewards/cosine_scaled_reward": 0.18944203667342663,
      "rewards/format_reward": 0.7291666772216558,
      "step": 283
    },
    {
      "completion_length": 1453.270881652832,
      "epoch": 0.32457142857142857,
      "grad_norm": 0.26391106843948364,
      "kl": 0.0076885223388671875,
      "learning_rate": 5.21744266211809e-07,
      "loss": 0.0003,
      "reward": 0.14405585872009397,
      "reward_std": 0.5576095655560493,
      "rewards/cosine_scaled_reward": -0.18134936597198248,
      "rewards/format_reward": 0.8333333432674408,
      "step": 284
    },
    {
      "completion_length": 1164.7291946411133,
      "epoch": 0.32571428571428573,
      "grad_norm": 0.40485504269599915,
      "kl": 0.0128326416015625,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0005,
      "reward": 0.3652733010239899,
      "reward_std": 0.7800908386707306,
      "rewards/cosine_scaled_reward": -0.04528142110211775,
      "rewards/format_reward": 0.8333333358168602,
      "step": 285
    },
    {
      "completion_length": 1260.0208587646484,
      "epoch": 0.32685714285714285,
      "grad_norm": 0.26052573323249817,
      "kl": 0.008724212646484375,
      "learning_rate": 5.154764373429315e-07,
      "loss": 0.0003,
      "reward": 0.4141525523737073,
      "reward_std": 0.6447485722601414,
      "rewards/cosine_scaled_reward": -0.043887258507311344,
      "rewards/format_reward": 0.9166666679084301,
      "step": 286
    },
    {
      "completion_length": 1336.2083702087402,
      "epoch": 0.328,
      "grad_norm": 0.3096228837966919,
      "kl": 0.012447357177734375,
      "learning_rate": 5.123449705004581e-07,
      "loss": 0.0005,
      "reward": 0.5274541154503822,
      "reward_std": 0.7009152993559837,
      "rewards/cosine_scaled_reward": 0.121887655579485,
      "rewards/format_reward": 0.7291666679084301,
      "step": 287
    },
    {
      "completion_length": 1846.3750228881836,
      "epoch": 0.3291428571428571,
      "grad_norm": 0.28548097610473633,
      "kl": 0.01815032958984375,
      "learning_rate": 5.09215338910999e-07,
      "loss": 0.0007,
      "reward": 0.12442206963896751,
      "reward_std": 0.5581525340676308,
      "rewards/cosine_scaled_reward": -0.1310774045996368,
      "rewards/format_reward": 0.7083333395421505,
      "step": 288
    },
    {
      "completion_length": 1571.437557220459,
      "epoch": 0.3302857142857143,
      "grad_norm": 0.3936711251735687,
      "kl": 0.012668609619140625,
      "learning_rate": 5.060876951083828e-07,
      "loss": 0.0005,
      "reward": 0.4288631723029539,
      "reward_std": 0.593647625297308,
      "rewards/cosine_scaled_reward": 0.030754741048440337,
      "rewards/format_reward": 0.7916666679084301,
      "step": 289
    },
    {
      "completion_length": 898.5833473205566,
      "epoch": 0.3314285714285714,
      "grad_norm": 0.32913920283317566,
      "kl": 0.009197235107421875,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.0004,
      "reward": 0.7029262520372868,
      "reward_std": 0.8086423352360725,
      "rewards/cosine_scaled_reward": 0.12271453440189362,
      "rewards/format_reward": 0.9583333432674408,
      "step": 290
    },
    {
      "completion_length": 1254.2917022705078,
      "epoch": 0.3325714285714286,
      "grad_norm": 0.24602723121643066,
      "kl": 0.010408401489257812,
      "learning_rate": 4.998389805071536e-07,
      "loss": 0.0004,
      "reward": 0.44253411889076233,
      "reward_std": 0.8852041102945805,
      "rewards/cosine_scaled_reward": -0.05237848265096545,
      "rewards/format_reward": 0.9375,
      "step": 291
    },
    {
      "completion_length": 1754.5000610351562,
      "epoch": 0.33371428571428574,
      "grad_norm": 0.24916251003742218,
      "kl": 0.015148162841796875,
      "learning_rate": 4.967182142620745e-07,
      "loss": 0.0006,
      "reward": 0.15450574783608317,
      "reward_std": 0.5213648546487093,
      "rewards/cosine_scaled_reward": -0.1516056777909398,
      "rewards/format_reward": 0.7916666772216558,
      "step": 292
    },
    {
      "completion_length": 1034.2917022705078,
      "epoch": 0.33485714285714285,
      "grad_norm": 0.31137824058532715,
      "kl": 0.012929916381835938,
      "learning_rate": 4.93600044896063e-07,
      "loss": 0.0005,
      "reward": 0.4599906969233416,
      "reward_std": 0.5716858878731728,
      "rewards/cosine_scaled_reward": -0.009167976677417755,
      "rewards/format_reward": 0.916666679084301,
      "step": 293
    },
    {
      "completion_length": 1682.0208740234375,
      "epoch": 0.336,
      "grad_norm": 0.27208465337753296,
      "kl": 0.013782501220703125,
      "learning_rate": 4.904846243842949e-07,
      "loss": 0.0006,
      "reward": 0.3077076869085431,
      "reward_std": 0.777203194797039,
      "rewards/cosine_scaled_reward": -0.05544556397944689,
      "rewards/format_reward": 0.7708333432674408,
      "step": 294
    },
    {
      "completion_length": 1438.083381652832,
      "epoch": 0.33714285714285713,
      "grad_norm": 0.4438267946243286,
      "kl": 0.017040252685546875,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0007,
      "reward": 0.7139077642932534,
      "reward_std": 0.7298413254320621,
      "rewards/cosine_scaled_reward": 0.1472441926598549,
      "rewards/format_reward": 0.9375000074505806,
      "step": 295
    },
    {
      "completion_length": 1397.895896911621,
      "epoch": 0.3382857142857143,
      "grad_norm": 0.3188610076904297,
      "kl": 0.01441192626953125,
      "learning_rate": 4.842626371469149e-07,
      "loss": 0.0006,
      "reward": 0.47962956223636866,
      "reward_std": 0.759897030889988,
      "rewards/cosine_scaled_reward": 0.025067659094929695,
      "rewards/format_reward": 0.8541666716337204,
      "step": 296
    },
    {
      "completion_length": 2094.354217529297,
      "epoch": 0.3394285714285714,
      "grad_norm": 0.472922682762146,
      "kl": 0.0243682861328125,
      "learning_rate": 4.811563736721829e-07,
      "loss": 0.001,
      "reward": 0.12652458110824227,
      "reward_std": 0.6632986180484295,
      "rewards/cosine_scaled_reward": -0.115331269800663,
      "rewards/format_reward": 0.6666666809469461,
      "step": 297
    },
    {
      "completion_length": 1431.7292175292969,
      "epoch": 0.3405714285714286,
      "grad_norm": 0.2858993113040924,
      "kl": 0.013988494873046875,
      "learning_rate": 4.780534655386743e-07,
      "loss": 0.0006,
      "reward": 0.41945501090958714,
      "reward_std": 0.8443580865859985,
      "rewards/cosine_scaled_reward": -0.003992303041741252,
      "rewards/format_reward": 0.8125000149011612,
      "step": 298
    },
    {
      "completion_length": 1634.4167175292969,
      "epoch": 0.3417142857142857,
      "grad_norm": 0.403289258480072,
      "kl": 0.019683837890625,
      "learning_rate": 4.749540639777539e-07,
      "loss": 0.0008,
      "reward": 0.3505248324945569,
      "reward_std": 0.7320738956332207,
      "rewards/cosine_scaled_reward": -0.020624496042728424,
      "rewards/format_reward": 0.7708333469927311,
      "step": 299
    },
    {
      "completion_length": 1821.9166870117188,
      "epoch": 0.34285714285714286,
      "grad_norm": 0.3450154662132263,
      "kl": 0.0260009765625,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.001,
      "reward": 0.28817289136350155,
      "reward_std": 0.6358778662979603,
      "rewards/cosine_scaled_reward": -0.05276927351951599,
      "rewards/format_reward": 0.7708333358168602,
      "step": 300
    },
    {
      "completion_length": 1875.5000381469727,
      "epoch": 0.344,
      "grad_norm": 0.4964129328727722,
      "kl": 0.031585693359375,
      "learning_rate": 4.68766384637248e-07,
      "loss": 0.0013,
      "reward": 0.23697214853018522,
      "reward_std": 0.731932707130909,
      "rewards/cosine_scaled_reward": -0.10828239191323519,
      "rewards/format_reward": 0.7916666753590107,
      "step": 301
    },
    {
      "completion_length": 1755.0000343322754,
      "epoch": 0.34514285714285714,
      "grad_norm": 0.3546035587787628,
      "kl": 0.022808074951171875,
      "learning_rate": 4.656784084364238e-07,
      "loss": 0.0009,
      "reward": 0.3061390779912472,
      "reward_std": 0.6646707132458687,
      "rewards/cosine_scaled_reward": -0.003997504012659192,
      "rewards/format_reward": 0.6875,
      "step": 302
    },
    {
      "completion_length": 1310.7291984558105,
      "epoch": 0.3462857142857143,
      "grad_norm": 0.3501085937023163,
      "kl": 0.017917633056640625,
      "learning_rate": 4.6259454195101267e-07,
      "loss": 0.0007,
      "reward": 0.49688062351197004,
      "reward_std": 0.7856386080384254,
      "rewards/cosine_scaled_reward": 0.005726959556341171,
      "rewards/format_reward": 0.916666679084301,
      "step": 303
    },
    {
      "completion_length": 1531.625015258789,
      "epoch": 0.3474285714285714,
      "grad_norm": 0.23862285912036896,
      "kl": 0.0212249755859375,
      "learning_rate": 4.59514935484316e-07,
      "loss": 0.0008,
      "reward": 0.33517973372363485,
      "reward_std": 0.8556637056171894,
      "rewards/cosine_scaled_reward": -0.07236111164093018,
      "rewards/format_reward": 0.8333333432674408,
      "step": 304
    },
    {
      "completion_length": 1325.145851135254,
      "epoch": 0.3485714285714286,
      "grad_norm": 0.3753778338432312,
      "kl": 0.013553619384765625,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.0005,
      "reward": 0.1812261645682156,
      "reward_std": 0.7640487253665924,
      "rewards/cosine_scaled_reward": -0.18994974298402667,
      "rewards/format_reward": 0.8750000223517418,
      "step": 305
    },
    {
      "completion_length": 1495.1875610351562,
      "epoch": 0.3497142857142857,
      "grad_norm": 0.43624046444892883,
      "kl": 0.028331756591796875,
      "learning_rate": 4.5336910277482155e-07,
      "loss": 0.0011,
      "reward": 0.5158186480402946,
      "reward_std": 0.8463861420750618,
      "rewards/cosine_scaled_reward": 0.07386103633325547,
      "rewards/format_reward": 0.791666679084301,
      "step": 306
    },
    {
      "completion_length": 1391.8750228881836,
      "epoch": 0.35085714285714287,
      "grad_norm": 0.36552807688713074,
      "kl": 0.01470184326171875,
      "learning_rate": 4.503031760712397e-07,
      "loss": 0.0006,
      "reward": 0.4544885288923979,
      "reward_std": 0.9182650446891785,
      "rewards/cosine_scaled_reward": 0.03140606731176376,
      "rewards/format_reward": 0.7916666716337204,
      "step": 307
    },
    {
      "completion_length": 2439.062545776367,
      "epoch": 0.352,
      "grad_norm": 0.25556859374046326,
      "kl": 0.02754974365234375,
      "learning_rate": 4.4724210845020494e-07,
      "loss": 0.0011,
      "reward": 0.16432497836649418,
      "reward_std": 0.7285211831331253,
      "rewards/cosine_scaled_reward": -0.06424459861591458,
      "rewards/format_reward": 0.6041666734963655,
      "step": 308
    },
    {
      "completion_length": 2007.7709045410156,
      "epoch": 0.35314285714285715,
      "grad_norm": 0.2907513380050659,
      "kl": 0.02512359619140625,
      "learning_rate": 4.441860491038345e-07,
      "loss": 0.001,
      "reward": 0.3333761217072606,
      "reward_std": 0.9507267326116562,
      "rewards/cosine_scaled_reward": -0.06830872967839241,
      "rewards/format_reward": 0.8125000074505806,
      "step": 309
    },
    {
      "completion_length": 1377.9375610351562,
      "epoch": 0.35428571428571426,
      "grad_norm": 0.6299467086791992,
      "kl": 0.019756317138671875,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0008,
      "reward": 0.14710421487689018,
      "reward_std": 0.5830218307673931,
      "rewards/cosine_scaled_reward": -0.2127772723324597,
      "rewards/format_reward": 0.8958333358168602,
      "step": 310
    },
    {
      "completion_length": 1264.770851135254,
      "epoch": 0.3554285714285714,
      "grad_norm": 0.3672167658805847,
      "kl": 0.01552581787109375,
      "learning_rate": 4.3808955077581546e-07,
      "loss": 0.0006,
      "reward": 0.5835366472601891,
      "reward_std": 0.8306069150567055,
      "rewards/cosine_scaled_reward": 0.05884586926549673,
      "rewards/format_reward": 0.9166666716337204,
      "step": 311
    },
    {
      "completion_length": 1228.6875228881836,
      "epoch": 0.3565714285714286,
      "grad_norm": 0.31035158038139343,
      "kl": 0.01593017578125,
      "learning_rate": 4.350494089288943e-07,
      "loss": 0.0006,
      "reward": 0.7780788261443377,
      "reward_std": 0.725801732391119,
      "rewards/cosine_scaled_reward": 0.23832012061029673,
      "rewards/format_reward": 0.8333333432674408,
      "step": 312
    },
    {
      "completion_length": 2067.1666831970215,
      "epoch": 0.3577142857142857,
      "grad_norm": 0.3178723454475403,
      "kl": 0.03451347351074219,
      "learning_rate": 4.3201486961161093e-07,
      "loss": 0.0014,
      "reward": 0.3173049371689558,
      "reward_std": 0.6579355709254742,
      "rewards/cosine_scaled_reward": 0.03718515514628962,
      "rewards/format_reward": 0.6250000111758709,
      "step": 313
    },
    {
      "completion_length": 1668.3333892822266,
      "epoch": 0.3588571428571429,
      "grad_norm": 0.46666356921195984,
      "kl": 0.031169891357421875,
      "learning_rate": 4.2898608072313045e-07,
      "loss": 0.0012,
      "reward": 0.28569703502580523,
      "reward_std": 0.6306734308600426,
      "rewards/cosine_scaled_reward": -0.056163689121603966,
      "rewards/format_reward": 0.7708333395421505,
      "step": 314
    },
    {
      "completion_length": 2082.7084045410156,
      "epoch": 0.36,
      "grad_norm": 0.2961288392543793,
      "kl": 0.05887603759765625,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.0024,
      "reward": 0.4379968661814928,
      "reward_std": 0.8023070320487022,
      "rewards/cosine_scaled_reward": 0.04415438207797706,
      "rewards/format_reward": 0.7500000074505806,
      "step": 315
    },
    {
      "completion_length": 2361.541732788086,
      "epoch": 0.36114285714285715,
      "grad_norm": 0.477490097284317,
      "kl": 0.0577545166015625,
      "learning_rate": 4.2294634442070553e-07,
      "loss": 0.0023,
      "reward": -0.013460966947604902,
      "reward_std": 0.6273909620940685,
      "rewards/cosine_scaled_reward": -0.19738789275288582,
      "rewards/format_reward": 0.6458333488553762,
      "step": 316
    },
    {
      "completion_length": 1918.9375686645508,
      "epoch": 0.36228571428571427,
      "grad_norm": 0.6062735319137573,
      "kl": 0.03968238830566406,
      "learning_rate": 4.1993569137498776e-07,
      "loss": 0.0016,
      "reward": 0.2778178099542856,
      "reward_std": 0.8473985716700554,
      "rewards/cosine_scaled_reward": -0.024428293108940125,
      "rewards/format_reward": 0.6666666828095913,
      "step": 317
    },
    {
      "completion_length": 1197.4792098999023,
      "epoch": 0.36342857142857143,
      "grad_norm": 0.5727298855781555,
      "kl": 0.025470733642578125,
      "learning_rate": 4.1693137748017915e-07,
      "loss": 0.001,
      "reward": 0.35039830300956964,
      "reward_std": 0.665754821151495,
      "rewards/cosine_scaled_reward": -0.09761649183928967,
      "rewards/format_reward": 0.9375000149011612,
      "step": 318
    },
    {
      "completion_length": 1549.0416946411133,
      "epoch": 0.36457142857142855,
      "grad_norm": 0.44952714443206787,
      "kl": 0.022411346435546875,
      "learning_rate": 4.1393354916230005e-07,
      "loss": 0.0009,
      "reward": 0.16405144333839417,
      "reward_std": 0.7381913363933563,
      "rewards/cosine_scaled_reward": -0.17837723344564438,
      "rewards/format_reward": 0.8333333488553762,
      "step": 319
    },
    {
      "completion_length": 1096.270866394043,
      "epoch": 0.3657142857142857,
      "grad_norm": 0.9068484902381897,
      "kl": 0.045513153076171875,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0018,
      "reward": 0.4451125105842948,
      "reward_std": 0.8512261882424355,
      "rewards/cosine_scaled_reward": -0.02465624047908932,
      "rewards/format_reward": 0.8958333395421505,
      "step": 320
    },
    {
      "completion_length": 1107.2083778381348,
      "epoch": 0.3668571428571429,
      "grad_norm": 0.33380454778671265,
      "kl": 0.016986846923828125,
      "learning_rate": 4.079579333738039e-07,
      "loss": 0.0007,
      "reward": 0.7083983863703907,
      "reward_std": 0.741938479244709,
      "rewards/cosine_scaled_reward": 0.13879141584038734,
      "rewards/format_reward": 0.9375000074505806,
      "step": 321
    },
    {
      "completion_length": 1815.958366394043,
      "epoch": 0.368,
      "grad_norm": 0.5257051587104797,
      "kl": 0.06414031982421875,
      "learning_rate": 4.0498043714627006e-07,
      "loss": 0.0026,
      "reward": 0.2582624601200223,
      "reward_std": 0.6789763048291206,
      "rewards/cosine_scaled_reward": -0.0668359762057662,
      "rewards/format_reward": 0.7500000149011612,
      "step": 322
    },
    {
      "completion_length": 1977.5834045410156,
      "epoch": 0.36914285714285716,
      "grad_norm": 0.5315828323364258,
      "kl": 0.07048416137695312,
      "learning_rate": 4.020100089676376e-07,
      "loss": 0.0028,
      "reward": 0.3757035471498966,
      "reward_std": 0.8692245557904243,
      "rewards/cosine_scaled_reward": 0.05770140094682574,
      "rewards/format_reward": 0.6458333469927311,
      "step": 323
    },
    {
      "completion_length": 2044.8125457763672,
      "epoch": 0.3702857142857143,
      "grad_norm": 0.5173450708389282,
      "kl": 0.06734085083007812,
      "learning_rate": 3.9904679361238526e-07,
      "loss": 0.0027,
      "reward": 0.1749881288560573,
      "reward_std": 0.8538544028997421,
      "rewards/cosine_scaled_reward": -0.14555134577676654,
      "rewards/format_reward": 0.7708333507180214,
      "step": 324
    },
    {
      "completion_length": 1892.9375267028809,
      "epoch": 0.37142857142857144,
      "grad_norm": 0.6045531034469604,
      "kl": 0.0422515869140625,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0017,
      "reward": 0.32839928939938545,
      "reward_std": 0.8866756781935692,
      "rewards/cosine_scaled_reward": -0.03144947811961174,
      "rewards/format_reward": 0.7500000093132257,
      "step": 325
    },
    {
      "completion_length": 1686.708366394043,
      "epoch": 0.37257142857142855,
      "grad_norm": 0.485423743724823,
      "kl": 0.061676025390625,
      "learning_rate": 3.931425787051832e-07,
      "loss": 0.0025,
      "reward": 0.39623109018430114,
      "reward_std": 0.8446017913520336,
      "rewards/cosine_scaled_reward": 0.0048162119928747416,
      "rewards/format_reward": 0.7708333414047956,
      "step": 326
    },
    {
      "completion_length": 1453.6458435058594,
      "epoch": 0.3737142857142857,
      "grad_norm": 0.2550465166568756,
      "kl": 0.0176849365234375,
      "learning_rate": 3.902018669163384e-07,
      "loss": 0.0007,
      "reward": 0.6734898695722222,
      "reward_std": 0.814283449202776,
      "rewards/cosine_scaled_reward": 0.15649090707302094,
      "rewards/format_reward": 0.8541666716337204,
      "step": 327
    },
    {
      "completion_length": 1698.7500457763672,
      "epoch": 0.37485714285714283,
      "grad_norm": 0.561379611492157,
      "kl": 0.03546142578125,
      "learning_rate": 3.872689434630585e-07,
      "loss": 0.0014,
      "reward": 0.06343854777514935,
      "reward_std": 0.6165986470878124,
      "rewards/cosine_scaled_reward": -0.18727782554924488,
      "rewards/format_reward": 0.7291666828095913,
      "step": 328
    },
    {
      "completion_length": 971.5000152587891,
      "epoch": 0.376,
      "grad_norm": 0.45329251885414124,
      "kl": 0.024200439453125,
      "learning_rate": 3.843439512918949e-07,
      "loss": 0.001,
      "reward": 0.6886619143188,
      "reward_std": 0.8372432589530945,
      "rewards/cosine_scaled_reward": 0.10075276345014572,
      "rewards/format_reward": 0.9791666716337204,
      "step": 329
    },
    {
      "completion_length": 1391.3542213439941,
      "epoch": 0.37714285714285717,
      "grad_norm": 0.7288292646408081,
      "kl": 0.056179046630859375,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0022,
      "reward": 0.17569798463955522,
      "reward_std": 0.639821320772171,
      "rewards/cosine_scaled_reward": -0.15391897410154343,
      "rewards/format_reward": 0.8125000055879354,
      "step": 330
    },
    {
      "completion_length": 1995.6250457763672,
      "epoch": 0.3782857142857143,
      "grad_norm": 0.7102898359298706,
      "kl": 0.07641983032226562,
      "learning_rate": 3.785183306423767e-07,
      "loss": 0.0031,
      "reward": 0.17617637664079666,
      "reward_std": 0.7594183348119259,
      "rewards/cosine_scaled_reward": -0.056006991614822255,
      "rewards/format_reward": 0.6041666734963655,
      "step": 331
    },
    {
      "completion_length": 1507.145866394043,
      "epoch": 0.37942857142857145,
      "grad_norm": 0.6774265170097351,
      "kl": 0.041957855224609375,
      "learning_rate": 3.7561798609655373e-07,
      "loss": 0.0017,
      "reward": 0.15361519530415535,
      "reward_std": 0.7348885871469975,
      "rewards/cosine_scaled_reward": -0.1517366673797369,
      "rewards/format_reward": 0.7708333414047956,
      "step": 332
    },
    {
      "completion_length": 1263.06254196167,
      "epoch": 0.38057142857142856,
      "grad_norm": 0.40281206369400024,
      "kl": 0.034458160400390625,
      "learning_rate": 3.72726140684072e-07,
      "loss": 0.0014,
      "reward": 0.4631691016256809,
      "reward_std": 0.8592428974807262,
      "rewards/cosine_scaled_reward": -0.046608994947746396,
      "rewards/format_reward": 0.9583333432674408,
      "step": 333
    },
    {
      "completion_length": 2013.9583740234375,
      "epoch": 0.38171428571428573,
      "grad_norm": 0.7402753829956055,
      "kl": 0.076751708984375,
      "learning_rate": 3.6984293534939737e-07,
      "loss": 0.0031,
      "reward": 0.025389771908521652,
      "reward_std": 0.8407883942127228,
      "rewards/cosine_scaled_reward": -0.1560281114652753,
      "rewards/format_reward": 0.583333345130086,
      "step": 334
    },
    {
      "completion_length": 1345.2291946411133,
      "epoch": 0.38285714285714284,
      "grad_norm": 0.40346795320510864,
      "kl": 0.027713775634765625,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.0011,
      "reward": 0.43492193752899766,
      "reward_std": 0.6728620305657387,
      "rewards/cosine_scaled_reward": -0.019926004111766815,
      "rewards/format_reward": 0.8958333358168602,
      "step": 335
    },
    {
      "completion_length": 1604.708381652832,
      "epoch": 0.384,
      "grad_norm": 0.46796372532844543,
      "kl": 0.049716949462890625,
      "learning_rate": 3.641030065789562e-07,
      "loss": 0.002,
      "reward": 0.4504225810524076,
      "reward_std": 0.7610187456011772,
      "rewards/cosine_scaled_reward": 0.016421111300587654,
      "rewards/format_reward": 0.833333358168602,
      "step": 336
    },
    {
      "completion_length": 1863.1875457763672,
      "epoch": 0.3851428571428571,
      "grad_norm": 0.5092529058456421,
      "kl": 0.09614944458007812,
      "learning_rate": 3.612465628992203e-07,
      "loss": 0.0038,
      "reward": 0.38631977140903473,
      "reward_std": 0.9551087282598019,
      "rewards/cosine_scaled_reward": -0.020930441562086344,
      "rewards/format_reward": 0.7916666753590107,
      "step": 337
    },
    {
      "completion_length": 1182.8542175292969,
      "epoch": 0.3862857142857143,
      "grad_norm": 0.5370055437088013,
      "kl": 0.021060943603515625,
      "learning_rate": 3.5839931879571725e-07,
      "loss": 0.0008,
      "reward": 0.48011522740125656,
      "reward_std": 0.71619638428092,
      "rewards/cosine_scaled_reward": 0.003307923674583435,
      "rewards/format_reward": 0.895833333954215,
      "step": 338
    },
    {
      "completion_length": 1895.6250610351562,
      "epoch": 0.38742857142857146,
      "grad_norm": 0.8054518699645996,
      "kl": 0.08332443237304688,
      "learning_rate": 3.555614130391079e-07,
      "loss": 0.0033,
      "reward": 0.19063638825900853,
      "reward_std": 0.5477262288331985,
      "rewards/cosine_scaled_reward": -0.08626681286841631,
      "rewards/format_reward": 0.7083333525806665,
      "step": 339
    },
    {
      "completion_length": 1568.1250305175781,
      "epoch": 0.38857142857142857,
      "grad_norm": 0.42964616417884827,
      "kl": 0.040958404541015625,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.0016,
      "reward": 0.2769077487755567,
      "reward_std": 0.7800903655588627,
      "rewards/cosine_scaled_reward": -0.062285197753226385,
      "rewards/format_reward": 0.7500000111758709,
      "step": 340
    },
    {
      "completion_length": 1290.6250228881836,
      "epoch": 0.38971428571428574,
      "grad_norm": 0.5903887748718262,
      "kl": 0.032962799072265625,
      "learning_rate": 3.4991416936678276e-07,
      "loss": 0.0013,
      "reward": 0.625646581640467,
      "reward_std": 0.7541246488690376,
      "rewards/cosine_scaled_reward": 0.13255258556455374,
      "rewards/format_reward": 0.8333333414047956,
      "step": 341
    },
    {
      "completion_length": 1680.229232788086,
      "epoch": 0.39085714285714285,
      "grad_norm": 0.6661213040351868,
      "kl": 0.07326126098632812,
      "learning_rate": 3.471051066897562e-07,
      "loss": 0.0029,
      "reward": 0.4082430477719754,
      "reward_std": 0.7617458440363407,
      "rewards/cosine_scaled_reward": -0.03322136774659157,
      "rewards/format_reward": 0.8750000037252903,
      "step": 342
    },
    {
      "completion_length": 1694.3125457763672,
      "epoch": 0.392,
      "grad_norm": 0.9473098516464233,
      "kl": 0.058170318603515625,
      "learning_rate": 3.4430593282358777e-07,
      "loss": 0.0023,
      "reward": 0.325529879424721,
      "reward_std": 0.8013235367834568,
      "rewards/cosine_scaled_reward": -0.08207336533814669,
      "rewards/format_reward": 0.854166679084301,
      "step": 343
    },
    {
      "completion_length": 1449.3958892822266,
      "epoch": 0.3931428571428571,
      "grad_norm": 0.3888903856277466,
      "kl": 0.06012725830078125,
      "learning_rate": 3.4151678419606233e-07,
      "loss": 0.0024,
      "reward": 0.9358495399355888,
      "reward_std": 0.7060995027422905,
      "rewards/cosine_scaled_reward": 0.31333022052422166,
      "rewards/format_reward": 0.8958333432674408,
      "step": 344
    },
    {
      "completion_length": 1473.0417098999023,
      "epoch": 0.3942857142857143,
      "grad_norm": 0.605884313583374,
      "kl": 0.04383659362792969,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0018,
      "reward": 0.25372491776943207,
      "reward_std": 0.7733400501310825,
      "rewards/cosine_scaled_reward": -0.10962994769215584,
      "rewards/format_reward": 0.8125000223517418,
      "step": 345
    },
    {
      "completion_length": 1525.5833892822266,
      "epoch": 0.3954285714285714,
      "grad_norm": 0.41509008407592773,
      "kl": 0.0490264892578125,
      "learning_rate": 3.359691059183761e-07,
      "loss": 0.002,
      "reward": 0.39042545296251774,
      "reward_std": 0.8485492803156376,
      "rewards/cosine_scaled_reward": -0.09566876385360956,
      "rewards/format_reward": 0.9583333432674408,
      "step": 346
    },
    {
      "completion_length": 1555.5208740234375,
      "epoch": 0.3965714285714286,
      "grad_norm": 0.6986513137817383,
      "kl": 0.058147430419921875,
      "learning_rate": 3.3321084665422803e-07,
      "loss": 0.0023,
      "reward": 0.1333858126308769,
      "reward_std": 0.7080183140933514,
      "rewards/cosine_scaled_reward": -0.23809866607189178,
      "rewards/format_reward": 0.9166666865348816,
      "step": 347
    },
    {
      "completion_length": 1694.0625686645508,
      "epoch": 0.3977142857142857,
      "grad_norm": 0.7975203394889832,
      "kl": 0.0868682861328125,
      "learning_rate": 3.3046315338757026e-07,
      "loss": 0.0035,
      "reward": 0.39133906550705433,
      "reward_std": 0.7762907817959785,
      "rewards/cosine_scaled_reward": -0.012924212962388992,
      "rewards/format_reward": 0.8125000111758709,
      "step": 348
    },
    {
      "completion_length": 1404.1667404174805,
      "epoch": 0.39885714285714285,
      "grad_norm": 3.111985445022583,
      "kl": 0.11113739013671875,
      "learning_rate": 3.2772616003709616e-07,
      "loss": 0.0044,
      "reward": 0.47680344711989164,
      "reward_std": 0.7106522209942341,
      "rewards/cosine_scaled_reward": 0.014676447957754135,
      "rewards/format_reward": 0.8750000149011612,
      "step": 349
    },
    {
      "completion_length": 820.4791831970215,
      "epoch": 0.4,
      "grad_norm": 0.2917865216732025,
      "kl": 0.01514434814453125,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.0006,
      "reward": 0.4086197968572378,
      "reward_std": 0.7495947815477848,
      "rewards/cosine_scaled_reward": -0.09753232356160879,
      "rewards/format_reward": 1.0,
      "step": 350
    },
    {
      "completion_length": 1446.520866394043,
      "epoch": 0.40114285714285713,
      "grad_norm": 0.7599288821220398,
      "kl": 0.07086944580078125,
      "learning_rate": 3.222848061454764e-07,
      "loss": 0.0028,
      "reward": 0.3731183987110853,
      "reward_std": 0.7764743529260159,
      "rewards/cosine_scaled_reward": -0.060001387260854244,
      "rewards/format_reward": 0.8750000055879354,
      "step": 351
    },
    {
      "completion_length": 1387.0000381469727,
      "epoch": 0.4022857142857143,
      "grad_norm": 0.7137279510498047,
      "kl": 0.06012725830078125,
      "learning_rate": 3.195807108082429e-07,
      "loss": 0.0024,
      "reward": 0.42939993026084267,
      "reward_std": 0.8450379781424999,
      "rewards/cosine_scaled_reward": 0.003599647810915485,
      "rewards/format_reward": 0.8125000055879354,
      "step": 352
    },
    {
      "completion_length": 1223.083366394043,
      "epoch": 0.4034285714285714,
      "grad_norm": 0.5175493359565735,
      "kl": 0.0400390625,
      "learning_rate": 3.168878457820915e-07,
      "loss": 0.0016,
      "reward": 0.716075923293829,
      "reward_std": 0.9309490397572517,
      "rewards/cosine_scaled_reward": 0.1352907968685031,
      "rewards/format_reward": 0.9375000074505806,
      "step": 353
    },
    {
      "completion_length": 1064.8333587646484,
      "epoch": 0.4045714285714286,
      "grad_norm": 0.6795687079429626,
      "kl": 0.04412841796875,
      "learning_rate": 3.142063423134644e-07,
      "loss": 0.0018,
      "reward": 0.5015737505163997,
      "reward_std": 0.5811274200677872,
      "rewards/cosine_scaled_reward": 0.009966753888875246,
      "rewards/format_reward": 0.9375,
      "step": 354
    },
    {
      "completion_length": 972.6875152587891,
      "epoch": 0.4057142857142857,
      "grad_norm": 0.3503471910953522,
      "kl": 0.02288818359375,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0009,
      "reward": 0.6657696301117539,
      "reward_std": 0.7398427277803421,
      "rewards/cosine_scaled_reward": 0.08280336670577526,
      "rewards/format_reward": 1.0,
      "step": 355
    },
    {
      "completion_length": 1551.833396911621,
      "epoch": 0.40685714285714286,
      "grad_norm": 0.4588610529899597,
      "kl": 0.06862258911132812,
      "learning_rate": 3.0887794225945143e-07,
      "loss": 0.0027,
      "reward": 0.3966330944094807,
      "reward_std": 0.6840399689972401,
      "rewards/cosine_scaled_reward": -0.058146869763731956,
      "rewards/format_reward": 0.9166666716337204,
      "step": 356
    },
    {
      "completion_length": 1772.1042175292969,
      "epoch": 0.408,
      "grad_norm": 1.0075832605361938,
      "kl": 0.100799560546875,
      "learning_rate": 3.062313053727671e-07,
      "loss": 0.004,
      "reward": 0.12496365327388048,
      "reward_std": 0.8355611003935337,
      "rewards/cosine_scaled_reward": -0.15991984121501446,
      "rewards/format_reward": 0.7291666828095913,
      "step": 357
    },
    {
      "completion_length": 1562.562515258789,
      "epoch": 0.40914285714285714,
      "grad_norm": 0.7331953644752502,
      "kl": 0.07072067260742188,
      "learning_rate": 3.0359654942835247e-07,
      "loss": 0.0028,
      "reward": 0.5806331331841648,
      "reward_std": 0.8575869612395763,
      "rewards/cosine_scaled_reward": 0.05609727092087269,
      "rewards/format_reward": 0.9166666716337204,
      "step": 358
    },
    {
      "completion_length": 942.1666870117188,
      "epoch": 0.4102857142857143,
      "grad_norm": 0.5968821048736572,
      "kl": 0.0392608642578125,
      "learning_rate": 3.0097380284049523e-07,
      "loss": 0.0016,
      "reward": 0.42657787445932627,
      "reward_std": 0.5996944792568684,
      "rewards/cosine_scaled_reward": -0.06319771538255736,
      "rewards/format_reward": 0.9791666716337204,
      "step": 359
    },
    {
      "completion_length": 1402.3125381469727,
      "epoch": 0.4114285714285714,
      "grad_norm": 0.7901036143302917,
      "kl": 0.07848739624023438,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0031,
      "reward": 0.5172299258410931,
      "reward_std": 0.8756576031446457,
      "rewards/cosine_scaled_reward": 0.017652488488238305,
      "rewards/format_reward": 0.8958333507180214,
      "step": 360
    },
    {
      "completion_length": 1255.0000534057617,
      "epoch": 0.4125714285714286,
      "grad_norm": 0.7914440035820007,
      "kl": 0.0751495361328125,
      "learning_rate": 2.9576484845877793e-07,
      "loss": 0.003,
      "reward": 0.24485921673476696,
      "reward_std": 0.7093796096742153,
      "rewards/cosine_scaled_reward": -0.14366490487009287,
      "rewards/format_reward": 0.8750000149011612,
      "step": 361
    },
    {
      "completion_length": 899.2916831970215,
      "epoch": 0.4137142857142857,
      "grad_norm": 0.6618494391441345,
      "kl": 0.0485382080078125,
      "learning_rate": 2.931788945420058e-07,
      "loss": 0.0019,
      "reward": 0.510432411916554,
      "reward_std": 0.6674188002943993,
      "rewards/cosine_scaled_reward": 0.010434551164507866,
      "rewards/format_reward": 0.9375000074505806,
      "step": 362
    },
    {
      "completion_length": 863.9166946411133,
      "epoch": 0.41485714285714287,
      "grad_norm": 0.7848973274230957,
      "kl": 0.029571533203125,
      "learning_rate": 2.9060545772359305e-07,
      "loss": 0.0012,
      "reward": 0.844133562874049,
      "reward_std": 0.6796937808394432,
      "rewards/cosine_scaled_reward": 0.20261266455054283,
      "rewards/format_reward": 1.0,
      "step": 363
    },
    {
      "completion_length": 1205.1666870117188,
      "epoch": 0.416,
      "grad_norm": 0.5032205581665039,
      "kl": 0.03617095947265625,
      "learning_rate": 2.8804466342921987e-07,
      "loss": 0.0014,
      "reward": 0.08574948133900762,
      "reward_std": 0.4307812377810478,
      "rewards/cosine_scaled_reward": -0.26402094028890133,
      "rewards/format_reward": 0.9375000074505806,
      "step": 364
    },
    {
      "completion_length": 1884.0625686645508,
      "epoch": 0.41714285714285715,
      "grad_norm": 0.9030827283859253,
      "kl": 0.1519775390625,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.0061,
      "reward": 0.17442028690129519,
      "reward_std": 0.685190960764885,
      "rewards/cosine_scaled_reward": -0.09430265240371227,
      "rewards/format_reward": 0.6875000093132257,
      "step": 365
    },
    {
      "completion_length": 1026.7708644866943,
      "epoch": 0.41828571428571426,
      "grad_norm": 0.6462038159370422,
      "kl": 0.0249786376953125,
      "learning_rate": 2.829615010283344e-07,
      "loss": 0.001,
      "reward": 0.7736296411603689,
      "reward_std": 0.8399604074656963,
      "rewards/cosine_scaled_reward": 0.15821054810658097,
      "rewards/format_reward": 0.9791666716337204,
      "step": 366
    },
    {
      "completion_length": 1750.541748046875,
      "epoch": 0.41942857142857143,
      "grad_norm": 0.8086258769035339,
      "kl": 0.12229537963867188,
      "learning_rate": 2.8043938066798645e-07,
      "loss": 0.0049,
      "reward": 0.40639928355813026,
      "reward_std": 0.6661986261606216,
      "rewards/cosine_scaled_reward": -0.005629323422908783,
      "rewards/format_reward": 0.8333333358168602,
      "step": 367
    },
    {
      "completion_length": 1828.7917289733887,
      "epoch": 0.4205714285714286,
      "grad_norm": 1.4647676944732666,
      "kl": 0.092529296875,
      "learning_rate": 2.7793039831193133e-07,
      "loss": 0.0037,
      "reward": 0.3596602795878425,
      "reward_std": 0.8183485567569733,
      "rewards/cosine_scaled_reward": -0.04840289568528533,
      "rewards/format_reward": 0.8333333395421505,
      "step": 368
    },
    {
      "completion_length": 1507.0208854675293,
      "epoch": 0.4217142857142857,
      "grad_norm": 0.9977928996086121,
      "kl": 0.105255126953125,
      "learning_rate": 2.7543467624442956e-07,
      "loss": 0.0042,
      "reward": 0.3323223125189543,
      "reward_std": 0.8554218038916588,
      "rewards/cosine_scaled_reward": -0.0706447935081087,
      "rewards/format_reward": 0.8333333432674408,
      "step": 369
    },
    {
      "completion_length": 1375.958366394043,
      "epoch": 0.4228571428571429,
      "grad_norm": 0.6904154419898987,
      "kl": 0.1561279296875,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0062,
      "reward": 0.2625284339301288,
      "reward_std": 0.57734365016222,
      "rewards/cosine_scaled_reward": -0.11408653669059277,
      "rewards/format_reward": 0.8541666679084301,
      "step": 370
    },
    {
      "completion_length": 747.4166831970215,
      "epoch": 0.424,
      "grad_norm": 0.7088914513587952,
      "kl": 0.04413604736328125,
      "learning_rate": 2.7048349887476037e-07,
      "loss": 0.0018,
      "reward": 0.6137365428730845,
      "reward_std": 0.7054285481572151,
      "rewards/cosine_scaled_reward": 0.06669859914109111,
      "rewards/format_reward": 0.9583333432674408,
      "step": 371
    },
    {
      "completion_length": 1583.250015258789,
      "epoch": 0.42514285714285716,
      "grad_norm": 1.3554654121398926,
      "kl": 0.08654022216796875,
      "learning_rate": 2.6802828488599294e-07,
      "loss": 0.0035,
      "reward": 0.5319191414746456,
      "reward_std": 0.6091607809066772,
      "rewards/cosine_scaled_reward": 0.052222222089767456,
      "rewards/format_reward": 0.8958333507180214,
      "step": 372
    },
    {
      "completion_length": 840.7500114440918,
      "epoch": 0.42628571428571427,
      "grad_norm": 0.823254406452179,
      "kl": 0.02849578857421875,
      "learning_rate": 2.655868138008171e-07,
      "loss": 0.0011,
      "reward": 0.23188667371869087,
      "reward_std": 0.6112702935934067,
      "rewards/cosine_scaled_reward": -0.20857458282262087,
      "rewards/format_reward": 1.0,
      "step": 373
    },
    {
      "completion_length": 1146.895881652832,
      "epoch": 0.42742857142857144,
      "grad_norm": 0.8236629962921143,
      "kl": 0.047637939453125,
      "learning_rate": 2.631592046130896e-07,
      "loss": 0.0019,
      "reward": 0.5981785822659731,
      "reward_std": 0.8187721818685532,
      "rewards/cosine_scaled_reward": 0.03387853177264333,
      "rewards/format_reward": 0.9791666716337204,
      "step": 374
    },
    {
      "completion_length": 1634.3750457763672,
      "epoch": 0.42857142857142855,
      "grad_norm": 1.242490530014038,
      "kl": 0.20917510986328125,
      "learning_rate": 2.6074557564105724e-07,
      "loss": 0.0084,
      "reward": 0.49338567443192005,
      "reward_std": 0.8198688104748726,
      "rewards/cosine_scaled_reward": 0.07417132705450058,
      "rewards/format_reward": 0.7708333469927311,
      "step": 375
    },
    {
      "completion_length": 1403.1875381469727,
      "epoch": 0.4297142857142857,
      "grad_norm": 1.0553152561187744,
      "kl": 0.1156768798828125,
      "learning_rate": 2.583460445215911e-07,
      "loss": 0.0046,
      "reward": 0.4271426647901535,
      "reward_std": 0.7959622256457806,
      "rewards/cosine_scaled_reward": -0.03355884738266468,
      "rewards/format_reward": 0.8958333432674408,
      "step": 376
    },
    {
      "completion_length": 1332.0417098999023,
      "epoch": 0.4308571428571429,
      "grad_norm": 0.9126045107841492,
      "kl": 0.1114654541015625,
      "learning_rate": 2.5596072820445254e-07,
      "loss": 0.0045,
      "reward": 0.5075240693986416,
      "reward_std": 0.855946060270071,
      "rewards/cosine_scaled_reward": -0.0022887131199240685,
      "rewards/format_reward": 0.9375000149011612,
      "step": 377
    },
    {
      "completion_length": 1077.4583854675293,
      "epoch": 0.432,
      "grad_norm": 0.5990425944328308,
      "kl": 0.030300140380859375,
      "learning_rate": 2.5358974294659373e-07,
      "loss": 0.0012,
      "reward": 0.5777185422666662,
      "reward_std": 0.9156284630298615,
      "rewards/cosine_scaled_reward": 0.008807800710201263,
      "rewards/format_reward": 1.0,
      "step": 378
    },
    {
      "completion_length": 1503.7708587646484,
      "epoch": 0.43314285714285716,
      "grad_norm": 0.6305628418922424,
      "kl": 0.13201522827148438,
      "learning_rate": 2.512332043064913e-07,
      "loss": 0.0053,
      "reward": 0.3502848669886589,
      "reward_std": 0.7390433885157108,
      "rewards/cosine_scaled_reward": -0.06138859502971172,
      "rewards/format_reward": 0.8541666679084301,
      "step": 379
    },
    {
      "completion_length": 1135.229175567627,
      "epoch": 0.4342857142857143,
      "grad_norm": 1.0688378810882568,
      "kl": 0.11534881591796875,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.0046,
      "reward": 0.415515000699088,
      "reward_std": 0.7041169926524162,
      "rewards/cosine_scaled_reward": -0.053237104788422585,
      "rewards/format_reward": 0.9375000149011612,
      "step": 380
    },
    {
      "completion_length": 1531.4583854675293,
      "epoch": 0.43542857142857144,
      "grad_norm": 1.1264177560806274,
      "kl": 0.13215255737304688,
      "learning_rate": 2.465639255873246e-07,
      "loss": 0.0053,
      "reward": 0.24819382751593366,
      "reward_std": 0.7195433788001537,
      "rewards/cosine_scaled_reward": -0.15285431523807347,
      "rewards/format_reward": 0.8958333507180214,
      "step": 381
    },
    {
      "completion_length": 1023.3750305175781,
      "epoch": 0.43657142857142855,
      "grad_norm": 0.40737247467041016,
      "kl": 0.0533447265625,
      "learning_rate": 2.4425141308231765e-07,
      "loss": 0.0021,
      "reward": 0.2534531052224338,
      "reward_std": 0.6627279557287693,
      "rewards/cosine_scaled_reward": -0.19920427445322275,
      "rewards/format_reward": 1.0,
      "step": 382
    },
    {
      "completion_length": 1496.5417251586914,
      "epoch": 0.4377142857142857,
      "grad_norm": 1.8750388622283936,
      "kl": 0.2808380126953125,
      "learning_rate": 2.4195380233209006e-07,
      "loss": 0.0112,
      "reward": 0.3533962171059102,
      "reward_std": 0.8847429379820824,
      "rewards/cosine_scaled_reward": -0.04301046393811703,
      "rewards/format_reward": 0.8125000074505806,
      "step": 383
    },
    {
      "completion_length": 1054.5833435058594,
      "epoch": 0.43885714285714283,
      "grad_norm": 0.5143475532531738,
      "kl": 0.0340118408203125,
      "learning_rate": 2.3967120531894857e-07,
      "loss": 0.0014,
      "reward": 0.9491607993841171,
      "reward_std": 0.8766042143106461,
      "rewards/cosine_scaled_reward": 0.2751440554857254,
      "rewards/format_reward": 0.9791666716337204,
      "step": 384
    },
    {
      "completion_length": 1428.645896911621,
      "epoch": 0.44,
      "grad_norm": 0.6951473355293274,
      "kl": 0.14729690551757812,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0059,
      "reward": 0.4135100084822625,
      "reward_std": 0.7766407653689384,
      "rewards/cosine_scaled_reward": -0.03412807872518897,
      "rewards/format_reward": 0.8750000111758709,
      "step": 385
    },
    {
      "completion_length": 1270.7291793823242,
      "epoch": 0.44114285714285717,
      "grad_norm": 1.0250437259674072,
      "kl": 0.1134185791015625,
      "learning_rate": 2.3515149676898552e-07,
      "loss": 0.0045,
      "reward": 0.47308777272701263,
      "reward_std": 0.6565175838768482,
      "rewards/cosine_scaled_reward": -0.013674074783921242,
      "rewards/format_reward": 0.9375,
      "step": 386
    },
    {
      "completion_length": 1528.3542213439941,
      "epoch": 0.4422857142857143,
      "grad_norm": 2.0494582653045654,
      "kl": 0.19681549072265625,
      "learning_rate": 2.3291460551638237e-07,
      "loss": 0.0079,
      "reward": 0.31283304444514215,
      "reward_std": 0.583834994584322,
      "rewards/cosine_scaled_reward": -0.06756466627120972,
      "rewards/format_reward": 0.8333333469927311,
      "step": 387
    },
    {
      "completion_length": 1097.583351135254,
      "epoch": 0.44342857142857145,
      "grad_norm": 0.5577840209007263,
      "kl": 0.0581207275390625,
      "learning_rate": 2.306931685585657e-07,
      "loss": 0.0023,
      "reward": 0.7101229609397706,
      "reward_std": 0.6756577789783478,
      "rewards/cosine_scaled_reward": 0.1360636167228222,
      "rewards/format_reward": 0.9583333358168602,
      "step": 388
    },
    {
      "completion_length": 1202.3958587646484,
      "epoch": 0.44457142857142856,
      "grad_norm": 0.6137182712554932,
      "kl": 0.0680389404296875,
      "learning_rate": 2.2848729416523859e-07,
      "loss": 0.0027,
      "reward": 0.27682637330144644,
      "reward_std": 0.7127226404845715,
      "rewards/cosine_scaled_reward": -0.1509529883041978,
      "rewards/format_reward": 0.9375000074505806,
      "step": 389
    },
    {
      "completion_length": 1435.2917213439941,
      "epoch": 0.44571428571428573,
      "grad_norm": 0.9228805303573608,
      "kl": 0.1553211212158203,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.0062,
      "reward": 0.21503479685634375,
      "reward_std": 0.7798956334590912,
      "rewards/cosine_scaled_reward": -0.11404814245179296,
      "rewards/format_reward": 0.7708333544433117,
      "step": 390
    },
    {
      "completion_length": 999.9167022705078,
      "epoch": 0.44685714285714284,
      "grad_norm": 0.9710776209831238,
      "kl": 0.08495330810546875,
      "learning_rate": 2.2412266235313973e-07,
      "loss": 0.0034,
      "reward": 0.6504320180974901,
      "reward_std": 0.6109317727386951,
      "rewards/cosine_scaled_reward": 0.1076424578204751,
      "rewards/format_reward": 0.9375000149011612,
      "step": 391
    },
    {
      "completion_length": 1423.895881652832,
      "epoch": 0.448,
      "grad_norm": 1.6490659713745117,
      "kl": 0.223846435546875,
      "learning_rate": 2.2196411766036487e-07,
      "loss": 0.009,
      "reward": 0.1661514127627015,
      "reward_std": 0.7237707450985909,
      "rewards/cosine_scaled_reward": -0.1779894083738327,
      "rewards/format_reward": 0.833333333954215,
      "step": 392
    },
    {
      "completion_length": 1450.4792022705078,
      "epoch": 0.4491428571428571,
      "grad_norm": 1.2397853136062622,
      "kl": 0.10369110107421875,
      "learning_rate": 2.1982156097370557e-07,
      "loss": 0.0042,
      "reward": 0.49226129427552223,
      "reward_std": 0.8764687478542328,
      "rewards/cosine_scaled_reward": 0.0059880828484892845,
      "rewards/format_reward": 0.8958333358168602,
      "step": 393
    },
    {
      "completion_length": 1569.3958702087402,
      "epoch": 0.4502857142857143,
      "grad_norm": 1.4030894041061401,
      "kl": 0.1822967529296875,
      "learning_rate": 2.1769509671835223e-07,
      "loss": 0.0073,
      "reward": 0.11027060728520155,
      "reward_std": 0.6850339062511921,
      "rewards/cosine_scaled_reward": -0.21509641967713833,
      "rewards/format_reward": 0.8333333432674408,
      "step": 394
    },
    {
      "completion_length": 1156.7917175292969,
      "epoch": 0.4514285714285714,
      "grad_norm": 0.9848508238792419,
      "kl": 0.077606201171875,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.0031,
      "reward": 0.41267195832915604,
      "reward_std": 0.7016527280211449,
      "rewards/cosine_scaled_reward": -0.04957490786910057,
      "rewards/format_reward": 0.916666679084301,
      "step": 395
    },
    {
      "completion_length": 1016.1458511352539,
      "epoch": 0.45257142857142857,
      "grad_norm": 0.6101754903793335,
      "kl": 0.020412445068359375,
      "learning_rate": 2.134908592756607e-07,
      "loss": 0.0008,
      "reward": 0.4085001898929477,
      "reward_std": 0.6500335298478603,
      "rewards/cosine_scaled_reward": -0.058247581124305725,
      "rewards/format_reward": 0.9375000074505806,
      "step": 396
    },
    {
      "completion_length": 1285.4583892822266,
      "epoch": 0.45371428571428574,
      "grad_norm": 0.699232280254364,
      "kl": 0.16654586791992188,
      "learning_rate": 2.1141329099692406e-07,
      "loss": 0.0067,
      "reward": 0.2716854903846979,
      "reward_std": 0.6572816967964172,
      "rewards/cosine_scaled_reward": -0.10944613942410797,
      "rewards/format_reward": 0.8541666697710752,
      "step": 397
    },
    {
      "completion_length": 1064.0625305175781,
      "epoch": 0.45485714285714285,
      "grad_norm": 0.4855163097381592,
      "kl": 0.034351348876953125,
      "learning_rate": 2.0935222495670968e-07,
      "loss": 0.0014,
      "reward": 0.40476914402097464,
      "reward_std": 0.6724813655018806,
      "rewards/cosine_scaled_reward": -0.0830375433433801,
      "rewards/format_reward": 0.9791666716337204,
      "step": 398
    },
    {
      "completion_length": 944.5417098999023,
      "epoch": 0.456,
      "grad_norm": 0.42374876141548157,
      "kl": 0.0376739501953125,
      "learning_rate": 2.0730776160846853e-07,
      "loss": 0.0015,
      "reward": 0.5400755191221833,
      "reward_std": 0.7856027092784643,
      "rewards/cosine_scaled_reward": -0.010072574485093355,
      "rewards/format_reward": 1.0,
      "step": 399
    },
    {
      "completion_length": 898.6875152587891,
      "epoch": 0.45714285714285713,
      "grad_norm": 0.6626017689704895,
      "kl": 0.0309600830078125,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.0012,
      "reward": 0.7812614720314741,
      "reward_std": 0.6693713776767254,
      "rewards/cosine_scaled_reward": 0.1816884014988318,
      "rewards/format_reward": 0.9583333358168602,
      "step": 400
    },
    {
      "completion_length": 1256.6458740234375,
      "epoch": 0.4582857142857143,
      "grad_norm": 1.2477692365646362,
      "kl": 0.08155059814453125,
      "learning_rate": 2.032690407508949e-07,
      "loss": 0.0033,
      "reward": 0.5257098386064172,
      "reward_std": 0.81386823579669,
      "rewards/cosine_scaled_reward": 0.011443385854363441,
      "rewards/format_reward": 0.9375000074505806,
      "step": 401
    },
    {
      "completion_length": 1103.5208587646484,
      "epoch": 0.4594285714285714,
      "grad_norm": 3.268733501434326,
      "kl": 0.19863510131835938,
      "learning_rate": 2.0127498008311922e-07,
      "loss": 0.0079,
      "reward": 0.4599206205457449,
      "reward_std": 0.5488756932318211,
      "rewards/cosine_scaled_reward": -0.017510680481791496,
      "rewards/format_reward": 0.9375000149011612,
      "step": 402
    },
    {
      "completion_length": 931.4375190734863,
      "epoch": 0.4605714285714286,
      "grad_norm": 1.2398287057876587,
      "kl": 0.09774017333984375,
      "learning_rate": 1.9929791578083655e-07,
      "loss": 0.0039,
      "reward": 0.45444739051163197,
      "reward_std": 0.4980122298002243,
      "rewards/cosine_scaled_reward": -0.007617715746164322,
      "rewards/format_reward": 0.9166666679084301,
      "step": 403
    },
    {
      "completion_length": 1211.6041870117188,
      "epoch": 0.4617142857142857,
      "grad_norm": 1.140418529510498,
      "kl": 0.15045547485351562,
      "learning_rate": 1.9733794420337213e-07,
      "loss": 0.006,
      "reward": 0.4547617736971006,
      "reward_std": 0.5626656413078308,
      "rewards/cosine_scaled_reward": -0.01060919463634491,
      "rewards/format_reward": 0.9166666716337204,
      "step": 404
    },
    {
      "completion_length": 974.6875305175781,
      "epoch": 0.46285714285714286,
      "grad_norm": 1.321244239807129,
      "kl": 0.094329833984375,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.0038,
      "reward": 0.7235147282481194,
      "reward_std": 0.8120486699044704,
      "rewards/cosine_scaled_reward": 0.14601082005538046,
      "rewards/format_reward": 0.9375000149011612,
      "step": 405
    },
    {
      "completion_length": 1133.5416946411133,
      "epoch": 0.464,
      "grad_norm": 0.6807760000228882,
      "kl": 0.1176910400390625,
      "learning_rate": 1.934696604901642e-07,
      "loss": 0.0047,
      "reward": 0.6748548084869981,
      "reward_std": 0.78925821185112,
      "rewards/cosine_scaled_reward": 0.10528674224042334,
      "rewards/format_reward": 0.9583333432674408,
      "step": 406
    },
    {
      "completion_length": 1320.2500305175781,
      "epoch": 0.46514285714285714,
      "grad_norm": 1.170783519744873,
      "kl": 0.21154022216796875,
      "learning_rate": 1.915615368891117e-07,
      "loss": 0.0085,
      "reward": 0.4940835009329021,
      "reward_std": 0.5547701120376587,
      "rewards/cosine_scaled_reward": 0.037424055859446526,
      "rewards/format_reward": 0.8750000055879354,
      "step": 407
    },
    {
      "completion_length": 1262.9375457763672,
      "epoch": 0.4662857142857143,
      "grad_norm": 0.8598933815956116,
      "kl": 0.14272689819335938,
      "learning_rate": 1.8967088307307e-07,
      "loss": 0.0057,
      "reward": 0.404069249285385,
      "reward_std": 0.7031322456896305,
      "rewards/cosine_scaled_reward": -0.0431424961425364,
      "rewards/format_reward": 0.895833333954215,
      "step": 408
    },
    {
      "completion_length": 1639.5000305175781,
      "epoch": 0.4674285714285714,
      "grad_norm": 1.15615975856781,
      "kl": 0.17174530029296875,
      "learning_rate": 1.8779779118983867e-07,
      "loss": 0.0069,
      "reward": 0.31328563997521996,
      "reward_std": 0.7064049206674099,
      "rewards/cosine_scaled_reward": -0.11326186126098037,
      "rewards/format_reward": 0.9166666865348816,
      "step": 409
    },
    {
      "completion_length": 1499.333366394043,
      "epoch": 0.4685714285714286,
      "grad_norm": 1.267825961112976,
      "kl": 0.22209548950195312,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0089,
      "reward": 0.3295501284301281,
      "reward_std": 0.6071663275361061,
      "rewards/cosine_scaled_reward": -0.09065644256770611,
      "rewards/format_reward": 0.8958333358168602,
      "step": 410
    },
    {
      "completion_length": 1507.6041946411133,
      "epoch": 0.4697142857142857,
      "grad_norm": 1.4931728839874268,
      "kl": 0.13777923583984375,
      "learning_rate": 1.8410465752883758e-07,
      "loss": 0.0055,
      "reward": 0.3418419687077403,
      "reward_std": 0.8959544003009796,
      "rewards/cosine_scaled_reward": -0.10114361811429262,
      "rewards/format_reward": 0.8958333395421505,
      "step": 411
    },
    {
      "completion_length": 1131.0208587646484,
      "epoch": 0.47085714285714286,
      "grad_norm": 1.660409688949585,
      "kl": 0.13482666015625,
      "learning_rate": 1.822847957491922e-07,
      "loss": 0.0054,
      "reward": 0.5560126416385174,
      "reward_std": 0.7937713004648685,
      "rewards/cosine_scaled_reward": 0.06759718805551529,
      "rewards/format_reward": 0.8750000149011612,
      "step": 412
    },
    {
      "completion_length": 1269.9375267028809,
      "epoch": 0.472,
      "grad_norm": 0.629540205001831,
      "kl": 0.112335205078125,
      "learning_rate": 1.804828558898332e-07,
      "loss": 0.0045,
      "reward": 0.46968742460012436,
      "reward_std": 0.6266082711517811,
      "rewards/cosine_scaled_reward": 0.005679788533598185,
      "rewards/format_reward": 0.8958333358168602,
      "step": 413
    },
    {
      "completion_length": 1644.041732788086,
      "epoch": 0.47314285714285714,
      "grad_norm": 1.6342582702636719,
      "kl": 0.16037750244140625,
      "learning_rate": 1.7869892577476722e-07,
      "loss": 0.0064,
      "reward": 0.17194935493171215,
      "reward_std": 0.7680409215390682,
      "rewards/cosine_scaled_reward": -0.1542492527514696,
      "rewards/format_reward": 0.7916666846722364,
      "step": 414
    },
    {
      "completion_length": 1607.4167022705078,
      "epoch": 0.4742857142857143,
      "grad_norm": 2.081047296524048,
      "kl": 0.315277099609375,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0126,
      "reward": 0.15895176446065307,
      "reward_std": 0.7137060277163982,
      "rewards/cosine_scaled_reward": -0.0956956222653389,
      "rewards/format_reward": 0.6666666865348816,
      "step": 415
    },
    {
      "completion_length": 1069.0208587646484,
      "epoch": 0.4754285714285714,
      "grad_norm": 1.11202871799469,
      "kl": 0.07001304626464844,
      "learning_rate": 1.7518544168045524e-07,
      "loss": 0.0028,
      "reward": 0.8406836800277233,
      "reward_std": 0.7853529006242752,
      "rewards/cosine_scaled_reward": 0.20668393187224865,
      "rewards/format_reward": 0.9791666716337204,
      "step": 416
    },
    {
      "completion_length": 1633.3125534057617,
      "epoch": 0.4765714285714286,
      "grad_norm": 2.8300588130950928,
      "kl": 0.274017333984375,
      "learning_rate": 1.7345605894346726e-07,
      "loss": 0.011,
      "reward": 0.2972092959098518,
      "reward_std": 0.8013018406927586,
      "rewards/cosine_scaled_reward": -0.08885959023609757,
      "rewards/format_reward": 0.833333358168602,
      "step": 417
    },
    {
      "completion_length": 1035.1041870117188,
      "epoch": 0.4777142857142857,
      "grad_norm": 0.9021016359329224,
      "kl": 0.11577606201171875,
      "learning_rate": 1.7174502842694212e-07,
      "loss": 0.0046,
      "reward": 0.8001389261335135,
      "reward_std": 0.8382466398179531,
      "rewards/cosine_scaled_reward": 0.18563345912843943,
      "rewards/format_reward": 0.9583333358168602,
      "step": 418
    },
    {
      "completion_length": 1300.9792175292969,
      "epoch": 0.47885714285714287,
      "grad_norm": 1.542521595954895,
      "kl": 0.1188812255859375,
      "learning_rate": 1.7005243352409333e-07,
      "loss": 0.0048,
      "reward": 0.47798004280775785,
      "reward_std": 0.6525602787733078,
      "rewards/cosine_scaled_reward": -0.022527330555021763,
      "rewards/format_reward": 0.9583333432674408,
      "step": 419
    },
    {
      "completion_length": 774.5833587646484,
      "epoch": 0.48,
      "grad_norm": 0.5312778949737549,
      "kl": 0.01689910888671875,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0007,
      "reward": 0.34607438833336346,
      "reward_std": 0.6675878167152405,
      "rewards/cosine_scaled_reward": -0.12272587232291698,
      "rewards/format_reward": 0.9791666716337204,
      "step": 420
    },
    {
      "completion_length": 1449.6250381469727,
      "epoch": 0.48114285714285715,
      "grad_norm": 1.6367040872573853,
      "kl": 0.283203125,
      "learning_rate": 1.6672287963562852e-07,
      "loss": 0.0114,
      "reward": 0.3528535794466734,
      "reward_std": 0.7183955535292625,
      "rewards/cosine_scaled_reward": -0.09002240933477879,
      "rewards/format_reward": 0.916666679084301,
      "step": 421
    },
    {
      "completion_length": 1520.2917251586914,
      "epoch": 0.48228571428571426,
      "grad_norm": 1.4518808126449585,
      "kl": 0.29302215576171875,
      "learning_rate": 1.6508608292777203e-07,
      "loss": 0.0117,
      "reward": 0.46497548231855035,
      "reward_std": 0.6263095885515213,
      "rewards/cosine_scaled_reward": 0.00397377647459507,
      "rewards/format_reward": 0.8958333432674408,
      "step": 422
    },
    {
      "completion_length": 1500.2500534057617,
      "epoch": 0.48342857142857143,
      "grad_norm": 1.8499155044555664,
      "kl": 0.3067626953125,
      "learning_rate": 1.6346804638120098e-07,
      "loss": 0.0122,
      "reward": 0.27377578942105174,
      "reward_std": 0.6246924735605717,
      "rewards/cosine_scaled_reward": -0.08783163502812386,
      "rewards/format_reward": 0.812500013038516,
      "step": 423
    },
    {
      "completion_length": 1651.6250610351562,
      "epoch": 0.4845714285714286,
      "grad_norm": 2.137979745864868,
      "kl": 0.3133392333984375,
      "learning_rate": 1.6186884885673413e-07,
      "loss": 0.0125,
      "reward": 0.17908507003448904,
      "reward_std": 0.8118347823619843,
      "rewards/cosine_scaled_reward": -0.165109351859428,
      "rewards/format_reward": 0.812500013038516,
      "step": 424
    },
    {
      "completion_length": 1094.2500381469727,
      "epoch": 0.4857142857142857,
      "grad_norm": 1.6124348640441895,
      "kl": 0.13685989379882812,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0055,
      "reward": 0.9797765575349331,
      "reward_std": 0.7934582978487015,
      "rewards/cosine_scaled_reward": 0.33494018763303757,
      "rewards/format_reward": 0.916666679084301,
      "step": 425
    },
    {
      "completion_length": 1124.4166831970215,
      "epoch": 0.4868571428571429,
      "grad_norm": 1.3242241144180298,
      "kl": 0.1932373046875,
      "learning_rate": 1.5872728172265146e-07,
      "loss": 0.0077,
      "reward": 0.3963227402418852,
      "reward_std": 0.6691669598221779,
      "rewards/cosine_scaled_reward": -0.06672874744981527,
      "rewards/format_reward": 0.9375,
      "step": 426
    },
    {
      "completion_length": 1525.520896911621,
      "epoch": 0.488,
      "grad_norm": 1.3827719688415527,
      "kl": 0.24054718017578125,
      "learning_rate": 1.5718506522858572e-07,
      "loss": 0.0096,
      "reward": 0.38459044555202127,
      "reward_std": 0.8512180484831333,
      "rewards/cosine_scaled_reward": -0.07769208890385926,
      "rewards/format_reward": 0.9166666865348816,
      "step": 427
    },
    {
      "completion_length": 1536.3542098999023,
      "epoch": 0.48914285714285716,
      "grad_norm": 1.885048270225525,
      "kl": 0.2565765380859375,
      "learning_rate": 1.5566199398026147e-07,
      "loss": 0.0103,
      "reward": 0.1980901760980487,
      "reward_std": 0.6689082272350788,
      "rewards/cosine_scaled_reward": -0.1701727721374482,
      "rewards/format_reward": 0.8750000074505806,
      "step": 428
    },
    {
      "completion_length": 803.3333587646484,
      "epoch": 0.49028571428571427,
      "grad_norm": 0.38952067494392395,
      "kl": 0.01854705810546875,
      "learning_rate": 1.5415814221002265e-07,
      "loss": 0.0007,
      "reward": 0.3733687801286578,
      "reward_std": 0.6376638635993004,
      "rewards/cosine_scaled_reward": -0.11222685221582651,
      "rewards/format_reward": 1.0,
      "step": 429
    },
    {
      "completion_length": 1186.7291984558105,
      "epoch": 0.49142857142857144,
      "grad_norm": 1.133076548576355,
      "kl": 0.19187164306640625,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0077,
      "reward": 0.529668789356947,
      "reward_std": 0.6670900508761406,
      "rewards/cosine_scaled_reward": 0.011455949861556292,
      "rewards/format_reward": 0.9583333358168602,
      "step": 430
    },
    {
      "completion_length": 1090.479190826416,
      "epoch": 0.49257142857142855,
      "grad_norm": 2.763601064682007,
      "kl": 0.3094367980957031,
      "learning_rate": 1.5120838934595337e-07,
      "loss": 0.0124,
      "reward": 0.4001455968245864,
      "reward_std": 0.6080403476953506,
      "rewards/cosine_scaled_reward": -0.03941826708614826,
      "rewards/format_reward": 0.8958333507180214,
      "step": 431
    },
    {
      "completion_length": 1532.8541870117188,
      "epoch": 0.4937142857142857,
      "grad_norm": 1.8087928295135498,
      "kl": 0.3389739990234375,
      "learning_rate": 1.4976263201891613e-07,
      "loss": 0.0135,
      "reward": 0.2658024498960003,
      "reward_std": 0.7025109715759754,
      "rewards/cosine_scaled_reward": -0.10354285500943661,
      "rewards/format_reward": 0.8333333432674408,
      "step": 432
    },
    {
      "completion_length": 1383.3542022705078,
      "epoch": 0.4948571428571429,
      "grad_norm": 1.609007477760315,
      "kl": 0.202850341796875,
      "learning_rate": 1.483363816965435e-07,
      "loss": 0.0081,
      "reward": 0.47822056571021676,
      "reward_std": 0.6413930989801884,
      "rewards/cosine_scaled_reward": -0.032901763916015625,
      "rewards/format_reward": 0.9791666716337204,
      "step": 433
    },
    {
      "completion_length": 1405.7500381469727,
      "epoch": 0.496,
      "grad_norm": 1.7864830493927002,
      "kl": 0.16643524169921875,
      "learning_rate": 1.469297078922642e-07,
      "loss": 0.0067,
      "reward": 0.09692841861397028,
      "reward_std": 0.5686304904520512,
      "rewards/cosine_scaled_reward": -0.22341302986023948,
      "rewards/format_reward": 0.8541666865348816,
      "step": 434
    },
    {
      "completion_length": 894.8541984558105,
      "epoch": 0.49714285714285716,
      "grad_norm": 1.2933272123336792,
      "kl": 0.20261383056640625,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.0081,
      "reward": 0.3131554089486599,
      "reward_std": 0.5978329069912434,
      "rewards/cosine_scaled_reward": -0.12059530150145292,
      "rewards/format_reward": 0.9375000074505806,
      "step": 435
    },
    {
      "completion_length": 1181.6875343322754,
      "epoch": 0.4982857142857143,
      "grad_norm": 2.0181899070739746,
      "kl": 0.34568023681640625,
      "learning_rate": 1.4417536311769885e-07,
      "loss": 0.0138,
      "reward": 0.5736293898262375,
      "reward_std": 0.6847642697393894,
      "rewards/cosine_scaled_reward": 0.08606236800551414,
      "rewards/format_reward": 0.8750000149011612,
      "step": 436
    },
    {
      "completion_length": 1276.4791946411133,
      "epoch": 0.49942857142857144,
      "grad_norm": 2.564283847808838,
      "kl": 0.2025604248046875,
      "learning_rate": 1.4282782639029128e-07,
      "loss": 0.0081,
      "reward": 0.4678001292049885,
      "reward_std": 0.7040045224130154,
      "rewards/cosine_scaled_reward": -0.008855259045958519,
      "rewards/format_reward": 0.916666679084301,
      "step": 437
    },
    {
      "completion_length": 1573.0833587646484,
      "epoch": 0.5005714285714286,
      "grad_norm": 2.7367420196533203,
      "kl": 0.5104904174804688,
      "learning_rate": 1.4150013466019114e-07,
      "loss": 0.0204,
      "reward": 0.17007037345319986,
      "reward_std": 0.5956090353429317,
      "rewards/cosine_scaled_reward": -0.1865391266765073,
      "rewards/format_reward": 0.8750000055879354,
      "step": 438
    },
    {
      "completion_length": 1149.6875305175781,
      "epoch": 0.5017142857142857,
      "grad_norm": 1.5625011920928955,
      "kl": 0.2465057373046875,
      "learning_rate": 1.4019235263722034e-07,
      "loss": 0.0099,
      "reward": 0.29737665317952633,
      "reward_std": 0.6380414590239525,
      "rewards/cosine_scaled_reward": -0.1426175870001316,
      "rewards/format_reward": 0.9583333432674408,
      "step": 439
    },
    {
      "completion_length": 1237.354206085205,
      "epoch": 0.5028571428571429,
      "grad_norm": 5.108010768890381,
      "kl": 0.387542724609375,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0155,
      "reward": 0.17432179488241673,
      "reward_std": 0.6026643626391888,
      "rewards/cosine_scaled_reward": -0.20669769623782486,
      "rewards/format_reward": 0.9166666865348816,
      "step": 440
    },
    {
      "completion_length": 1345.958381652832,
      "epoch": 0.504,
      "grad_norm": 3.1474623680114746,
      "kl": 0.32805633544921875,
      "learning_rate": 1.3763677169699217e-07,
      "loss": 0.0131,
      "reward": 0.5082171498797834,
      "reward_std": 0.5840157195925713,
      "rewards/cosine_scaled_reward": 0.023840421810746193,
      "rewards/format_reward": 0.916666679084301,
      "step": 441
    },
    {
      "completion_length": 1032.1042022705078,
      "epoch": 0.5051428571428571,
      "grad_norm": 6.944667339324951,
      "kl": 0.20575714111328125,
      "learning_rate": 1.3638909733514452e-07,
      "loss": 0.0082,
      "reward": 0.43416818673722446,
      "reward_std": 0.6829791888594627,
      "rewards/cosine_scaled_reward": -0.0446524852886796,
      "rewards/format_reward": 0.9375000149011612,
      "step": 442
    },
    {
      "completion_length": 1498.2708587646484,
      "epoch": 0.5062857142857143,
      "grad_norm": 2.155977725982666,
      "kl": 0.36579132080078125,
      "learning_rate": 1.351615817851748e-07,
      "loss": 0.0147,
      "reward": 0.35938314939267,
      "reward_std": 0.6541690826416016,
      "rewards/cosine_scaled_reward": -0.07123458385467529,
      "rewards/format_reward": 0.8958333432674408,
      "step": 443
    },
    {
      "completion_length": 1179.6666870117188,
      "epoch": 0.5074285714285715,
      "grad_norm": 1.1800274848937988,
      "kl": 0.2142486572265625,
      "learning_rate": 1.3395428487445914e-07,
      "loss": 0.0086,
      "reward": 0.20739420503377914,
      "reward_std": 0.665216438472271,
      "rewards/cosine_scaled_reward": -0.16437909565865993,
      "rewards/format_reward": 0.8750000149011612,
      "step": 444
    },
    {
      "completion_length": 1355.6042098999023,
      "epoch": 0.5085714285714286,
      "grad_norm": 3.1479392051696777,
      "kl": 0.36163330078125,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0145,
      "reward": 0.3757004216313362,
      "reward_std": 0.7862649708986282,
      "rewards/cosine_scaled_reward": -0.08248981460928917,
      "rewards/format_reward": 0.9166666716337204,
      "step": 445
    },
    {
      "completion_length": 1242.4792175292969,
      "epoch": 0.5097142857142857,
      "grad_norm": 1.7266709804534912,
      "kl": 0.15660858154296875,
      "learning_rate": 1.316005813502869e-07,
      "loss": 0.0063,
      "reward": 0.41175389010459185,
      "reward_std": 0.6945022568106651,
      "rewards/cosine_scaled_reward": -0.06201653182506561,
      "rewards/format_reward": 0.9375000074505806,
      "step": 446
    },
    {
      "completion_length": 1084.6042098999023,
      "epoch": 0.5108571428571429,
      "grad_norm": 1.3974360227584839,
      "kl": 0.21521759033203125,
      "learning_rate": 1.3045428945301953e-07,
      "loss": 0.0086,
      "reward": 0.36824747081846,
      "reward_std": 0.5542174242436886,
      "rewards/cosine_scaled_reward": -0.06647356506437063,
      "rewards/format_reward": 0.916666679084301,
      "step": 447
    },
    {
      "completion_length": 1107.3125228881836,
      "epoch": 0.512,
      "grad_norm": 1.4814095497131348,
      "kl": 0.2049713134765625,
      "learning_rate": 1.2932844562179352e-07,
      "loss": 0.0082,
      "reward": 0.4160159872844815,
      "reward_std": 0.5563804637640715,
      "rewards/cosine_scaled_reward": -0.026282913982868195,
      "rewards/format_reward": 0.8958333507180214,
      "step": 448
    },
    {
      "completion_length": 1043.958351135254,
      "epoch": 0.5131428571428571,
      "grad_norm": 0.7390405535697937,
      "kl": 0.09143447875976562,
      "learning_rate": 1.2822310472864885e-07,
      "loss": 0.0037,
      "reward": 0.14539668832730968,
      "reward_std": 0.49666889011859894,
      "rewards/cosine_scaled_reward": -0.22875094041228294,
      "rewards/format_reward": 0.9375000149011612,
      "step": 449
    },
    {
      "completion_length": 1121.479175567627,
      "epoch": 0.5142857142857142,
      "grad_norm": 1.1870522499084473,
      "kl": 0.17742919921875,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.0071,
      "reward": 0.31525158043950796,
      "reward_std": 0.5219849050045013,
      "rewards/cosine_scaled_reward": -0.11353826522827148,
      "rewards/format_reward": 0.9375000074505806,
      "step": 450
    },
    {
      "completion_length": 1166.9583587646484,
      "epoch": 0.5154285714285715,
      "grad_norm": 1.581584095954895,
      "kl": 0.31623077392578125,
      "learning_rate": 1.260741462457165e-07,
      "loss": 0.0127,
      "reward": 0.34822911536321044,
      "reward_std": 0.6658046580851078,
      "rewards/cosine_scaled_reward": -0.1033438453450799,
      "rewards/format_reward": 0.9375000149011612,
      "step": 451
    },
    {
      "completion_length": 1243.687515258789,
      "epoch": 0.5165714285714286,
      "grad_norm": 2.1429476737976074,
      "kl": 0.2833709716796875,
      "learning_rate": 1.2503063339313356e-07,
      "loss": 0.0113,
      "reward": 0.6556259745266289,
      "reward_std": 0.8615197539329529,
      "rewards/cosine_scaled_reward": 0.09724390879273415,
      "rewards/format_reward": 0.9375000149011612,
      "step": 452
    },
    {
      "completion_length": 1214.7708740234375,
      "epoch": 0.5177142857142857,
      "grad_norm": 0.8739696741104126,
      "kl": 0.06499481201171875,
      "learning_rate": 1.2400783294793668e-07,
      "loss": 0.0026,
      "reward": 0.3872159831225872,
      "reward_std": 0.7029216475784779,
      "rewards/cosine_scaled_reward": -0.08512475527822971,
      "rewards/format_reward": 0.9583333432674408,
      "step": 453
    },
    {
      "completion_length": 1117.5417098999023,
      "epoch": 0.5188571428571429,
      "grad_norm": 2.8257322311401367,
      "kl": 0.198486328125,
      "learning_rate": 1.2300579475997657e-07,
      "loss": 0.0079,
      "reward": 0.3443953925743699,
      "reward_std": 0.6449627317488194,
      "rewards/cosine_scaled_reward": -0.09044338436797261,
      "rewards/format_reward": 0.9166666716337204,
      "step": 454
    },
    {
      "completion_length": 1421.6042022705078,
      "epoch": 0.52,
      "grad_norm": 1.7228955030441284,
      "kl": 0.4829292297363281,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.0193,
      "reward": 0.14169234223663807,
      "reward_std": 0.591641578823328,
      "rewards/cosine_scaled_reward": -0.18374849221436307,
      "rewards/format_reward": 0.8333333414047956,
      "step": 455
    },
    {
      "completion_length": 1353.6250267028809,
      "epoch": 0.5211428571428571,
      "grad_norm": 1.9811756610870361,
      "kl": 0.2818756103515625,
      "learning_rate": 1.2106419949317388e-07,
      "loss": 0.0113,
      "reward": 0.26744416062138043,
      "reward_std": 0.691377304494381,
      "rewards/cosine_scaled_reward": -0.15542185143567622,
      "rewards/format_reward": 0.9375000149011612,
      "step": 456
    },
    {
      "completion_length": 1305.8125305175781,
      "epoch": 0.5222857142857142,
      "grad_norm": 2.8912620544433594,
      "kl": 0.4429473876953125,
      "learning_rate": 1.2012473704494537e-07,
      "loss": 0.0177,
      "reward": 0.33940141764469445,
      "reward_std": 0.8498760014772415,
      "rewards/cosine_scaled_reward": -0.04779240628704429,
      "rewards/format_reward": 0.7916666753590107,
      "step": 457
    },
    {
      "completion_length": 1106.770866394043,
      "epoch": 0.5234285714285715,
      "grad_norm": 1.6600767374038696,
      "kl": 0.252227783203125,
      "learning_rate": 1.1920622611056974e-07,
      "loss": 0.0101,
      "reward": 0.3232785561122,
      "reward_std": 0.5980268009006977,
      "rewards/cosine_scaled_reward": -0.10486513609066606,
      "rewards/format_reward": 0.9166666716337204,
      "step": 458
    },
    {
      "completion_length": 1032.0625228881836,
      "epoch": 0.5245714285714286,
      "grad_norm": 1.426925778388977,
      "kl": 0.23378753662109375,
      "learning_rate": 1.1830871145697412e-07,
      "loss": 0.0094,
      "reward": 0.5453407493187115,
      "reward_std": 0.7394749782979488,
      "rewards/cosine_scaled_reward": 0.02821425348520279,
      "rewards/format_reward": 0.9375000074505806,
      "step": 459
    },
    {
      "completion_length": 1541.208366394043,
      "epoch": 0.5257142857142857,
      "grad_norm": 1.5413358211517334,
      "kl": 0.431365966796875,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.0172,
      "reward": 0.19518441951368004,
      "reward_std": 0.6402174085378647,
      "rewards/cosine_scaled_reward": -0.1528447875753045,
      "rewards/format_reward": 0.8333333395421505,
      "step": 460
    },
    {
      "completion_length": 1445.0000610351562,
      "epoch": 0.5268571428571428,
      "grad_norm": 1.153205156326294,
      "kl": 0.4829826354980469,
      "learning_rate": 1.1657684494105386e-07,
      "loss": 0.0194,
      "reward": 0.6377177089452744,
      "reward_std": 1.0267005078494549,
      "rewards/cosine_scaled_reward": 0.11436054221121594,
      "rewards/format_reward": 0.8541666772216558,
      "step": 461
    },
    {
      "completion_length": 989.8333511352539,
      "epoch": 0.528,
      "grad_norm": 0.7328640818595886,
      "kl": 0.04534912109375,
      "learning_rate": 1.1574257748745986e-07,
      "loss": 0.0018,
      "reward": 0.2644703108817339,
      "reward_std": 0.6658349372446537,
      "rewards/cosine_scaled_reward": -0.180008752271533,
      "rewards/format_reward": 0.9791666716337204,
      "step": 462
    },
    {
      "completion_length": 1494.3125457763672,
      "epoch": 0.5291428571428571,
      "grad_norm": 2.0038442611694336,
      "kl": 0.386627197265625,
      "learning_rate": 1.1492947512799328e-07,
      "loss": 0.0155,
      "reward": 0.5318789854645729,
      "reward_std": 0.9075686037540436,
      "rewards/cosine_scaled_reward": 0.03875172859989107,
      "rewards/format_reward": 0.8750000074505806,
      "step": 463
    },
    {
      "completion_length": 992.7500381469727,
      "epoch": 0.5302857142857142,
      "grad_norm": 1.684259295463562,
      "kl": 0.20226287841796875,
      "learning_rate": 1.1413757749211602e-07,
      "loss": 0.0081,
      "reward": 0.7598909301450476,
      "reward_std": 0.5075674168765545,
      "rewards/cosine_scaled_reward": 0.17828124994412065,
      "rewards/format_reward": 0.9583333358168602,
      "step": 464
    },
    {
      "completion_length": 1197.208366394043,
      "epoch": 0.5314285714285715,
      "grad_norm": 2.0846643447875977,
      "kl": 0.156219482421875,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0063,
      "reward": 0.36710425605997443,
      "reward_std": 0.7569303959608078,
      "rewards/cosine_scaled_reward": -0.09665499581024051,
      "rewards/format_reward": 0.9375000149011612,
      "step": 465
    },
    {
      "completion_length": 1219.9375381469727,
      "epoch": 0.5325714285714286,
      "grad_norm": 1.655044436454773,
      "kl": 0.1450958251953125,
      "learning_rate": 1.1261754973965422e-07,
      "loss": 0.0058,
      "reward": 0.589016193524003,
      "reward_std": 0.6661927625536919,
      "rewards/cosine_scaled_reward": 0.03179914876818657,
      "rewards/format_reward": 1.0,
      "step": 466
    },
    {
      "completion_length": 1338.2292137145996,
      "epoch": 0.5337142857142857,
      "grad_norm": 1.6192094087600708,
      "kl": 0.24442291259765625,
      "learning_rate": 1.1188949370707787e-07,
      "loss": 0.0098,
      "reward": 0.32355972472578287,
      "reward_std": 0.788430068641901,
      "rewards/cosine_scaled_reward": -0.11671263433527201,
      "rewards/format_reward": 0.9166666716337204,
      "step": 467
    },
    {
      "completion_length": 1452.9791870117188,
      "epoch": 0.5348571428571428,
      "grad_norm": 2.234537124633789,
      "kl": 0.47918701171875,
      "learning_rate": 1.1118279056249653e-07,
      "loss": 0.0191,
      "reward": 0.42969178780913353,
      "reward_std": 0.7676932998001575,
      "rewards/cosine_scaled_reward": -0.04107399005442858,
      "rewards/format_reward": 0.9166666865348816,
      "step": 468
    },
    {
      "completion_length": 1134.1250114440918,
      "epoch": 0.536,
      "grad_norm": 3.1756770610809326,
      "kl": 0.333251953125,
      "learning_rate": 1.1049747474962444e-07,
      "loss": 0.0133,
      "reward": 0.36575854755938053,
      "reward_std": 0.7719821371138096,
      "rewards/cosine_scaled_reward": -0.0800532667490188,
      "rewards/format_reward": 0.8958333507180214,
      "step": 469
    },
    {
      "completion_length": 1417.6042175292969,
      "epoch": 0.5371428571428571,
      "grad_norm": 1.5227336883544922,
      "kl": 0.470703125,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.0188,
      "reward": 0.3243638591375202,
      "reward_std": 0.8565945476293564,
      "rewards/cosine_scaled_reward": -0.0990256522782147,
      "rewards/format_reward": 0.8750000223517418,
      "step": 470
    },
    {
      "completion_length": 1389.9167098999023,
      "epoch": 0.5382857142857143,
      "grad_norm": 0.8987606167793274,
      "kl": 0.13502120971679688,
      "learning_rate": 1.0919113768029517e-07,
      "loss": 0.0054,
      "reward": 0.5736090987920761,
      "reward_std": 0.7181516140699387,
      "rewards/cosine_scaled_reward": 0.048229770036414266,
      "rewards/format_reward": 0.9375000074505806,
      "step": 471
    },
    {
      "completion_length": 1417.3125305175781,
      "epoch": 0.5394285714285715,
      "grad_norm": 1.6695373058319092,
      "kl": 0.477447509765625,
      "learning_rate": 1.0857018009286381e-07,
      "loss": 0.019,
      "reward": 0.2978296782821417,
      "reward_std": 0.8513829745352268,
      "rewards/cosine_scaled_reward": -0.10584009531885386,
      "rewards/format_reward": 0.8541666753590107,
      "step": 472
    },
    {
      "completion_length": 1296.083351135254,
      "epoch": 0.5405714285714286,
      "grad_norm": 1.4655067920684814,
      "kl": 0.205474853515625,
      "learning_rate": 1.0797073717209013e-07,
      "loss": 0.0082,
      "reward": 0.1657012979267165,
      "reward_std": 0.5747975409030914,
      "rewards/cosine_scaled_reward": -0.20028305146843195,
      "rewards/format_reward": 0.8958333358168602,
      "step": 473
    },
    {
      "completion_length": 1289.0833778381348,
      "epoch": 0.5417142857142857,
      "grad_norm": 2.321017026901245,
      "kl": 0.4053497314453125,
      "learning_rate": 1.0739283813397639e-07,
      "loss": 0.0162,
      "reward": 0.743865036405623,
      "reward_std": 0.733939703553915,
      "rewards/cosine_scaled_reward": 0.17521634395234287,
      "rewards/format_reward": 0.916666679084301,
      "step": 474
    },
    {
      "completion_length": 1455.7500228881836,
      "epoch": 0.5428571428571428,
      "grad_norm": 1.4909836053848267,
      "kl": 0.37940216064453125,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.0152,
      "reward": 0.3271640567108989,
      "reward_std": 0.7808557823300362,
      "rewards/cosine_scaled_reward": -0.08086569933220744,
      "rewards/format_reward": 0.8541666679084301,
      "step": 475
    },
    {
      "completion_length": 1167.2291793823242,
      "epoch": 0.544,
      "grad_norm": 2.0223841667175293,
      "kl": 0.2843170166015625,
      "learning_rate": 1.063017833182728e-07,
      "loss": 0.0114,
      "reward": 0.5520191243849695,
      "reward_std": 0.9013429544866085,
      "rewards/cosine_scaled_reward": 0.02163805003510788,
      "rewards/format_reward": 0.9375000074505806,
      "step": 476
    },
    {
      "completion_length": 1188.1041946411133,
      "epoch": 0.5451428571428572,
      "grad_norm": 0.9858556985855103,
      "kl": 0.3222007751464844,
      "learning_rate": 1.0578868071715544e-07,
      "loss": 0.0129,
      "reward": 0.6779396012425423,
      "reward_std": 0.7415244840085506,
      "rewards/cosine_scaled_reward": 0.11190658155828714,
      "rewards/format_reward": 0.9583333432674408,
      "step": 477
    },
    {
      "completion_length": 1321.9375381469727,
      "epoch": 0.5462857142857143,
      "grad_norm": 1.826804280281067,
      "kl": 0.29107666015625,
      "learning_rate": 1.0529722834905125e-07,
      "loss": 0.0116,
      "reward": 0.23478064546361566,
      "reward_std": 0.6792666539549828,
      "rewards/cosine_scaled_reward": -0.1663953149691224,
      "rewards/format_reward": 0.9166666865348816,
      "step": 478
    },
    {
      "completion_length": 1477.6041946411133,
      "epoch": 0.5474285714285714,
      "grad_norm": 2.102496862411499,
      "kl": 0.60791015625,
      "learning_rate": 1.0482745016665526e-07,
      "loss": 0.0243,
      "reward": 0.5314750894904137,
      "reward_std": 0.8037730753421783,
      "rewards/cosine_scaled_reward": 0.02458848152309656,
      "rewards/format_reward": 0.9166666716337204,
      "step": 479
    },
    {
      "completion_length": 1360.0208587646484,
      "epoch": 0.5485714285714286,
      "grad_norm": 1.7093435525894165,
      "kl": 0.4275360107421875,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0171,
      "reward": 0.3702342016622424,
      "reward_std": 0.6831196062266827,
      "rewards/cosine_scaled_reward": -0.07479118811897933,
      "rewards/format_reward": 0.9166666716337204,
      "step": 480
    },
    {
      "completion_length": 1480.3750534057617,
      "epoch": 0.5497142857142857,
      "grad_norm": 2.413257122039795,
      "kl": 0.6222076416015625,
      "learning_rate": 1.0395300688680625e-07,
      "loss": 0.0249,
      "reward": 0.1323686043615453,
      "reward_std": 0.6688967496156693,
      "rewards/cosine_scaled_reward": -0.17295160831417888,
      "rewards/format_reward": 0.7916666753590107,
      "step": 481
    },
    {
      "completion_length": 1245.3750381469727,
      "epoch": 0.5508571428571428,
      "grad_norm": 1.1108511686325073,
      "kl": 0.3742408752441406,
      "learning_rate": 1.0354838440848501e-07,
      "loss": 0.015,
      "reward": 0.5778923179022968,
      "reward_std": 0.7943699173629284,
      "rewards/cosine_scaled_reward": 0.05815020017325878,
      "rewards/format_reward": 0.9166666865348816,
      "step": 482
    },
    {
      "completion_length": 1345.7917022705078,
      "epoch": 0.552,
      "grad_norm": 3.296278238296509,
      "kl": 0.32867431640625,
      "learning_rate": 1.0316552135205837e-07,
      "loss": 0.0131,
      "reward": 0.4724856864195317,
      "reward_std": 0.7850163578987122,
      "rewards/cosine_scaled_reward": 0.012874770443886518,
      "rewards/format_reward": 0.8750000149011612,
      "step": 483
    },
    {
      "completion_length": 1077.2083625793457,
      "epoch": 0.5531428571428572,
      "grad_norm": 1.8179315328598022,
      "kl": 0.19476699829101562,
      "learning_rate": 1.0280443637773163e-07,
      "loss": 0.0078,
      "reward": 0.460749551653862,
      "reward_std": 0.6464533470571041,
      "rewards/cosine_scaled_reward": -0.013641191995702684,
      "rewards/format_reward": 0.9166666716337204,
      "step": 484
    },
    {
      "completion_length": 1128.0208740234375,
      "epoch": 0.5542857142857143,
      "grad_norm": 3.131510019302368,
      "kl": 0.27252197265625,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.0109,
      "reward": 0.29226525872945786,
      "reward_std": 0.67261578515172,
      "rewards/cosine_scaled_reward": -0.1184451412409544,
      "rewards/format_reward": 0.8958333432674408,
      "step": 485
    },
    {
      "completion_length": 904.479190826416,
      "epoch": 0.5554285714285714,
      "grad_norm": 2.3228588104248047,
      "kl": 0.23955535888671875,
      "learning_rate": 1.0214767000817596e-07,
      "loss": 0.0096,
      "reward": 0.4756124352570623,
      "reward_std": 0.6488616764545441,
      "rewards/cosine_scaled_reward": -0.003937863744795322,
      "rewards/format_reward": 0.9166666865348816,
      "step": 486
    },
    {
      "completion_length": 1126.666690826416,
      "epoch": 0.5565714285714286,
      "grad_norm": 1.264155626296997,
      "kl": 0.18535614013671875,
      "learning_rate": 1.0185202062281336e-07,
      "loss": 0.0074,
      "reward": 0.8187248446047306,
      "reward_std": 0.7255717366933823,
      "rewards/cosine_scaled_reward": 0.23560354206711054,
      "rewards/format_reward": 0.8958333432674408,
      "step": 487
    },
    {
      "completion_length": 975.666690826416,
      "epoch": 0.5577142857142857,
      "grad_norm": 1.1237510442733765,
      "kl": 0.089996337890625,
      "learning_rate": 1.0157821333772304e-07,
      "loss": 0.0036,
      "reward": 0.3682096116244793,
      "reward_std": 0.5310206785798073,
      "rewards/cosine_scaled_reward": -0.08794048149138689,
      "rewards/format_reward": 0.9583333432674408,
      "step": 488
    },
    {
      "completion_length": 1439.4167289733887,
      "epoch": 0.5588571428571428,
      "grad_norm": 1.723633050918579,
      "kl": 0.398651123046875,
      "learning_rate": 1.013262614978859e-07,
      "loss": 0.016,
      "reward": 0.12414154410362244,
      "reward_std": 0.5313785709440708,
      "rewards/cosine_scaled_reward": -0.21272556111216545,
      "rewards/format_reward": 0.8750000298023224,
      "step": 489
    },
    {
      "completion_length": 1332.0833740234375,
      "epoch": 0.56,
      "grad_norm": 1.4947898387908936,
      "kl": 0.33425140380859375,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.0134,
      "reward": 0.318634120747447,
      "reward_std": 0.5996614284813404,
      "rewards/cosine_scaled_reward": -0.1064935065805912,
      "rewards/format_reward": 0.9166666716337204,
      "step": 490
    },
    {
      "completion_length": 1447.0833587646484,
      "epoch": 0.5611428571428572,
      "grad_norm": 2.106166362762451,
      "kl": 0.3012504577636719,
      "learning_rate": 1.0088797220727779e-07,
      "loss": 0.0121,
      "reward": 0.4496698835864663,
      "reward_std": 0.8973959572613239,
      "rewards/cosine_scaled_reward": -0.014357679523527622,
      "rewards/format_reward": 0.8750000149011612,
      "step": 491
    },
    {
      "completion_length": 1104.6041831970215,
      "epoch": 0.5622857142857143,
      "grad_norm": 2.4530482292175293,
      "kl": 0.1375885009765625,
      "learning_rate": 1.0070165611810855e-07,
      "loss": 0.0055,
      "reward": 0.3822294343262911,
      "reward_std": 0.6128358133137226,
      "rewards/cosine_scaled_reward": -0.07346190325915813,
      "rewards/format_reward": 0.9375000074505806,
      "step": 492
    },
    {
      "completion_length": 1073.3541793823242,
      "epoch": 0.5634285714285714,
      "grad_norm": 1.1550400257110596,
      "kl": 0.18117523193359375,
      "learning_rate": 1.005372381963547e-07,
      "loss": 0.0072,
      "reward": 0.4816288612782955,
      "reward_std": 0.8117332980036736,
      "rewards/cosine_scaled_reward": -0.030845604138448834,
      "rewards/format_reward": 0.9583333432674408,
      "step": 493
    },
    {
      "completion_length": 1056.62504196167,
      "epoch": 0.5645714285714286,
      "grad_norm": 0.7965182662010193,
      "kl": 0.1199188232421875,
      "learning_rate": 1.0039472645551372e-07,
      "loss": 0.0048,
      "reward": 0.3902022670954466,
      "reward_std": 0.7675591520965099,
      "rewards/cosine_scaled_reward": -0.09161333832889795,
      "rewards/format_reward": 0.9583333358168602,
      "step": 494
    },
    {
      "completion_length": 1542.6042175292969,
      "epoch": 0.5657142857142857,
      "grad_norm": 2.6710989475250244,
      "kl": 0.34729766845703125,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.0139,
      "reward": 0.5254024369642138,
      "reward_std": 0.9325296804308891,
      "rewards/cosine_scaled_reward": 0.017028740607202053,
      "rewards/format_reward": 0.916666679084301,
      "step": 495
    },
    {
      "completion_length": 1316.7292022705078,
      "epoch": 0.5668571428571428,
      "grad_norm": 1.4742833375930786,
      "kl": 0.5881500244140625,
      "learning_rate": 1.0017544823184055e-07,
      "loss": 0.0235,
      "reward": 0.619772095582448,
      "reward_std": 0.8898240327835083,
      "rewards/cosine_scaled_reward": 0.13812840729951859,
      "rewards/format_reward": 0.7916666846722364,
      "step": 496
    },
    {
      "completion_length": 1185.708351135254,
      "epoch": 0.568,
      "grad_norm": 1.6387114524841309,
      "kl": 0.5859603881835938,
      "learning_rate": 1.0009869243631952e-07,
      "loss": 0.0235,
      "reward": 0.7584062092937529,
      "reward_std": 0.6907122731208801,
      "rewards/cosine_scaled_reward": 0.20853716135025024,
      "rewards/format_reward": 0.8750000111758709,
      "step": 497
    },
    {
      "completion_length": 1406.3958587646484,
      "epoch": 0.5691428571428572,
      "grad_norm": 2.6451492309570312,
      "kl": 0.383056640625,
      "learning_rate": 1.000438641958131e-07,
      "loss": 0.0153,
      "reward": 0.33473338062322,
      "reward_std": 0.7394461110234261,
      "rewards/cosine_scaled_reward": -0.10398351773619652,
      "rewards/format_reward": 0.916666679084301,
      "step": 498
    },
    {
      "completion_length": 1517.770881652832,
      "epoch": 0.5702857142857143,
      "grad_norm": 2.2294161319732666,
      "kl": 0.6592826843261719,
      "learning_rate": 1.0001096618257236e-07,
      "loss": 0.0264,
      "reward": 0.40126605180557817,
      "reward_std": 0.7697222679853439,
      "rewards/cosine_scaled_reward": -0.006456421993789263,
      "rewards/format_reward": 0.8125000186264515,
      "step": 499
    },
    {
      "completion_length": 1280.7500305175781,
      "epoch": 0.5714285714285714,
      "grad_norm": 2.027796983718872,
      "kl": 0.46686553955078125,
      "learning_rate": 1e-07,
      "loss": 0.0186,
      "reward": 0.2269948897883296,
      "reward_std": 0.7286034636199474,
      "rewards/cosine_scaled_reward": -0.13150667655281723,
      "rewards/format_reward": 0.8333333507180214,
      "step": 500
    },
    {
      "epoch": 0.5714285714285714,
      "step": 500,
      "total_flos": 0.0,
      "train_loss": 0.0027833052817963252,
      "train_runtime": 55581.6431,
      "train_samples_per_second": 0.432,
      "train_steps_per_second": 0.009
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}