{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5714285714285714,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 2571.2083587646484,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.19501760601997375,
      "kl": 0.0,
      "learning_rate": 2e-08,
      "loss": -0.0,
      "reward": 0.08349451050162315,
      "reward_std": 0.14101681299507618,
      "rewards/cosine_scaled_reward": -0.015534311532974243,
      "rewards/format_reward": 0.5208333488553762,
      "step": 1
    },
    {
      "completion_length": 2804.395881652832,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.18392972648143768,
      "kl": 0.0,
      "learning_rate": 4e-08,
      "loss": -0.0,
      "reward": 0.04647743375971913,
      "reward_std": 0.071280462667346,
      "rewards/cosine_scaled_reward": -0.04980122856795788,
      "rewards/format_reward": 0.37500000558793545,
      "step": 2
    },
    {
      "completion_length": 3326.5208435058594,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.16814576089382172,
      "kl": 4.73707914352417e-05,
      "learning_rate": 6e-08,
      "loss": 0.0,
      "reward": -0.05517918919213116,
      "reward_std": 0.06846281955949962,
      "rewards/cosine_scaled_reward": -0.23461355827748775,
      "rewards/format_reward": 0.1458333395421505,
      "step": 3
    },
    {
      "completion_length": 2271.854202270508,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.20840850472450256,
      "kl": 3.003329038619995e-05,
      "learning_rate": 8e-08,
      "loss": 0.0,
      "reward": 0.104565495159477,
      "reward_std": 0.1621289630420506,
      "rewards/cosine_scaled_reward": -0.016989090479910374,
      "rewards/format_reward": 0.6458333358168602,
      "step": 4
    },
    {
      "completion_length": 3269.250030517578,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.20661523938179016,
      "kl": 3.783777356147766e-05,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "reward": -0.027549213060410693,
      "reward_std": 0.11831692652776837,
      "rewards/cosine_scaled_reward": -0.22735669882968068,
      "rewards/format_reward": 0.29166667349636555,
      "step": 5
    },
    {
      "completion_length": 3207.125,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.25092315673828125,
      "kl": 4.430115222930908e-05,
      "learning_rate": 1.2e-07,
      "loss": 0.0,
      "reward": 0.0003576022572815418,
      "reward_std": 0.13864743057638407,
      "rewards/cosine_scaled_reward": -0.12470872118137777,
      "rewards/format_reward": 0.2500000074505806,
      "step": 6
    },
    {
      "completion_length": 3157.375045776367,
      "epoch": 0.008,
      "grad_norm": 0.17134779691696167,
      "kl": 2.024322748184204e-05,
      "learning_rate": 1.4e-07,
      "loss": 0.0,
      "reward": 0.024647740297950804,
      "reward_std": 0.12343645561486483,
      "rewards/cosine_scaled_reward": -0.10497256461530924,
      "rewards/format_reward": 0.354166679084301,
      "step": 7
    },
    {
      "completion_length": 2640.6875610351562,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.1863798052072525,
      "kl": 1.722201704978943e-05,
      "learning_rate": 1.6e-07,
      "loss": 0.0,
      "reward": 0.11913361493498087,
      "reward_std": 0.13311758637428284,
      "rewards/cosine_scaled_reward": 0.08812191832112148,
      "rewards/format_reward": 0.520833345130086,
      "step": 8
    },
    {
      "completion_length": 3171.4166870117188,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.20052321255207062,
      "kl": 4.750490188598633e-05,
      "learning_rate": 1.8e-07,
      "loss": 0.0,
      "reward": 0.04684965265914798,
      "reward_std": 0.10927984630689025,
      "rewards/cosine_scaled_reward": -0.030095582733338233,
      "rewards/format_reward": 0.3333333469927311,
      "step": 9
    },
    {
      "completion_length": 2619.479217529297,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.1900119185447693,
      "kl": 2.4830922484397888e-05,
      "learning_rate": 2e-07,
      "loss": 0.0,
      "reward": 0.0658606368524488,
      "reward_std": 0.13385961623862386,
      "rewards/cosine_scaled_reward": -0.01448088325560093,
      "rewards/format_reward": 0.4166666716337204,
      "step": 10
    },
    {
      "completion_length": 3247.4583740234375,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.15901124477386475,
      "kl": 2.709031105041504e-05,
      "learning_rate": 2.1999999999999998e-07,
      "loss": 0.0,
      "reward": -0.04511611349880695,
      "reward_std": 0.10903473664075136,
      "rewards/cosine_scaled_reward": -0.21695117373019457,
      "rewards/format_reward": 0.16666667349636555,
      "step": 11
    },
    {
      "completion_length": 2527.062515258789,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.21673277020454407,
      "kl": 4.7400593757629395e-05,
      "learning_rate": 2.4e-07,
      "loss": 0.0,
      "reward": 0.07390611409209669,
      "reward_std": 0.13868055026978254,
      "rewards/cosine_scaled_reward": -0.0959229115396738,
      "rewards/format_reward": 0.6250000074505806,
      "step": 12
    },
    {
      "completion_length": 2923.7083740234375,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.18947269022464752,
      "kl": 3.428012132644653e-05,
      "learning_rate": 2.6e-07,
      "loss": 0.0,
      "reward": 0.035209502559155226,
      "reward_std": 0.11472400068305433,
      "rewards/cosine_scaled_reward": -0.0937328040599823,
      "rewards/format_reward": 0.39583334140479565,
      "step": 13
    },
    {
      "completion_length": 2934.9166870117188,
      "epoch": 0.016,
      "grad_norm": 0.24321378767490387,
      "kl": 3.0156224966049194e-05,
      "learning_rate": 2.8e-07,
      "loss": 0.0,
      "reward": 0.022672508843243122,
      "reward_std": 0.12115806993097067,
      "rewards/cosine_scaled_reward": -0.1290947226807475,
      "rewards/format_reward": 0.39583333767950535,
      "step": 14
    },
    {
      "completion_length": 2780.0833587646484,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.2025020718574524,
      "kl": 2.3249536752700806e-05,
      "learning_rate": 3e-07,
      "loss": 0.0,
      "reward": 0.07990049698855728,
      "reward_std": 0.07795312092639506,
      "rewards/cosine_scaled_reward": 0.005295965820550919,
      "rewards/format_reward": 0.45833334140479565,
      "step": 15
    },
    {
      "completion_length": 3492.312530517578,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.18354374170303345,
      "kl": 4.486739635467529e-05,
      "learning_rate": 3.2e-07,
      "loss": 0.0,
      "reward": -0.06231466308236122,
      "reward_std": 0.08755358215421438,
      "rewards/cosine_scaled_reward": -0.22571247071027756,
      "rewards/format_reward": 0.0833333358168602,
      "step": 16
    },
    {
      "completion_length": 2405.56254196167,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.2691311836242676,
      "kl": 3.308616578578949e-05,
      "learning_rate": 3.4000000000000003e-07,
      "loss": 0.0,
      "reward": 0.07129185972735286,
      "reward_std": 0.13615732779726386,
      "rewards/cosine_scaled_reward": -0.07220900629181415,
      "rewards/format_reward": 0.562500013038516,
      "step": 17
    },
    {
      "completion_length": 2926.854232788086,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.16645942628383636,
      "kl": 2.5579705834388733e-05,
      "learning_rate": 3.6e-07,
      "loss": 0.0,
      "reward": 0.018077057087793946,
      "reward_std": 0.09778019832447171,
      "rewards/cosine_scaled_reward": -0.12368878477718681,
      "rewards/format_reward": 0.35416667349636555,
      "step": 18
    },
    {
      "completion_length": 2740.2083740234375,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.36254823207855225,
      "kl": 2.2433698177337646e-05,
      "learning_rate": 3.7999999999999996e-07,
      "loss": 0.0,
      "reward": 0.11815963860135525,
      "reward_std": 0.17526093125343323,
      "rewards/cosine_scaled_reward": 0.10508107638452202,
      "rewards/format_reward": 0.4791666753590107,
      "step": 19
    },
    {
      "completion_length": 2277.77091217041,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.19969557225704193,
      "kl": 1.2811273336410522e-05,
      "learning_rate": 4e-07,
      "loss": 0.0,
      "reward": 0.12608362920582294,
      "reward_std": 0.16070688236504793,
      "rewards/cosine_scaled_reward": 0.003691190853714943,
      "rewards/format_reward": 0.7291666865348816,
      "step": 20
    },
    {
      "completion_length": 2652.770896911621,
      "epoch": 0.024,
      "grad_norm": 0.28585541248321533,
      "kl": 4.544854164123535e-05,
      "learning_rate": 4.1999999999999995e-07,
      "loss": 0.0,
      "reward": 0.02955322596244514,
      "reward_std": 0.12703021708875895,
      "rewards/cosine_scaled_reward": -0.10260925628244877,
      "rewards/format_reward": 0.3750000037252903,
      "step": 21
    },
    {
      "completion_length": 1915.7292022705078,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.2661941349506378,
      "kl": 2.355128526687622e-05,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0,
      "reward": 0.11611313896719366,
      "reward_std": 0.1277361772954464,
      "rewards/cosine_scaled_reward": -0.013740219175815582,
      "rewards/format_reward": 0.7083333469927311,
      "step": 22
    },
    {
      "completion_length": 2549.1458892822266,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.2124091237783432,
      "kl": 3.099720925092697e-05,
      "learning_rate": 4.6e-07,
      "loss": 0.0,
      "reward": 0.05105366797943134,
      "reward_std": 0.1019338914193213,
      "rewards/cosine_scaled_reward": -0.10208869446069002,
      "rewards/format_reward": 0.500000013038516,
      "step": 23
    },
    {
      "completion_length": 2841.8333740234375,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.23832456767559052,
      "kl": 2.9023736715316772e-05,
      "learning_rate": 4.8e-07,
      "loss": 0.0,
      "reward": 0.055271712597459555,
      "reward_std": 0.15443201549351215,
      "rewards/cosine_scaled_reward": -0.05593502405099571,
      "rewards/format_reward": 0.43750000931322575,
      "step": 24
    },
    {
      "completion_length": 2734.6250228881836,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.20576398074626923,
      "kl": 4.264712333679199e-05,
      "learning_rate": 5e-07,
      "loss": 0.0,
      "reward": 0.03582445718348026,
      "reward_std": 0.14606032520532608,
      "rewards/cosine_scaled_reward": -0.0837564684334211,
      "rewards/format_reward": 0.3750000074505806,
      "step": 25
    },
    {
      "completion_length": 2954.3958740234375,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.16195496916770935,
      "kl": 3.225356340408325e-05,
      "learning_rate": 5.2e-07,
      "loss": 0.0,
      "reward": 0.06916803470812738,
      "reward_std": 0.08843644242733717,
      "rewards/cosine_scaled_reward": -0.023365769535303116,
      "rewards/format_reward": 0.45833334140479565,
      "step": 26
    },
    {
      "completion_length": 2985.125045776367,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.18544963002204895,
      "kl": 2.6202760636806488e-05,
      "learning_rate": 5.4e-07,
      "loss": 0.0,
      "reward": 0.0820373228052631,
      "reward_std": 0.15796185052022338,
      "rewards/cosine_scaled_reward": -0.017732856795191765,
      "rewards/format_reward": 0.5208333432674408,
      "step": 27
    },
    {
      "completion_length": 2774.625015258789,
      "epoch": 0.032,
      "grad_norm": 0.1868346929550171,
      "kl": 3.4227967262268066e-05,
      "learning_rate": 5.6e-07,
      "loss": 0.0,
      "reward": 0.09084612363949418,
      "reward_std": 0.08862769743427634,
      "rewards/cosine_scaled_reward": 0.03848753869533539,
      "rewards/format_reward": 0.4583333432674408,
      "step": 28
    },
    {
      "completion_length": 3335.4791870117188,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.1792682260274887,
      "kl": 2.7485191822052002e-05,
      "learning_rate": 5.8e-07,
      "loss": 0.0,
      "reward": -0.04747861542273313,
      "reward_std": 0.10171156749129295,
      "rewards/cosine_scaled_reward": -0.22336439974606037,
      "rewards/format_reward": 0.16666666977107525,
      "step": 29
    },
    {
      "completion_length": 3182.291717529297,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.20437775552272797,
      "kl": 2.625095658004284e-05,
      "learning_rate": 6e-07,
      "loss": 0.0,
      "reward": 0.04219546925742179,
      "reward_std": 0.16496449895203114,
      "rewards/cosine_scaled_reward": -0.04208953632041812,
      "rewards/format_reward": 0.33333334140479565,
      "step": 30
    },
    {
      "completion_length": 3098.729202270508,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.22590124607086182,
      "kl": 3.269314765930176e-05,
      "learning_rate": 6.2e-07,
      "loss": 0.0,
      "reward": 0.005523839652596507,
      "reward_std": 0.14406571350991726,
      "rewards/cosine_scaled_reward": -0.10929703339934349,
      "rewards/format_reward": 0.2500000111758709,
      "step": 31
    },
    {
      "completion_length": 3180.729248046875,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.1668621450662613,
      "kl": 2.413243055343628e-05,
      "learning_rate": 6.4e-07,
      "loss": 0.0,
      "reward": 0.02685558469966054,
      "reward_std": 0.11427591601386666,
      "rewards/cosine_scaled_reward": -0.07639202522113919,
      "rewards/format_reward": 0.31250000186264515,
      "step": 32
    },
    {
      "completion_length": 3349.291748046875,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.14419840276241302,
      "kl": 3.168731927871704e-05,
      "learning_rate": 6.6e-07,
      "loss": 0.0,
      "reward": 0.039358544163405895,
      "reward_std": 0.16947886534035206,
      "rewards/cosine_scaled_reward": -0.06123522081179544,
      "rewards/format_reward": 0.3541666753590107,
      "step": 33
    },
    {
      "completion_length": 2380.9167098999023,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.22925761342048645,
      "kl": 2.7898699045181274e-05,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0,
      "reward": 0.10769698303192854,
      "reward_std": 0.1343193519860506,
      "rewards/cosine_scaled_reward": 0.024825414642691612,
      "rewards/format_reward": 0.5833333358168602,
      "step": 34
    },
    {
      "completion_length": 2984.8959197998047,
      "epoch": 0.04,
      "grad_norm": 0.22632969915866852,
      "kl": 3.349222242832184e-05,
      "learning_rate": 7e-07,
      "loss": 0.0,
      "reward": 0.008388399612158537,
      "reward_std": 0.10677651688456535,
      "rewards/cosine_scaled_reward": -0.172616648953408,
      "rewards/format_reward": 0.3958333395421505,
      "step": 35
    },
    {
      "completion_length": 3379.3333740234375,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.1803259551525116,
      "kl": 3.5781413316726685e-05,
      "learning_rate": 7.2e-07,
      "loss": 0.0,
      "reward": -0.04249414807418361,
      "reward_std": 0.11586784245446324,
      "rewards/cosine_scaled_reward": -0.21885607368312776,
      "rewards/format_reward": 0.18750000186264515,
      "step": 36
    },
    {
      "completion_length": 3416.3541870117188,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.14628717303276062,
      "kl": 1.6780570149421692e-05,
      "learning_rate": 7.4e-07,
      "loss": 0.0,
      "reward": -0.050946102710440755,
      "reward_std": 0.07268051384016871,
      "rewards/cosine_scaled_reward": -0.22305289562791586,
      "rewards/format_reward": 0.1458333395421505,
      "step": 37
    },
    {
      "completion_length": 3168.875015258789,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.15368561446666718,
      "kl": 2.38809734582901e-05,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0,
      "reward": -0.01665067719295621,
      "reward_std": 0.06848278688266873,
      "rewards/cosine_scaled_reward": -0.14153539016842842,
      "rewards/format_reward": 0.1875,
      "step": 38
    },
    {
      "completion_length": 2897.9583740234375,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.17705340683460236,
      "kl": 1.7940998077392578e-05,
      "learning_rate": 7.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.05753035913221538,
      "reward_std": 0.07578674505930394,
      "rewards/cosine_scaled_reward": -0.04679988697171211,
      "rewards/format_reward": 0.4375000037252903,
      "step": 39
    },
    {
      "completion_length": 2554.7708892822266,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.22783087193965912,
      "kl": 2.1063722670078278e-05,
      "learning_rate": 8e-07,
      "loss": 0.0,
      "reward": 0.04392236044805031,
      "reward_std": 0.11113758757710457,
      "rewards/cosine_scaled_reward": -0.11080039292573929,
      "rewards/format_reward": 0.4791666753590107,
      "step": 40
    },
    {
      "completion_length": 2975.8958740234375,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.22396467626094818,
      "kl": 1.806439831852913e-05,
      "learning_rate": 8.199999999999999e-07,
      "loss": 0.0,
      "reward": -0.02207160711986944,
      "reward_std": 0.12583239562809467,
      "rewards/cosine_scaled_reward": -0.24189986009150743,
      "rewards/format_reward": 0.3541666753590107,
      "step": 41
    },
    {
      "completion_length": 2725.9166984558105,
      "epoch": 0.048,
      "grad_norm": 0.27289333939552307,
      "kl": 3.649294376373291e-05,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.0,
      "reward": -0.031094663951080292,
      "reward_std": 0.07978959055617452,
      "rewards/cosine_scaled_reward": -0.28885440342128277,
      "rewards/format_reward": 0.3958333395421505,
      "step": 42
    },
    {
      "completion_length": 2959.916702270508,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.20727042853832245,
      "kl": 2.0106323063373566e-05,
      "learning_rate": 8.599999999999999e-07,
      "loss": 0.0,
      "reward": 0.02459817798808217,
      "reward_std": 0.1277715265750885,
      "rewards/cosine_scaled_reward": -0.08350535575300455,
      "rewards/format_reward": 0.3125000037252903,
      "step": 43
    },
    {
      "completion_length": 2700.6458740234375,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.2838585376739502,
      "kl": 5.9839338064193726e-05,
      "learning_rate": 8.799999999999999e-07,
      "loss": 0.0,
      "reward": 0.07408008817583323,
      "reward_std": 0.1297681350260973,
      "rewards/cosine_scaled_reward": -0.011715584434568882,
      "rewards/format_reward": 0.45833334140479565,
      "step": 44
    },
    {
      "completion_length": 3354.8125610351562,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.1483648717403412,
      "kl": 1.9919127225875854e-05,
      "learning_rate": 9e-07,
      "loss": 0.0,
      "reward": 0.042736097471788526,
      "reward_std": 0.13064279220998287,
      "rewards/cosine_scaled_reward": -0.017792840400943533,
      "rewards/format_reward": 0.2916666716337204,
      "step": 45
    },
    {
      "completion_length": 3156.3959045410156,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.18322643637657166,
      "kl": 2.984795719385147e-05,
      "learning_rate": 9.2e-07,
      "loss": 0.0,
      "reward": -0.0388933519134298,
      "reward_std": 0.09174182126298547,
      "rewards/cosine_scaled_reward": -0.21921097254380584,
      "rewards/format_reward": 0.2083333358168602,
      "step": 46
    },
    {
      "completion_length": 2675.1042098999023,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.22902333736419678,
      "kl": 3.225822001695633e-05,
      "learning_rate": 9.399999999999999e-07,
      "loss": 0.0,
      "reward": 0.11461905902251601,
      "reward_std": 0.14914221363142133,
      "rewards/cosine_scaled_reward": 0.06888513453304768,
      "rewards/format_reward": 0.5416666809469461,
      "step": 47
    },
    {
      "completion_length": 2940.250015258789,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.1693173050880432,
      "kl": 9.981123730540276e-05,
      "learning_rate": 9.6e-07,
      "loss": 0.0,
      "reward": 0.029834913242666516,
      "reward_std": 0.1260078912600875,
      "rewards/cosine_scaled_reward": -0.07712742034345865,
      "rewards/format_reward": 0.3333333358168602,
      "step": 48
    },
    {
      "completion_length": 2305.0000381469727,
      "epoch": 0.056,
      "grad_norm": 0.2164977639913559,
      "kl": 7.113814353942871e-05,
      "learning_rate": 9.8e-07,
      "loss": 0.0,
      "reward": 0.06258401460945606,
      "reward_std": 0.12052545137703419,
      "rewards/cosine_scaled_reward": -0.0762196818832308,
      "rewards/format_reward": 0.5208333376795053,
      "step": 49
    },
    {
      "completion_length": 2923.770866394043,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.165934756398201,
      "kl": 5.564838647842407e-05,
      "learning_rate": 1e-06,
      "loss": 0.0,
      "reward": 0.02972496929578483,
      "reward_std": 0.1184019073843956,
      "rewards/cosine_scaled_reward": -0.07874834412359633,
      "rewards/format_reward": 0.3333333358168602,
      "step": 50
    },
    {
      "completion_length": 2164.1250076293945,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.28015246987342834,
      "kl": 0.00023446721024811268,
      "learning_rate": 9.999890338174275e-07,
      "loss": 0.0,
      "reward": 0.06518604746088386,
      "reward_std": 0.08450535265728831,
      "rewards/cosine_scaled_reward": -0.08784076571464539,
      "rewards/format_reward": 0.5625000074505806,
      "step": 51
    },
    {
      "completion_length": 2815.583396911621,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.22428154945373535,
      "kl": 0.00012213829904794693,
      "learning_rate": 9.999561358041868e-07,
      "loss": 0.0,
      "reward": 0.08840094119659625,
      "reward_std": 0.17651066416874528,
      "rewards/cosine_scaled_reward": 0.02174525521695614,
      "rewards/format_reward": 0.47916667722165585,
      "step": 52
    },
    {
      "completion_length": 2765.291717529297,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.20454645156860352,
      "kl": 0.00013817846775054932,
      "learning_rate": 9.999013075636804e-07,
      "loss": 0.0,
      "reward": 0.13577738002641127,
      "reward_std": 0.15497551951557398,
      "rewards/cosine_scaled_reward": 0.10726968757808208,
      "rewards/format_reward": 0.5833333376795053,
      "step": 53
    },
    {
      "completion_length": 2602.104217529297,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.17332378029823303,
      "kl": 6.633996963500977e-05,
      "learning_rate": 9.998245517681593e-07,
      "loss": 0.0,
      "reward": 0.17019405495375395,
      "reward_std": 0.14906789222732186,
      "rewards/cosine_scaled_reward": 0.18696625716984272,
      "rewards/format_reward": 0.6250000167638063,
      "step": 54
    },
    {
      "completion_length": 2931.2708587646484,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.1711331158876419,
      "kl": 8.093938231468201e-05,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0,
      "reward": 0.06622584909200668,
      "reward_std": 0.12079534726217389,
      "rewards/cosine_scaled_reward": -0.012890823185443878,
      "rewards/format_reward": 0.41666666977107525,
      "step": 55
    },
    {
      "completion_length": 2785.833396911621,
      "epoch": 0.064,
      "grad_norm": 0.18859043717384338,
      "kl": 3.4786760807037354e-05,
      "learning_rate": 9.996052735444862e-07,
      "loss": 0.0,
      "reward": 0.058590125758200884,
      "reward_std": 0.0908779576420784,
      "rewards/cosine_scaled_reward": -0.045839957892894745,
      "rewards/format_reward": 0.43750001303851604,
      "step": 56
    },
    {
      "completion_length": 3269.0208740234375,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.12594451010227203,
      "kl": 1.267390325665474e-05,
      "learning_rate": 9.994627618036452e-07,
      "loss": 0.0,
      "reward": 0.023180216550827026,
      "reward_std": 0.12568830093368888,
      "rewards/cosine_scaled_reward": -0.09830697299912572,
      "rewards/format_reward": 0.3333333358168602,
      "step": 57
    },
    {
      "completion_length": 2111.645866394043,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.2272089719772339,
      "kl": 0.0004666820168495178,
      "learning_rate": 9.992983438818915e-07,
      "loss": 0.0,
      "reward": 0.1490978323854506,
      "reward_std": 0.143410362303257,
      "rewards/cosine_scaled_reward": 0.06337156053632498,
      "rewards/format_reward": 0.7500000149011612,
      "step": 58
    },
    {
      "completion_length": 2783.1250228881836,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.17396898567676544,
      "kl": 2.6125460863113403e-05,
      "learning_rate": 9.991120277927223e-07,
      "loss": 0.0,
      "reward": 0.01869871746748686,
      "reward_std": 0.12852068059146404,
      "rewards/cosine_scaled_reward": -0.1118423049338162,
      "rewards/format_reward": 0.3333333358168602,
      "step": 59
    },
    {
      "completion_length": 2860.500045776367,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.1613863706588745,
      "kl": 6.277300417423248e-05,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0,
      "reward": 0.036287183640524745,
      "reward_std": 0.12081348802894354,
      "rewards/cosine_scaled_reward": -0.10168062068987638,
      "rewards/format_reward": 0.41666667349636555,
      "step": 60
    },
    {
      "completion_length": 2981.479217529297,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.16875192523002625,
      "kl": 0.00013221928384155035,
      "learning_rate": 9.98673738502114e-07,
      "loss": 0.0,
      "reward": 0.050741570768877864,
      "reward_std": 0.14027578197419643,
      "rewards/cosine_scaled_reward": -0.10068160435184836,
      "rewards/format_reward": 0.5000000055879354,
      "step": 61
    },
    {
      "completion_length": 2615.1042251586914,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.19548830389976501,
      "kl": 0.0005001500248908997,
      "learning_rate": 9.98421786662277e-07,
      "loss": 0.0,
      "reward": 0.09628129447810352,
      "reward_std": 0.09381117532029748,
      "rewards/cosine_scaled_reward": -0.006604377180337906,
      "rewards/format_reward": 0.583333333954215,
      "step": 62
    },
    {
      "completion_length": 2180.791763305664,
      "epoch": 0.072,
      "grad_norm": 0.2137078046798706,
      "kl": 0.0017590373754501343,
      "learning_rate": 9.981479793771866e-07,
      "loss": 0.0001,
      "reward": 0.18858059402555227,
      "reward_std": 0.1375174829736352,
      "rewards/cosine_scaled_reward": 0.18239261582493782,
      "rewards/format_reward": 0.7500000074505806,
      "step": 63
    },
    {
      "completion_length": 2905.479217529297,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.17932051420211792,
      "kl": 0.0002357116900384426,
      "learning_rate": 9.97852329991824e-07,
      "loss": 0.0,
      "reward": 0.041001017816597596,
      "reward_std": 0.1474486207589507,
      "rewards/cosine_scaled_reward": -0.05582154542207718,
      "rewards/format_reward": 0.35416667722165585,
      "step": 64
    },
    {
      "completion_length": 2707.2917137145996,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.18318000435829163,
      "kl": 0.0001243998558493331,
      "learning_rate": 9.975348529157229e-07,
      "loss": 0.0,
      "reward": 0.05281507736071944,
      "reward_std": 0.10035080322995782,
      "rewards/cosine_scaled_reward": -0.08306887093931437,
      "rewards/format_reward": 0.47916666977107525,
      "step": 65
    },
    {
      "completion_length": 2048.8333435058594,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.2723003029823303,
      "kl": 0.0007075890898704529,
      "learning_rate": 9.971955636222684e-07,
      "loss": 0.0,
      "reward": 0.08446738217025995,
      "reward_std": 0.12608648044988513,
      "rewards/cosine_scaled_reward": -0.031810659915208817,
      "rewards/format_reward": 0.5625000018626451,
      "step": 66
    },
    {
      "completion_length": 3272.250015258789,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.14235161244869232,
      "kl": 0.0003021508455276489,
      "learning_rate": 9.968344786479415e-07,
      "loss": 0.0,
      "reward": -0.09321667347103357,
      "reward_std": 0.06564409867860377,
      "rewards/cosine_scaled_reward": -0.358583465218544,
      "rewards/format_reward": 0.16666667349636555,
      "step": 67
    },
    {
      "completion_length": 1971.250015258789,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.25673460960388184,
      "kl": 0.001732461154460907,
      "learning_rate": 9.964516155915151e-07,
      "loss": 0.0001,
      "reward": 0.10094248503446579,
      "reward_std": 0.14019617764279246,
      "rewards/cosine_scaled_reward": -0.018839816562831402,
      "rewards/format_reward": 0.6250000074505806,
      "step": 68
    },
    {
      "completion_length": 2486.1666870117188,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.24227525293827057,
      "kl": 0.0012736618518829346,
      "learning_rate": 9.960469931131936e-07,
      "loss": 0.0001,
      "reward": 0.002093482413329184,
      "reward_std": 0.1131673906929791,
      "rewards/cosine_scaled_reward": -0.22560235299170017,
      "rewards/format_reward": 0.45833334513008595,
      "step": 69
    },
    {
      "completion_length": 3097.7708740234375,
      "epoch": 0.08,
      "grad_norm": 0.16386379301548004,
      "kl": 0.0010862918570637703,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0,
      "reward": 0.003557512885890901,
      "reward_std": 0.07532872771844268,
      "rewards/cosine_scaled_reward": -0.1663934402167797,
      "rewards/format_reward": 0.35416666977107525,
      "step": 70
    },
    {
      "completion_length": 2676.9166946411133,
      "epoch": 0.08114285714285714,
      "grad_norm": 0.3169231712818146,
      "kl": 0.0008103922009468079,
      "learning_rate": 9.951725498333448e-07,
      "loss": 0.0,
      "reward": 0.057090925984084606,
      "reward_std": 0.12368696788325906,
      "rewards/cosine_scaled_reward": -0.009727515280246735,
      "rewards/format_reward": 0.35416666977107525,
      "step": 71
    },
    {
      "completion_length": 2502.604202270508,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.22013245522975922,
      "kl": 0.0007383376359939575,
      "learning_rate": 9.947027716509488e-07,
      "loss": 0.0,
      "reward": 0.034887210465967655,
      "reward_std": 0.1007420509122312,
      "rewards/cosine_scaled_reward": -0.14844494126737118,
      "rewards/format_reward": 0.5000000149011612,
      "step": 72
    },
    {
      "completion_length": 3457.125,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.14216576516628265,
      "kl": 0.00022399425506591797,
      "learning_rate": 9.942113192828444e-07,
      "loss": 0.0,
      "reward": 0.0063769330736249685,
      "reward_std": 0.0942028573481366,
      "rewards/cosine_scaled_reward": -0.07475030946079642,
      "rewards/format_reward": 0.1875000074505806,
      "step": 73
    },
    {
      "completion_length": 3151.3125610351562,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.16055600345134735,
      "kl": 0.0008836947381496429,
      "learning_rate": 9.93698216681727e-07,
      "loss": 0.0,
      "reward": 0.08290100377053022,
      "reward_std": 0.16621626261621714,
      "rewards/cosine_scaled_reward": 0.06829957757145166,
      "rewards/format_reward": 0.35416666977107525,
      "step": 74
    },
    {
      "completion_length": 2834.812530517578,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.17003442347049713,
      "kl": 0.0006369650363922119,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0,
      "reward": 0.05851969541981816,
      "reward_std": 0.09551909612491727,
      "rewards/cosine_scaled_reward": -0.021192173473536968,
      "rewards/format_reward": 0.37500000186264515,
      "step": 75
    },
    {
      "completion_length": 2987.1458435058594,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.17524327337741852,
      "kl": 0.0001463182270526886,
      "learning_rate": 9.926071618660237e-07,
      "loss": 0.0,
      "reward": -0.009090241976082325,
      "reward_std": 0.06974720861762762,
      "rewards/cosine_scaled_reward": -0.21435680365539156,
      "rewards/format_reward": 0.3750000037252903,
      "step": 76
    },
    {
      "completion_length": 3154.2708740234375,
      "epoch": 0.088,
      "grad_norm": 0.1655508130788803,
      "kl": 0.0002579297870397568,
      "learning_rate": 9.9202926282791e-07,
      "loss": 0.0,
      "reward": -0.01178191090002656,
      "reward_std": 0.10129856411367655,
      "rewards/cosine_scaled_reward": -0.1495479578152299,
      "rewards/format_reward": 0.2291666753590107,
      "step": 77
    },
    {
      "completion_length": 3120.6041870117188,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.1709078699350357,
      "kl": 0.0009910427033901215,
      "learning_rate": 9.91429819907136e-07,
      "loss": 0.0,
      "reward": 0.03289509087335318,
      "reward_std": 0.09486977197229862,
      "rewards/cosine_scaled_reward": -0.06091844476759434,
      "rewards/format_reward": 0.3125000074505806,
      "step": 78
    },
    {
      "completion_length": 2322.7708702087402,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.2289884388446808,
      "kl": 0.001061469316482544,
      "learning_rate": 9.908088623197048e-07,
      "loss": 0.0,
      "reward": 0.08358410373330116,
      "reward_std": 0.09495366807095706,
      "rewards/cosine_scaled_reward": -0.00746832974255085,
      "rewards/format_reward": 0.5000000055879354,
      "step": 79
    },
    {
      "completion_length": 3293.2083740234375,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.18912553787231445,
      "kl": 0.0006625503301620483,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.0,
      "reward": -0.006389252142980695,
      "reward_std": 0.13851621747016907,
      "rewards/cosine_scaled_reward": -0.1551999393850565,
      "rewards/format_reward": 0.2708333358168602,
      "step": 80
    },
    {
      "completion_length": 3064.145866394043,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.22587524354457855,
      "kl": 0.002162039279937744,
      "learning_rate": 9.895025252503755e-07,
      "loss": 0.0001,
      "reward": -0.021476033609360456,
      "reward_std": 0.09152220841497183,
      "rewards/cosine_scaled_reward": -0.19870448019355536,
      "rewards/format_reward": 0.27083333767950535,
      "step": 81
    },
    {
      "completion_length": 2900.9791717529297,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.19733589887619019,
      "kl": 0.000873371958732605,
      "learning_rate": 9.888172094375033e-07,
      "loss": 0.0,
      "reward": 0.03715384565293789,
      "reward_std": 0.1097968677058816,
      "rewards/cosine_scaled_reward": -0.05757363699376583,
      "rewards/format_reward": 0.33333333395421505,
      "step": 82
    },
    {
      "completion_length": 2747.500045776367,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.23122522234916687,
      "kl": 0.0008766204118728638,
      "learning_rate": 9.881105062929221e-07,
      "loss": 0.0,
      "reward": 0.021435680333524942,
      "reward_std": 0.12779827043414116,
      "rewards/cosine_scaled_reward": -0.11562954680994153,
      "rewards/format_reward": 0.35416666977107525,
      "step": 83
    },
    {
      "completion_length": 3090.750045776367,
      "epoch": 0.096,
      "grad_norm": 0.1742715984582901,
      "kl": 0.00035139918327331543,
      "learning_rate": 9.873824502603459e-07,
      "loss": 0.0,
      "reward": 0.061643086373806,
      "reward_std": 0.16731480974704027,
      "rewards/cosine_scaled_reward": -0.016695552330929786,
      "rewards/format_reward": 0.3958333432674408,
      "step": 84
    },
    {
      "completion_length": 2862.5000762939453,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.15780915319919586,
      "kl": 0.00032967329025268555,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0,
      "reward": 0.059395642252638936,
      "reward_std": 0.18405951745808125,
      "rewards/cosine_scaled_reward": -0.07413570675998926,
      "rewards/format_reward": 0.5000000093132257,
      "step": 85
    },
    {
      "completion_length": 2822.458366394043,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.18948958814144135,
      "kl": 0.0012461543083190918,
      "learning_rate": 9.85862422507884e-07,
      "loss": 0.0,
      "reward": 0.058563592843711376,
      "reward_std": 0.13514422718435526,
      "rewards/cosine_scaled_reward": -0.023547479882836342,
      "rewards/format_reward": 0.3958333432674408,
      "step": 86
    },
    {
      "completion_length": 2582.0208892822266,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.24197638034820557,
      "kl": 0.001571571920067072,
      "learning_rate": 9.850705248720068e-07,
      "loss": 0.0001,
      "reward": 0.06321019981987774,
      "reward_std": 0.1301463134586811,
      "rewards/cosine_scaled_reward": -0.0854906840249896,
      "rewards/format_reward": 0.5416666679084301,
      "step": 87
    },
    {
      "completion_length": 2717.666717529297,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.2206323742866516,
      "kl": 0.0015416741371154785,
      "learning_rate": 9.8425742251254e-07,
      "loss": 0.0001,
      "reward": 0.10002893407363445,
      "reward_std": 0.15120992343872786,
      "rewards/cosine_scaled_reward": 0.02375661302357912,
      "rewards/format_reward": 0.5416666902601719,
      "step": 88
    },
    {
      "completion_length": 3077.7708587646484,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.18466134369373322,
      "kl": 0.0015412569046020508,
      "learning_rate": 9.83423155058946e-07,
      "loss": 0.0001,
      "reward": 0.019450924504781142,
      "reward_std": 0.1520949569530785,
      "rewards/cosine_scaled_reward": -0.10057051916373894,
      "rewards/format_reward": 0.31250000931322575,
      "step": 89
    },
    {
      "completion_length": 2275.2708587646484,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.317700058221817,
      "kl": 0.0030652284622192383,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0001,
      "reward": 0.020127289928495884,
      "reward_std": 0.09609310049563646,
      "rewards/cosine_scaled_reward": -0.2033098302781582,
      "rewards/format_reward": 0.5208333395421505,
      "step": 90
    },
    {
      "completion_length": 3104.2291870117188,
      "epoch": 0.104,
      "grad_norm": 0.16374754905700684,
      "kl": 0.0012220889329910278,
      "learning_rate": 9.816912885430258e-07,
      "loss": 0.0,
      "reward": 0.02835351601243019,
      "reward_std": 0.0958831796888262,
      "rewards/cosine_scaled_reward": -0.08190420269966125,
      "rewards/format_reward": 0.3333333432674408,
      "step": 91
    },
    {
      "completion_length": 2564.9583892822266,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.23695100843906403,
      "kl": 0.0031346678733825684,
      "learning_rate": 9.807937738894303e-07,
      "loss": 0.0001,
      "reward": 0.031697872560471296,
      "reward_std": 0.1256707413122058,
      "rewards/cosine_scaled_reward": -0.1565206847153604,
      "rewards/format_reward": 0.5000000055879354,
      "step": 92
    },
    {
      "completion_length": 3429.2916870117188,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.18266721069812775,
      "kl": 0.0016658008098602295,
      "learning_rate": 9.798752629550546e-07,
      "loss": 0.0001,
      "reward": -0.06380185973830521,
      "reward_std": 0.07419165363535285,
      "rewards/cosine_scaled_reward": -0.21879931539297104,
      "rewards/format_reward": 0.06250000186264515,
      "step": 93
    },
    {
      "completion_length": 2926.041717529297,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.183439701795578,
      "kl": 0.0018378198146820068,
      "learning_rate": 9.78935800506826e-07,
      "loss": 0.0001,
      "reward": 0.02929869778745342,
      "reward_std": 0.0936040470842272,
      "rewards/cosine_scaled_reward": -0.0937141003087163,
      "rewards/format_reward": 0.3541666716337204,
      "step": 94
    },
    {
      "completion_length": 3466.7708740234375,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.1366991549730301,
      "kl": 0.0007553547620773315,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.0,
      "reward": -0.011318721110001206,
      "reward_std": 0.11944798147305846,
      "rewards/cosine_scaled_reward": -0.1375442687422037,
      "rewards/format_reward": 0.20833333767950535,
      "step": 95
    },
    {
      "completion_length": 2708.729202270508,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.17744414508342743,
      "kl": 0.002089708112180233,
      "learning_rate": 9.769942052400235e-07,
      "loss": 0.0001,
      "reward": 0.08605838101357222,
      "reward_std": 0.11997871845960617,
      "rewards/cosine_scaled_reward": 0.024741460452787578,
      "rewards/format_reward": 0.45833334140479565,
      "step": 96
    },
    {
      "completion_length": 3173.854248046875,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.18968307971954346,
      "kl": 0.0014801472425460815,
      "learning_rate": 9.759921670520634e-07,
      "loss": 0.0001,
      "reward": 0.031303239753469825,
      "reward_std": 0.11792595777660608,
      "rewards/cosine_scaled_reward": -0.06253299303352833,
      "rewards/format_reward": 0.31250000931322575,
      "step": 97
    },
    {
      "completion_length": 2743.937545776367,
      "epoch": 0.112,
      "grad_norm": 0.17238333821296692,
      "kl": 0.0005875229835510254,
      "learning_rate": 9.749693666068663e-07,
      "loss": 0.0,
      "reward": 0.03780742874369025,
      "reward_std": 0.09086114913225174,
      "rewards/cosine_scaled_reward": -0.14788446575403214,
      "rewards/format_reward": 0.5208333432674408,
      "step": 98
    },
    {
      "completion_length": 2848.7083587646484,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.24863041937351227,
      "kl": 0.0012496709823608398,
      "learning_rate": 9.739258537542835e-07,
      "loss": 0.0001,
      "reward": 0.007481225358787924,
      "reward_std": 0.09957948396913707,
      "rewards/cosine_scaled_reward": -0.11191552877426147,
      "rewards/format_reward": 0.2708333395421505,
      "step": 99
    },
    {
      "completion_length": 2628.6458740234375,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.2551541328430176,
      "kl": 0.007792949676513672,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0003,
      "reward": 0.11142967082560062,
      "reward_std": 0.1601211791858077,
      "rewards/cosine_scaled_reward": 0.0880345068871975,
      "rewards/format_reward": 0.4791666828095913,
      "step": 100
    },
    {
      "completion_length": 2490.229202270508,
      "epoch": 0.11542857142857142,
      "grad_norm": 0.21603932976722717,
      "kl": 0.0017938017845153809,
      "learning_rate": 9.717768952713511e-07,
      "loss": 0.0001,
      "reward": 0.05852094758301973,
      "reward_std": 0.09475950035266578,
      "rewards/cosine_scaled_reward": -0.0666491650044918,
      "rewards/format_reward": 0.47916666977107525,
      "step": 101
    },
    {
      "completion_length": 2286.395866394043,
      "epoch": 0.11657142857142858,
      "grad_norm": 0.22003820538520813,
      "kl": 0.0035685300827026367,
      "learning_rate": 9.706715543782064e-07,
      "loss": 0.0001,
      "reward": 0.06951136235147715,
      "reward_std": 0.11986843310296535,
      "rewards/cosine_scaled_reward": -0.09702310990542173,
      "rewards/format_reward": 0.6041666716337204,
      "step": 102
    },
    {
      "completion_length": 2706.5833854675293,
      "epoch": 0.11771428571428572,
      "grad_norm": 0.24211692810058594,
      "kl": 0.0033190250396728516,
      "learning_rate": 9.695457105469804e-07,
      "loss": 0.0001,
      "reward": 0.02106085862033069,
      "reward_std": 0.13761026575230062,
      "rewards/cosine_scaled_reward": -0.1569200656376779,
      "rewards/format_reward": 0.43750000931322575,
      "step": 103
    },
    {
      "completion_length": 2745.125011444092,
      "epoch": 0.11885714285714286,
      "grad_norm": 0.2112489640712738,
      "kl": 0.0039566755294799805,
      "learning_rate": 9.683994186497132e-07,
      "loss": 0.0002,
      "reward": 0.013099167263135314,
      "reward_std": 0.1101553007028997,
      "rewards/cosine_scaled_reward": -0.14944082498550415,
      "rewards/format_reward": 0.3750000037252903,
      "step": 104
    },
    {
      "completion_length": 2631.812545776367,
      "epoch": 0.12,
      "grad_norm": 0.19800451397895813,
      "kl": 0.002551555633544922,
      "learning_rate": 9.672327345550543e-07,
      "loss": 0.0001,
      "reward": 0.08774281479418278,
      "reward_std": 0.15658214688301086,
      "rewards/cosine_scaled_reward": 0.02480911184102297,
      "rewards/format_reward": 0.4583333395421505,
      "step": 105
    },
    {
      "completion_length": 2240.000026702881,
      "epoch": 0.12114285714285715,
      "grad_norm": 0.1994587928056717,
      "kl": 0.0023127198219299316,
      "learning_rate": 9.66045715125541e-07,
      "loss": 0.0001,
      "reward": 0.1655233004130423,
      "reward_std": 0.13684139621909708,
      "rewards/cosine_scaled_reward": 0.17548487707972527,
      "rewards/format_reward": 0.6250000111758709,
      "step": 106
    },
    {
      "completion_length": 2713.1250915527344,
      "epoch": 0.12228571428571429,
      "grad_norm": 0.23116803169250488,
      "kl": 0.0034139156341552734,
      "learning_rate": 9.648384182148252e-07,
      "loss": 0.0001,
      "reward": 0.07083619991317391,
      "reward_std": 0.12138741742819548,
      "rewards/cosine_scaled_reward": -0.05151521973311901,
      "rewards/format_reward": 0.5208333488553762,
      "step": 107
    },
    {
      "completion_length": 2540.8125762939453,
      "epoch": 0.12342857142857143,
      "grad_norm": 0.20976760983467102,
      "kl": 0.0018079280853271484,
      "learning_rate": 9.636109026648554e-07,
      "loss": 0.0001,
      "reward": 0.07085073599591851,
      "reward_std": 0.15451500099152327,
      "rewards/cosine_scaled_reward": -0.041776820085942745,
      "rewards/format_reward": 0.5000000074505806,
      "step": 108
    },
    {
      "completion_length": 3043.500045776367,
      "epoch": 0.12457142857142857,
      "grad_norm": 0.16290034353733063,
      "kl": 0.0009417533874511719,
      "learning_rate": 9.623632283030077e-07,
      "loss": 0.0,
      "reward": 0.03415968408808112,
      "reward_std": 0.09731752565130591,
      "rewards/cosine_scaled_reward": -0.08652829378843307,
      "rewards/format_reward": 0.37500000931322575,
      "step": 109
    },
    {
      "completion_length": 2716.9583587646484,
      "epoch": 0.12571428571428572,
      "grad_norm": 0.202264204621315,
      "kl": 0.0016897767782211304,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0001,
      "reward": 0.030952309258282185,
      "reward_std": 0.13009351352229714,
      "rewards/cosine_scaled_reward": -0.11608889419585466,
      "rewards/format_reward": 0.41666666977107525,
      "step": 110
    },
    {
      "completion_length": 3076.3959045410156,
      "epoch": 0.12685714285714286,
      "grad_norm": 0.1983112394809723,
      "kl": 0.002537250518798828,
      "learning_rate": 9.598076473627796e-07,
      "loss": 0.0001,
      "reward": 0.059007523115724325,
      "reward_std": 0.16636842489242554,
      "rewards/cosine_scaled_reward": -0.0232586320489645,
      "rewards/format_reward": 0.39583334140479565,
      "step": 111
    },
    {
      "completion_length": 3080.041717529297,
      "epoch": 0.128,
      "grad_norm": 0.15135328471660614,
      "kl": 0.0012042894959449768,
      "learning_rate": 9.58499865339809e-07,
      "loss": 0.0,
      "reward": 0.06036388734355569,
      "reward_std": 0.11590251373127103,
      "rewards/cosine_scaled_reward": -0.008873747196048498,
      "rewards/format_reward": 0.37500000931322575,
      "step": 112
    },
    {
      "completion_length": 2676.9583435058594,
      "epoch": 0.12914285714285714,
      "grad_norm": 0.2907523810863495,
      "kl": 0.0033299922943115234,
      "learning_rate": 9.571721736097088e-07,
      "loss": 0.0001,
      "reward": 0.021284373477101326,
      "reward_std": 0.12445678655058146,
      "rewards/cosine_scaled_reward": -0.12851847242563963,
      "rewards/format_reward": 0.37500001303851604,
      "step": 113
    },
    {
      "completion_length": 2599.645866394043,
      "epoch": 0.13028571428571428,
      "grad_norm": 0.21558967232704163,
      "kl": 0.0046939849853515625,
      "learning_rate": 9.55824636882301e-07,
      "loss": 0.0002,
      "reward": 0.030540801119059324,
      "reward_std": 0.09902108740061522,
      "rewards/cosine_scaled_reward": -0.19062614813446999,
      "rewards/format_reward": 0.562500013038516,
      "step": 114
    },
    {
      "completion_length": 2822.729179382324,
      "epoch": 0.13142857142857142,
      "grad_norm": 0.20229722559452057,
      "kl": 0.0031037330627441406,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.0001,
      "reward": 0.016621847171336412,
      "reward_std": 0.1235029874369502,
      "rewards/cosine_scaled_reward": -0.14894464937970042,
      "rewards/format_reward": 0.3958333432674408,
      "step": 115
    },
    {
      "completion_length": 3348.0833435058594,
      "epoch": 0.13257142857142856,
      "grad_norm": 0.17120520770549774,
      "kl": 0.0019791126251220703,
      "learning_rate": 9.530702921077358e-07,
      "loss": 0.0001,
      "reward": -0.046343902591615915,
      "reward_std": 0.09335399139672518,
      "rewards/cosine_scaled_reward": -0.2216574940830469,
      "rewards/format_reward": 0.16666667349636555,
      "step": 116
    },
    {
      "completion_length": 2949.8958740234375,
      "epoch": 0.1337142857142857,
      "grad_norm": 0.27388995885849,
      "kl": 0.0031032562255859375,
      "learning_rate": 9.516636183034564e-07,
      "loss": 0.0001,
      "reward": -0.03218040708452463,
      "reward_std": 0.08237923681735992,
      "rewards/cosine_scaled_reward": -0.22971701715141535,
      "rewards/format_reward": 0.27083334140479565,
      "step": 117
    },
    {
      "completion_length": 2783.3542098999023,
      "epoch": 0.13485714285714287,
      "grad_norm": 0.17567184567451477,
      "kl": 0.001497507095336914,
      "learning_rate": 9.502373679810839e-07,
      "loss": 0.0001,
      "reward": 0.12685386650264263,
      "reward_std": 0.13957502879202366,
      "rewards/cosine_scaled_reward": 0.1258874498307705,
      "rewards/format_reward": 0.5000000018626451,
      "step": 118
    },
    {
      "completion_length": 2406.875030517578,
      "epoch": 0.136,
      "grad_norm": 0.2555752098560333,
      "kl": 0.004542350769042969,
      "learning_rate": 9.487916106540465e-07,
      "loss": 0.0002,
      "reward": 0.094550846144557,
      "reward_std": 0.12869207374751568,
      "rewards/cosine_scaled_reward": -0.0038661109283566475,
      "rewards/format_reward": 0.562500013038516,
      "step": 119
    },
    {
      "completion_length": 2285.6250762939453,
      "epoch": 0.13714285714285715,
      "grad_norm": 0.23725992441177368,
      "kl": 0.002517223358154297,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0001,
      "reward": 0.08064356981776655,
      "reward_std": 0.12271030526608229,
      "rewards/cosine_scaled_reward": -0.0756605202332139,
      "rewards/format_reward": 0.6250000074505806,
      "step": 120
    },
    {
      "completion_length": 1854.8542022705078,
      "epoch": 0.1382857142857143,
      "grad_norm": 0.2627831697463989,
      "kl": 0.003121614456176758,
      "learning_rate": 9.458418577899774e-07,
      "loss": 0.0001,
      "reward": 0.11522941256407648,
      "reward_std": 0.12531911802943796,
      "rewards/cosine_scaled_reward": -0.03698302572593093,
      "rewards/format_reward": 0.750000013038516,
      "step": 121
    },
    {
      "completion_length": 2925.1458892822266,
      "epoch": 0.13942857142857143,
      "grad_norm": 0.18577584624290466,
      "kl": 0.0021719932556152344,
      "learning_rate": 9.443380060197385e-07,
      "loss": 0.0001,
      "reward": 0.07248353259637952,
      "reward_std": 0.15761876897886395,
      "rewards/cosine_scaled_reward": 0.006229955703020096,
      "rewards/format_reward": 0.4166666753590107,
      "step": 122
    },
    {
      "completion_length": 2714.7708740234375,
      "epoch": 0.14057142857142857,
      "grad_norm": 0.17778825759887695,
      "kl": 0.002131819725036621,
      "learning_rate": 9.428149347714143e-07,
      "loss": 0.0001,
      "reward": 0.04195330070797354,
      "reward_std": 0.11853250442072749,
      "rewards/cosine_scaled_reward": -0.11678100191056728,
      "rewards/format_reward": 0.47916667349636555,
      "step": 123
    },
    {
      "completion_length": 2183.958381652832,
      "epoch": 0.1417142857142857,
      "grad_norm": 0.2191576361656189,
      "kl": 0.005345821380615234,
      "learning_rate": 9.412727182773486e-07,
      "loss": 0.0002,
      "reward": 0.0987518366164295,
      "reward_std": 0.1706788558512926,
      "rewards/cosine_scaled_reward": -0.011822802014648914,
      "rewards/format_reward": 0.6041666753590107,
      "step": 124
    },
    {
      "completion_length": 2888.354202270508,
      "epoch": 0.14285714285714285,
      "grad_norm": 0.16155032813549042,
      "kl": 0.0020842552185058594,
      "learning_rate": 9.397114317029974e-07,
      "loss": 0.0001,
      "reward": 0.04314285283908248,
      "reward_std": 0.1018235448282212,
      "rewards/cosine_scaled_reward": -0.029325059265829623,
      "rewards/format_reward": 0.31250000186264515,
      "step": 125
    },
    {
      "completion_length": 2848.166702270508,
      "epoch": 0.144,
      "grad_norm": 0.1632193773984909,
      "kl": 0.0012958049774169922,
      "learning_rate": 9.381311511432658e-07,
      "loss": 0.0001,
      "reward": 0.05145277862902731,
      "reward_std": 0.1218188302591443,
      "rewards/cosine_scaled_reward": -0.06772069446742535,
      "rewards/format_reward": 0.43750000558793545,
      "step": 126
    },
    {
      "completion_length": 2989.2708892822266,
      "epoch": 0.14514285714285713,
      "grad_norm": 0.1904735267162323,
      "kl": 0.0023894309997558594,
      "learning_rate": 9.36531953618799e-07,
      "loss": 0.0001,
      "reward": 0.010273065650835633,
      "reward_std": 0.11918974481523037,
      "rewards/cosine_scaled_reward": -0.1677860009167489,
      "rewards/format_reward": 0.39583334513008595,
      "step": 127
    },
    {
      "completion_length": 2901.5625228881836,
      "epoch": 0.1462857142857143,
      "grad_norm": 0.18506519496440887,
      "kl": 0.003477931022644043,
      "learning_rate": 9.34913917072228e-07,
      "loss": 0.0001,
      "reward": 0.08515205327421427,
      "reward_std": 0.14246997330337763,
      "rewards/cosine_scaled_reward": 0.07529417611658573,
      "rewards/format_reward": 0.3541666753590107,
      "step": 128
    },
    {
      "completion_length": 3311.187530517578,
      "epoch": 0.14742857142857144,
      "grad_norm": 0.22474369406700134,
      "kl": 0.0030045509338378906,
      "learning_rate": 9.332771203643714e-07,
      "loss": 0.0001,
      "reward": -0.026094807864865288,
      "reward_std": 0.12371798837557435,
      "rewards/cosine_scaled_reward": -0.19261731766164303,
      "rewards/format_reward": 0.22916667349636555,
      "step": 129
    },
    {
      "completion_length": 2830.0833587646484,
      "epoch": 0.14857142857142858,
      "grad_norm": 0.18044407665729523,
      "kl": 0.0024716854095458984,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0001,
      "reward": -0.006927699316293001,
      "reward_std": 0.08703188924118876,
      "rewards/cosine_scaled_reward": -0.1649078167974949,
      "rewards/format_reward": 0.29166667349636555,
      "step": 130
    },
    {
      "completion_length": 2923.0000228881836,
      "epoch": 0.14971428571428572,
      "grad_norm": 0.22359904646873474,
      "kl": 0.0043888092041015625,
      "learning_rate": 9.299475664759068e-07,
      "loss": 0.0002,
      "reward": 0.04467150871641934,
      "reward_std": 0.11750007548835129,
      "rewards/cosine_scaled_reward": -0.02377407788299024,
      "rewards/format_reward": 0.31250000558793545,
      "step": 131
    },
    {
      "completion_length": 2468.4583587646484,
      "epoch": 0.15085714285714286,
      "grad_norm": 0.20087915658950806,
      "kl": 0.0021352767944335938,
      "learning_rate": 9.282549715730579e-07,
      "loss": 0.0001,
      "reward": 0.04917083401232958,
      "reward_std": 0.10673188930377364,
      "rewards/cosine_scaled_reward": -0.07255440950393677,
      "rewards/format_reward": 0.43750000186264515,
      "step": 132
    },
    {
      "completion_length": 2899.750030517578,
      "epoch": 0.152,
      "grad_norm": 0.2198459506034851,
      "kl": 0.0033211708068847656,
      "learning_rate": 9.265439410565328e-07,
      "loss": 0.0001,
      "reward": 0.007418630411848426,
      "reward_std": 0.09727612743154168,
      "rewards/cosine_scaled_reward": -0.16582289477810264,
      "rewards/format_reward": 0.3750000111758709,
      "step": 133
    },
    {
      "completion_length": 2317.9375228881836,
      "epoch": 0.15314285714285714,
      "grad_norm": 0.19853614270687103,
      "kl": 0.004344940185546875,
      "learning_rate": 9.248145583195447e-07,
      "loss": 0.0002,
      "reward": 0.09111709147691727,
      "reward_std": 0.11203813040629029,
      "rewards/cosine_scaled_reward": -0.02176826912909746,
      "rewards/format_reward": 0.5833333432674408,
      "step": 134
    },
    {
      "completion_length": 1802.4791946411133,
      "epoch": 0.15428571428571428,
      "grad_norm": 0.26218482851982117,
      "kl": 0.0051059722900390625,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0002,
      "reward": 0.19630146119743586,
      "reward_std": 0.15289967821445316,
      "rewards/cosine_scaled_reward": 0.23608196713030338,
      "rewards/format_reward": 0.6875000111758709,
      "step": 135
    },
    {
      "completion_length": 2570.854248046875,
      "epoch": 0.15542857142857142,
      "grad_norm": 0.20145438611507416,
      "kl": 0.0037708282470703125,
      "learning_rate": 9.213010742252327e-07,
      "loss": 0.0002,
      "reward": 0.0829686057404615,
      "reward_std": 0.1752019147388637,
      "rewards/cosine_scaled_reward": -0.026120582595467567,
      "rewards/format_reward": 0.5416666753590107,
      "step": 136
    },
    {
      "completion_length": 2661.2500381469727,
      "epoch": 0.15657142857142858,
      "grad_norm": 0.19357463717460632,
      "kl": 0.0031414031982421875,
      "learning_rate": 9.195171441101668e-07,
      "loss": 0.0001,
      "reward": 0.0038152660708874464,
      "reward_std": 0.10811280179768801,
      "rewards/cosine_scaled_reward": -0.21752969082444906,
      "rewards/format_reward": 0.45833334513008595,
      "step": 137
    },
    {
      "completion_length": 2141.166702270508,
      "epoch": 0.15771428571428572,
      "grad_norm": 0.2089914232492447,
      "kl": 0.002676725387573242,
      "learning_rate": 9.177152042508077e-07,
      "loss": 0.0001,
      "reward": 0.14542343048378825,
      "reward_std": 0.15844334475696087,
      "rewards/cosine_scaled_reward": 0.04235106392297894,
      "rewards/format_reward": 0.7708333618938923,
      "step": 138
    },
    {
      "completion_length": 2820.8750610351562,
      "epoch": 0.15885714285714286,
      "grad_norm": 0.18096813559532166,
      "kl": 0.0042743682861328125,
      "learning_rate": 9.158953424711624e-07,
      "loss": 0.0002,
      "reward": 0.06168772419914603,
      "reward_std": 0.14358124835416675,
      "rewards/cosine_scaled_reward": -0.07845509238541126,
      "rewards/format_reward": 0.520833345130086,
      "step": 139
    },
    {
      "completion_length": 2436.354217529297,
      "epoch": 0.16,
      "grad_norm": 0.27698883414268494,
      "kl": 0.0058422088623046875,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0002,
      "reward": 0.044038140535121784,
      "reward_std": 0.07816965272650123,
      "rewards/cosine_scaled_reward": -0.11026089265942574,
      "rewards/format_reward": 0.4791666716337204,
      "step": 140
    },
    {
      "completion_length": 2359.375045776367,
      "epoch": 0.16114285714285714,
      "grad_norm": 0.1908435970544815,
      "kl": 0.0043811798095703125,
      "learning_rate": 9.122022088101613e-07,
      "loss": 0.0002,
      "reward": 0.03223781171254814,
      "reward_std": 0.11182020884007215,
      "rewards/cosine_scaled_reward": -0.1976433489471674,
      "rewards/format_reward": 0.5833333414047956,
      "step": 141
    },
    {
      "completion_length": 2460.187530517578,
      "epoch": 0.16228571428571428,
      "grad_norm": 0.1981201469898224,
      "kl": 0.0038270950317382812,
      "learning_rate": 9.103291169269299e-07,
      "loss": 0.0002,
      "reward": 0.08847181824967265,
      "reward_std": 0.13877713168039918,
      "rewards/cosine_scaled_reward": -0.06368941674008965,
      "rewards/format_reward": 0.645833345130086,
      "step": 142
    },
    {
      "completion_length": 2437.791702270508,
      "epoch": 0.16342857142857142,
      "grad_norm": 0.3880097270011902,
      "kl": 0.023929595947265625,
      "learning_rate": 9.084384631108882e-07,
      "loss": 0.001,
      "reward": 0.016975378792267293,
      "reward_std": 0.12177514331415296,
      "rewards/cosine_scaled_reward": -0.20094774826429784,
      "rewards/format_reward": 0.5000000111758709,
      "step": 143
    },
    {
      "completion_length": 2708.916732788086,
      "epoch": 0.16457142857142856,
      "grad_norm": 0.21965523064136505,
      "kl": 0.00417327880859375,
      "learning_rate": 9.065303395098358e-07,
      "loss": 0.0002,
      "reward": 0.030230441665480612,
      "reward_std": 0.17415813449770212,
      "rewards/cosine_scaled_reward": -0.11869808053597808,
      "rewards/format_reward": 0.4166666753590107,
      "step": 144
    },
    {
      "completion_length": 1839.4375228881836,
      "epoch": 0.1657142857142857,
      "grad_norm": 0.2578926384449005,
      "kl": 0.004568576812744141,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.0002,
      "reward": 0.15131515043321997,
      "reward_std": 0.15678277891129255,
      "rewards/cosine_scaled_reward": 0.0914019983028993,
      "rewards/format_reward": 0.7083333358168602,
      "step": 145
    },
    {
      "completion_length": 1990.5833892822266,
      "epoch": 0.16685714285714287,
      "grad_norm": 0.19752271473407745,
      "kl": 0.003320455551147461,
      "learning_rate": 9.026620557966279e-07,
      "loss": 0.0001,
      "reward": 0.09106699889525771,
      "reward_std": 0.10312592587433755,
      "rewards/cosine_scaled_reward": -0.1385479016462341,
      "rewards/format_reward": 0.8125000149011612,
      "step": 146
    },
    {
      "completion_length": 1966.604232788086,
      "epoch": 0.168,
      "grad_norm": 0.26733097434043884,
      "kl": 0.006213188171386719,
      "learning_rate": 9.007020842191634e-07,
      "loss": 0.0002,
      "reward": 0.12648878013715148,
      "reward_std": 0.1677848151884973,
      "rewards/cosine_scaled_reward": 0.016043312381953,
      "rewards/format_reward": 0.7083333469927311,
      "step": 147
    },
    {
      "completion_length": 1654.6250457763672,
      "epoch": 0.16914285714285715,
      "grad_norm": 0.2192990928888321,
      "kl": 0.004794120788574219,
      "learning_rate": 8.987250199168808e-07,
      "loss": 0.0002,
      "reward": 0.09204956935718656,
      "reward_std": 0.08866735780611634,
      "rewards/cosine_scaled_reward": -0.13534213416278362,
      "rewards/format_reward": 0.8125000149011612,
      "step": 148
    },
    {
      "completion_length": 2345.729232788086,
      "epoch": 0.1702857142857143,
      "grad_norm": 0.2061564177274704,
      "kl": 0.0041484832763671875,
      "learning_rate": 8.967309592491052e-07,
      "loss": 0.0002,
      "reward": 0.08000461710616946,
      "reward_std": 0.12683053640648723,
      "rewards/cosine_scaled_reward": -0.06624547764658928,
      "rewards/format_reward": 0.6041666828095913,
      "step": 149
    },
    {
      "completion_length": 2006.6250381469727,
      "epoch": 0.17142857142857143,
      "grad_norm": 0.24100010097026825,
      "kl": 0.00368499755859375,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0001,
      "reward": 0.08540961390826851,
      "reward_std": 0.15422609634697437,
      "rewards/cosine_scaled_reward": -0.0530257155187428,
      "rewards/format_reward": 0.6041666734963655,
      "step": 150
    },
    {
      "completion_length": 1996.520881652832,
      "epoch": 0.17257142857142857,
      "grad_norm": 0.28235289454460144,
      "kl": 0.005840301513671875,
      "learning_rate": 8.926922383915315e-07,
      "loss": 0.0002,
      "reward": 0.12831606157124043,
      "reward_std": 0.131237086141482,
      "rewards/cosine_scaled_reward": 0.031276384368538857,
      "rewards/format_reward": 0.6875000204890966,
      "step": 151
    },
    {
      "completion_length": 2301.0208435058594,
      "epoch": 0.1737142857142857,
      "grad_norm": 0.3784143626689911,
      "kl": 0.0074005126953125,
      "learning_rate": 8.906477750432903e-07,
      "loss": 0.0003,
      "reward": 0.025953251402825117,
      "reward_std": 0.11646852549165487,
      "rewards/cosine_scaled_reward": -0.18527186242863536,
      "rewards/format_reward": 0.5208333488553762,
      "step": 152
    },
    {
      "completion_length": 2342.0834045410156,
      "epoch": 0.17485714285714285,
      "grad_norm": 0.20556333661079407,
      "kl": 0.0073909759521484375,
      "learning_rate": 8.88586709003076e-07,
      "loss": 0.0003,
      "reward": 0.03766886703670025,
      "reward_std": 0.08948473795317113,
      "rewards/cosine_scaled_reward": -0.19193847686983645,
      "rewards/format_reward": 0.6041666753590107,
      "step": 153
    },
    {
      "completion_length": 3022.5833740234375,
      "epoch": 0.176,
      "grad_norm": 0.22437125444412231,
      "kl": 0.00362396240234375,
      "learning_rate": 8.865091407243394e-07,
      "loss": 0.0001,
      "reward": 0.06817820528522134,
      "reward_std": 0.16511888336390257,
      "rewards/cosine_scaled_reward": -0.018517197109758854,
      "rewards/format_reward": 0.4375000074505806,
      "step": 154
    },
    {
      "completion_length": 2435.0625610351562,
      "epoch": 0.17714285714285713,
      "grad_norm": 0.21803762018680573,
      "kl": 0.004116058349609375,
      "learning_rate": 8.844151714648274e-07,
      "loss": 0.0002,
      "reward": 0.11520516406744719,
      "reward_std": 0.13615657854825258,
      "rewards/cosine_scaled_reward": 0.03556065930752084,
      "rewards/format_reward": 0.6041666734963655,
      "step": 155
    },
    {
      "completion_length": 2320.583351135254,
      "epoch": 0.1782857142857143,
      "grad_norm": 0.1894351989030838,
      "kl": 0.00403594970703125,
      "learning_rate": 8.823049032816478e-07,
      "loss": 0.0002,
      "reward": 0.09686407796107233,
      "reward_std": 0.13217018358409405,
      "rewards/cosine_scaled_reward": 0.03549210913479328,
      "rewards/format_reward": 0.5000000055879354,
      "step": 156
    },
    {
      "completion_length": 2527.5834197998047,
      "epoch": 0.17942857142857144,
      "grad_norm": 0.2570294737815857,
      "kl": 0.005597114562988281,
      "learning_rate": 8.801784390262943e-07,
      "loss": 0.0002,
      "reward": 0.046291103353723884,
      "reward_std": 0.09627518011257052,
      "rewards/cosine_scaled_reward": -0.15388192608952522,
      "rewards/format_reward": 0.5833333395421505,
      "step": 157
    },
    {
      "completion_length": 2241.729248046875,
      "epoch": 0.18057142857142858,
      "grad_norm": 0.21536891162395477,
      "kl": 0.0056514739990234375,
      "learning_rate": 8.780358823396352e-07,
      "loss": 0.0002,
      "reward": 0.15251322067342699,
      "reward_std": 0.13267278019338846,
      "rewards/cosine_scaled_reward": 0.06854456290602684,
      "rewards/format_reward": 0.7500000186264515,
      "step": 158
    },
    {
      "completion_length": 2231.7708587646484,
      "epoch": 0.18171428571428572,
      "grad_norm": 0.1829807013273239,
      "kl": 0.004848480224609375,
      "learning_rate": 8.758773376468604e-07,
      "loss": 0.0002,
      "reward": 0.044310242868959904,
      "reward_std": 0.11434727860614657,
      "rewards/cosine_scaled_reward": -0.16176462545990944,
      "rewards/format_reward": 0.5833333432674408,
      "step": 159
    },
    {
      "completion_length": 2064.312530517578,
      "epoch": 0.18285714285714286,
      "grad_norm": 0.23337005078792572,
      "kl": 0.006267547607421875,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.0003,
      "reward": 0.10007608711021021,
      "reward_std": 0.1771828606724739,
      "rewards/cosine_scaled_reward": -0.04113523324485868,
      "rewards/format_reward": 0.6666666828095913,
      "step": 160
    },
    {
      "completion_length": 2270.937545776367,
      "epoch": 0.184,
      "grad_norm": 0.3289494812488556,
      "kl": 0.007640838623046875,
      "learning_rate": 8.715127058347614e-07,
      "loss": 0.0003,
      "reward": 0.053364482591859996,
      "reward_std": 0.14938203245401382,
      "rewards/cosine_scaled_reward": -0.11477282957639545,
      "rewards/format_reward": 0.541666679084301,
      "step": 161
    },
    {
      "completion_length": 2312.416702270508,
      "epoch": 0.18514285714285714,
      "grad_norm": 0.26157301664352417,
      "kl": 0.008253097534179688,
      "learning_rate": 8.693068314414344e-07,
      "loss": 0.0003,
      "reward": 0.04589501162990928,
      "reward_std": 0.12189697381108999,
      "rewards/cosine_scaled_reward": -0.13589882757514715,
      "rewards/format_reward": 0.5416666753590107,
      "step": 162
    },
    {
      "completion_length": 2191.4583740234375,
      "epoch": 0.18628571428571428,
      "grad_norm": 0.2897321283817291,
      "kl": 0.00688934326171875,
      "learning_rate": 8.670853944836176e-07,
      "loss": 0.0003,
      "reward": 0.13259745202958584,
      "reward_std": 0.06983533198945224,
      "rewards/cosine_scaled_reward": 0.07808320969343185,
      "rewards/format_reward": 0.625,
      "step": 163
    },
    {
      "completion_length": 1521.0625610351562,
      "epoch": 0.18742857142857142,
      "grad_norm": 0.2747092843055725,
      "kl": 0.009218215942382812,
      "learning_rate": 8.648485032310144e-07,
      "loss": 0.0004,
      "reward": 0.12581492541357875,
      "reward_std": 0.09997158916667104,
      "rewards/cosine_scaled_reward": -0.03640655893832445,
      "rewards/format_reward": 0.8125000055879354,
      "step": 164
    },
    {
      "completion_length": 1848.2500228881836,
      "epoch": 0.18857142857142858,
      "grad_norm": 0.275541216135025,
      "kl": 0.006542205810546875,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.0003,
      "reward": 0.0523092825897038,
      "reward_std": 0.12433975096791983,
      "rewards/cosine_scaled_reward": -0.1790934158489108,
      "rewards/format_reward": 0.6666666716337204,
      "step": 165
    },
    {
      "completion_length": 2049.0417251586914,
      "epoch": 0.18971428571428572,
      "grad_norm": 0.1916612833738327,
      "kl": 0.0053234100341796875,
      "learning_rate": 8.603287946810513e-07,
      "loss": 0.0002,
      "reward": 0.09825841523706913,
      "reward_std": 0.15324107883498073,
      "rewards/cosine_scaled_reward": -0.0768615622073412,
      "rewards/format_reward": 0.7291666734963655,
      "step": 166
    },
    {
      "completion_length": 1554.1666946411133,
      "epoch": 0.19085714285714286,
      "grad_norm": 0.22473838925361633,
      "kl": 0.0043048858642578125,
      "learning_rate": 8.580461976679099e-07,
      "loss": 0.0002,
      "reward": 0.12430966179817915,
      "reward_std": 0.12945860624313354,
      "rewards/cosine_scaled_reward": -0.07466088375076652,
      "rewards/format_reward": 0.8750000111758709,
      "step": 167
    },
    {
      "completion_length": 2357.5209045410156,
      "epoch": 0.192,
      "grad_norm": 0.218026265501976,
      "kl": 0.0063018798828125,
      "learning_rate": 8.557485869176825e-07,
      "loss": 0.0003,
      "reward": 0.0786146642640233,
      "reward_std": 0.14164651185274124,
      "rewards/cosine_scaled_reward": -0.08221817389130592,
      "rewards/format_reward": 0.6250000223517418,
      "step": 168
    },
    {
      "completion_length": 1244.0625381469727,
      "epoch": 0.19314285714285714,
      "grad_norm": 0.2136813998222351,
      "kl": 0.005023002624511719,
      "learning_rate": 8.534360744126753e-07,
      "loss": 0.0002,
      "reward": 0.2644330468028784,
      "reward_std": 0.10949333664029837,
      "rewards/cosine_scaled_reward": 0.2890866380184889,
      "rewards/format_reward": 0.9791666716337204,
      "step": 169
    },
    {
      "completion_length": 2086.333366394043,
      "epoch": 0.19428571428571428,
      "grad_norm": 0.23367033898830414,
      "kl": 0.0049877166748046875,
      "learning_rate": 8.511087728614862e-07,
      "loss": 0.0002,
      "reward": 0.13441222603432834,
      "reward_std": 0.12571046565426514,
      "rewards/cosine_scaled_reward": 0.07233863137662411,
      "rewards/format_reward": 0.6458333395421505,
      "step": 170
    },
    {
      "completion_length": 2040.7708587646484,
      "epoch": 0.19542857142857142,
      "grad_norm": 0.2201111614704132,
      "kl": 0.004878044128417969,
      "learning_rate": 8.487667956935087e-07,
      "loss": 0.0002,
      "reward": 0.07799222506582737,
      "reward_std": 0.1219726437702775,
      "rewards/cosine_scaled_reward": -0.09505355032160878,
      "rewards/format_reward": 0.6458333395421505,
      "step": 171
    },
    {
      "completion_length": 2090.562557220459,
      "epoch": 0.19657142857142856,
      "grad_norm": 0.24928785860538483,
      "kl": 0.008022308349609375,
      "learning_rate": 8.464102570534061e-07,
      "loss": 0.0003,
      "reward": 0.1555600226856768,
      "reward_std": 0.12234124867245555,
      "rewards/cosine_scaled_reward": 0.14208754245191813,
      "rewards/format_reward": 0.6250000149011612,
      "step": 172
    },
    {
      "completion_length": 1355.5000228881836,
      "epoch": 0.1977142857142857,
      "grad_norm": 0.2922389805316925,
      "kl": 0.00618743896484375,
      "learning_rate": 8.440392717955475e-07,
      "loss": 0.0002,
      "reward": 0.07720496540423483,
      "reward_std": 0.08944753208197653,
      "rewards/cosine_scaled_reward": -0.16825812682509422,
      "rewards/format_reward": 0.7916666679084301,
      "step": 173
    },
    {
      "completion_length": 1297.6042098999023,
      "epoch": 0.19885714285714284,
      "grad_norm": 0.21333926916122437,
      "kl": 0.008691787719726562,
      "learning_rate": 8.416539554784089e-07,
      "loss": 0.0003,
      "reward": 0.1351238526403904,
      "reward_std": 0.09852686384692788,
      "rewards/cosine_scaled_reward": -0.0807552793994546,
      "rewards/format_reward": 0.9583333432674408,
      "step": 174
    },
    {
      "completion_length": 1959.2708892822266,
      "epoch": 0.2,
      "grad_norm": 0.22018226981163025,
      "kl": 0.006298065185546875,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0003,
      "reward": 0.14479633374139667,
      "reward_std": 0.0969647653400898,
      "rewards/cosine_scaled_reward": 0.06122639961540699,
      "rewards/format_reward": 0.7291666753590107,
      "step": 175
    },
    {
      "completion_length": 1673.6875381469727,
      "epoch": 0.20114285714285715,
      "grad_norm": 0.2576465308666229,
      "kl": 0.0056438446044921875,
      "learning_rate": 8.368407953869103e-07,
      "loss": 0.0002,
      "reward": 0.0873019965365529,
      "reward_std": 0.11411390919238329,
      "rewards/cosine_scaled_reward": -0.12017827155068517,
      "rewards/format_reward": 0.7500000111758709,
      "step": 176
    },
    {
      "completion_length": 2214.270881652832,
      "epoch": 0.2022857142857143,
      "grad_norm": 0.28319552540779114,
      "kl": 0.0062694549560546875,
      "learning_rate": 8.344131861991828e-07,
      "loss": 0.0003,
      "reward": 0.08295768650714308,
      "reward_std": 0.10326328268274665,
      "rewards/cosine_scaled_reward": -0.09064689744263887,
      "rewards/format_reward": 0.6666666753590107,
      "step": 177
    },
    {
      "completion_length": 1769.8542022705078,
      "epoch": 0.20342857142857143,
      "grad_norm": 0.28282982110977173,
      "kl": 0.00852203369140625,
      "learning_rate": 8.319717151140072e-07,
      "loss": 0.0003,
      "reward": 0.1297212722711265,
      "reward_std": 0.15833280980587006,
      "rewards/cosine_scaled_reward": 0.026437816210091114,
      "rewards/format_reward": 0.7083333432674408,
      "step": 178
    },
    {
      "completion_length": 2106.7291717529297,
      "epoch": 0.20457142857142857,
      "grad_norm": 0.21418659389019012,
      "kl": 0.00591278076171875,
      "learning_rate": 8.295165011252396e-07,
      "loss": 0.0002,
      "reward": 0.051371646230109036,
      "reward_std": 0.10319430893287063,
      "rewards/cosine_scaled_reward": -0.15093043667729944,
      "rewards/format_reward": 0.6041666697710752,
      "step": 179
    },
    {
      "completion_length": 1345.2083778381348,
      "epoch": 0.2057142857142857,
      "grad_norm": 0.28628435730934143,
      "kl": 0.00799560546875,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0003,
      "reward": 0.2407036293298006,
      "reward_std": 0.17751038947608322,
      "rewards/cosine_scaled_reward": 0.25933289900422096,
      "rewards/format_reward": 0.8958333432674408,
      "step": 180
    },
    {
      "completion_length": 1969.208381652832,
      "epoch": 0.20685714285714285,
      "grad_norm": 0.24428333342075348,
      "kl": 0.00662994384765625,
      "learning_rate": 8.245653237555705e-07,
      "loss": 0.0003,
      "reward": 0.09334775037132204,
      "reward_std": 0.12076092883944511,
      "rewards/cosine_scaled_reward": -0.08935475163161755,
      "rewards/format_reward": 0.7291666828095913,
      "step": 181
    },
    {
      "completion_length": 1827.2291946411133,
      "epoch": 0.208,
      "grad_norm": 0.19231781363487244,
      "kl": 0.0035953521728515625,
      "learning_rate": 8.220696016880687e-07,
      "loss": 0.0001,
      "reward": 0.10479269758798182,
      "reward_std": 0.11822887184098363,
      "rewards/cosine_scaled_reward": -0.06516919657588005,
      "rewards/format_reward": 0.7500000055879354,
      "step": 182
    },
    {
      "completion_length": 1392.8125457763672,
      "epoch": 0.20914285714285713,
      "grad_norm": 0.264604389667511,
      "kl": 0.009449005126953125,
      "learning_rate": 8.195606193320136e-07,
      "loss": 0.0004,
      "reward": 0.1556489015929401,
      "reward_std": 0.10147193586453795,
      "rewards/cosine_scaled_reward": 0.019236549735069275,
      "rewards/format_reward": 0.8750000074505806,
      "step": 183
    },
    {
      "completion_length": 1635.062551498413,
      "epoch": 0.2102857142857143,
      "grad_norm": 0.2975788414478302,
      "kl": 0.00707244873046875,
      "learning_rate": 8.170384989716657e-07,
      "loss": 0.0003,
      "reward": 0.0682187182828784,
      "reward_std": 0.0552776656113565,
      "rewards/cosine_scaled_reward": -0.20538464561104774,
      "rewards/format_reward": 0.8125000018626451,
      "step": 184
    },
    {
      "completion_length": 1627.1250686645508,
      "epoch": 0.21142857142857144,
      "grad_norm": 0.26065441966056824,
      "kl": 0.0062122344970703125,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.0002,
      "reward": 0.06265506497584283,
      "reward_std": 0.11735227680765092,
      "rewards/cosine_scaled_reward": -0.19095464050769806,
      "rewards/format_reward": 0.7500000093132257,
      "step": 185
    },
    {
      "completion_length": 1840.1458587646484,
      "epoch": 0.21257142857142858,
      "grad_norm": 0.2078036367893219,
      "kl": 0.007350921630859375,
      "learning_rate": 8.119553365707802e-07,
      "loss": 0.0003,
      "reward": 0.09769681794568896,
      "reward_std": 0.09951704926788807,
      "rewards/cosine_scaled_reward": -0.06353580858558416,
      "rewards/format_reward": 0.7083333414047956,
      "step": 186
    },
    {
      "completion_length": 1557.5416870117188,
      "epoch": 0.21371428571428572,
      "grad_norm": 0.44674429297447205,
      "kl": 0.008470535278320312,
      "learning_rate": 8.093945422764069e-07,
      "loss": 0.0003,
      "reward": 0.09892327198758721,
      "reward_std": 0.0855756844393909,
      "rewards/cosine_scaled_reward": -0.1240589041262865,
      "rewards/format_reward": 0.8333333414047956,
      "step": 187
    },
    {
      "completion_length": 2297.3333587646484,
      "epoch": 0.21485714285714286,
      "grad_norm": 0.2045762985944748,
      "kl": 0.008823394775390625,
      "learning_rate": 8.068211054579943e-07,
      "loss": 0.0004,
      "reward": 0.05021746223792434,
      "reward_std": 0.06788925174623728,
      "rewards/cosine_scaled_reward": -0.15488196723163128,
      "rewards/format_reward": 0.6041666697710752,
      "step": 188
    },
    {
      "completion_length": 1355.6875381469727,
      "epoch": 0.216,
      "grad_norm": 0.29116666316986084,
      "kl": 0.0076541900634765625,
      "learning_rate": 8.04235151541222e-07,
      "loss": 0.0003,
      "reward": 0.07565085194073617,
      "reward_std": 0.08206533431075513,
      "rewards/cosine_scaled_reward": -0.19497415097430348,
      "rewards/format_reward": 0.8333333358168602,
      "step": 189
    },
    {
      "completion_length": 1135.4167022705078,
      "epoch": 0.21714285714285714,
      "grad_norm": 0.2219790816307068,
      "kl": 0.006549835205078125,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0003,
      "reward": 0.15915191872045398,
      "reward_std": 0.08802533126436174,
      "rewards/cosine_scaled_reward": -0.022108266479335725,
      "rewards/format_reward": 0.9791666716337204,
      "step": 190
    },
    {
      "completion_length": 1071.7708587646484,
      "epoch": 0.21828571428571428,
      "grad_norm": 0.26136839389801025,
      "kl": 0.006053924560546875,
      "learning_rate": 7.990261971595048e-07,
      "loss": 0.0002,
      "reward": 0.20304076466709375,
      "reward_std": 0.12852966412901878,
      "rewards/cosine_scaled_reward": 0.10831178847001866,
      "rewards/format_reward": 0.9791666716337204,
      "step": 191
    },
    {
      "completion_length": 1712.4583740234375,
      "epoch": 0.21942857142857142,
      "grad_norm": 0.20474177598953247,
      "kl": 0.006473541259765625,
      "learning_rate": 7.964034505716476e-07,
      "loss": 0.0003,
      "reward": 0.09229559730738401,
      "reward_std": 0.1016106829047203,
      "rewards/cosine_scaled_reward": -0.1470730509608984,
      "rewards/format_reward": 0.8333333432674408,
      "step": 192
    },
    {
      "completion_length": 2210.395851135254,
      "epoch": 0.22057142857142858,
      "grad_norm": 1.7929117679595947,
      "kl": 0.055267333984375,
      "learning_rate": 7.93768694627233e-07,
      "loss": 0.0022,
      "reward": 0.03142698993906379,
      "reward_std": 0.12175118830054998,
      "rewards/cosine_scaled_reward": -0.18109365366399288,
      "rewards/format_reward": 0.5416666679084301,
      "step": 193
    },
    {
      "completion_length": 2227.291717529297,
      "epoch": 0.22171428571428572,
      "grad_norm": 0.21808066964149475,
      "kl": 0.00843048095703125,
      "learning_rate": 7.911220577405484e-07,
      "loss": 0.0003,
      "reward": 0.1474433816038072,
      "reward_std": 0.17017615539953113,
      "rewards/cosine_scaled_reward": 0.07256746315397322,
      "rewards/format_reward": 0.7291666716337204,
      "step": 194
    },
    {
      "completion_length": 1330.1458435058594,
      "epoch": 0.22285714285714286,
      "grad_norm": 0.2461828738451004,
      "kl": 0.008304595947265625,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0003,
      "reward": 0.13772209081798792,
      "reward_std": 0.13301934953778982,
      "rewards/cosine_scaled_reward": -0.04500130284577608,
      "rewards/format_reward": 0.8958333507180214,
      "step": 195
    },
    {
      "completion_length": 2083.5000762939453,
      "epoch": 0.224,
      "grad_norm": 0.2871836721897125,
      "kl": 0.0097198486328125,
      "learning_rate": 7.857936576865356e-07,
      "loss": 0.0004,
      "reward": 0.11253911699168384,
      "reward_std": 0.1284322296269238,
      "rewards/cosine_scaled_reward": -0.02314686682075262,
      "rewards/format_reward": 0.7083333544433117,
      "step": 196
    },
    {
      "completion_length": 1072.708366394043,
      "epoch": 0.22514285714285714,
      "grad_norm": 0.28244540095329285,
      "kl": 0.006664276123046875,
      "learning_rate": 7.831121542179086e-07,
      "loss": 0.0003,
      "reward": 0.14220814127475023,
      "reward_std": 0.1526435911655426,
      "rewards/cosine_scaled_reward": -0.0398729182779789,
      "rewards/format_reward": 0.9166666679084301,
      "step": 197
    },
    {
      "completion_length": 1305.8750305175781,
      "epoch": 0.22628571428571428,
      "grad_norm": 0.3013690710067749,
      "kl": 0.009075164794921875,
      "learning_rate": 7.804192891917571e-07,
      "loss": 0.0004,
      "reward": 0.1608109144726768,
      "reward_std": 0.13874499686062336,
      "rewards/cosine_scaled_reward": 0.03152369521558285,
      "rewards/format_reward": 0.8750000149011612,
      "step": 198
    },
    {
      "completion_length": 1484.0000457763672,
      "epoch": 0.22742857142857142,
      "grad_norm": 0.21533454954624176,
      "kl": 0.0067596435546875,
      "learning_rate": 7.777151938545235e-07,
      "loss": 0.0003,
      "reward": 0.11915545212104917,
      "reward_std": 0.10307862563058734,
      "rewards/cosine_scaled_reward": -0.1427280263742432,
      "rewards/format_reward": 0.9791666716337204,
      "step": 199
    },
    {
      "completion_length": 1291.8750228881836,
      "epoch": 0.22857142857142856,
      "grad_norm": 0.2203797996044159,
      "kl": 0.008068084716796875,
      "learning_rate": 7.75e-07,
      "loss": 0.0003,
      "reward": 0.1579569444875233,
      "reward_std": 0.1443558344617486,
      "rewards/cosine_scaled_reward": -0.0025847081560641527,
      "rewards/format_reward": 0.9375,
      "step": 200
    },
    {
      "completion_length": 1646.3958892822266,
      "epoch": 0.2297142857142857,
      "grad_norm": 0.24083541333675385,
      "kl": 0.00879669189453125,
      "learning_rate": 7.72273839962904e-07,
      "loss": 0.0004,
      "reward": 0.21994818467646837,
      "reward_std": 0.11868779285578057,
      "rewards/cosine_scaled_reward": 0.23895522952079773,
      "rewards/format_reward": 0.791666679084301,
      "step": 201
    },
    {
      "completion_length": 1287.6041946411133,
      "epoch": 0.23085714285714284,
      "grad_norm": 0.3120401203632355,
      "kl": 0.016357421875,
      "learning_rate": 7.695368466124296e-07,
      "loss": 0.0007,
      "reward": 0.21438546478748322,
      "reward_std": 0.06259978096932173,
      "rewards/cosine_scaled_reward": 0.1961576696485281,
      "rewards/format_reward": 0.875,
      "step": 202
    },
    {
      "completion_length": 1295.270896911621,
      "epoch": 0.232,
      "grad_norm": 0.25308915972709656,
      "kl": 0.008411407470703125,
      "learning_rate": 7.667891533457718e-07,
      "loss": 0.0003,
      "reward": 0.16257827286608517,
      "reward_std": 0.13092664163559675,
      "rewards/cosine_scaled_reward": 0.02888377010822296,
      "rewards/format_reward": 0.8958333432674408,
      "step": 203
    },
    {
      "completion_length": 1253.0417022705078,
      "epoch": 0.23314285714285715,
      "grad_norm": 0.36675336956977844,
      "kl": 0.01125335693359375,
      "learning_rate": 7.640308940816239e-07,
      "loss": 0.0005,
      "reward": 0.19385905005037785,
      "reward_std": 0.12034193379804492,
      "rewards/cosine_scaled_reward": 0.07674635015428066,
      "rewards/format_reward": 0.9583333358168602,
      "step": 204
    },
    {
      "completion_length": 1501.5417251586914,
      "epoch": 0.2342857142857143,
      "grad_norm": 0.2521401345729828,
      "kl": 0.00830841064453125,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0003,
      "reward": 0.24430920276790857,
      "reward_std": 0.17064828611910343,
      "rewards/cosine_scaled_reward": 0.26697138883173466,
      "rewards/format_reward": 0.8958333432674408,
      "step": 205
    },
    {
      "completion_length": 1941.3958740234375,
      "epoch": 0.23542857142857143,
      "grad_norm": 0.30331704020500183,
      "kl": 0.00870513916015625,
      "learning_rate": 7.584832158039378e-07,
      "loss": 0.0003,
      "reward": 0.045394688844680786,
      "reward_std": 0.09498168341815472,
      "rewards/cosine_scaled_reward": -0.24192016012966633,
      "rewards/format_reward": 0.7500000149011612,
      "step": 206
    },
    {
      "completion_length": 1438.8333587646484,
      "epoch": 0.23657142857142857,
      "grad_norm": 0.31282100081443787,
      "kl": 0.0115203857421875,
      "learning_rate": 7.556940671764124e-07,
      "loss": 0.0005,
      "reward": 0.11734095495194197,
      "reward_std": 0.1474492819979787,
      "rewards/cosine_scaled_reward": -0.08522756304591894,
      "rewards/format_reward": 0.8541666939854622,
      "step": 207
    },
    {
      "completion_length": 1113.7500228881836,
      "epoch": 0.2377142857142857,
      "grad_norm": 0.2624795734882355,
      "kl": 0.008823394775390625,
      "learning_rate": 7.528948933102438e-07,
      "loss": 0.0004,
      "reward": 0.17374667339026928,
      "reward_std": 0.10973423393443227,
      "rewards/cosine_scaled_reward": 0.023771056905388832,
      "rewards/format_reward": 0.9583333432674408,
      "step": 208
    },
    {
      "completion_length": 1030.7708625793457,
      "epoch": 0.23885714285714285,
      "grad_norm": 0.32483094930648804,
      "kl": 0.00803375244140625,
      "learning_rate": 7.500858306332172e-07,
      "loss": 0.0003,
      "reward": 0.18928107433021069,
      "reward_std": 0.10979128838516772,
      "rewards/cosine_scaled_reward": 0.07668027561157942,
      "rewards/format_reward": 0.9583333432674408,
      "step": 209
    },
    {
      "completion_length": 1972.9167022705078,
      "epoch": 0.24,
      "grad_norm": 0.17738643288612366,
      "kl": 0.008514404296875,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.0003,
      "reward": 0.14470667950809002,
      "reward_std": 0.13229582412168384,
      "rewards/cosine_scaled_reward": 0.037202537059783936,
      "rewards/format_reward": 0.7708333395421505,
      "step": 210
    },
    {
      "completion_length": 1598.7500228881836,
      "epoch": 0.24114285714285713,
      "grad_norm": 0.27847760915756226,
      "kl": 0.011180877685546875,
      "learning_rate": 7.444385869608921e-07,
      "loss": 0.0004,
      "reward": 0.14864719624165446,
      "reward_std": 0.11434817017288879,
      "rewards/cosine_scaled_reward": 0.04204285331070423,
      "rewards/format_reward": 0.7916666772216558,
      "step": 211
    },
    {
      "completion_length": 1177.6875190734863,
      "epoch": 0.2422857142857143,
      "grad_norm": 0.28754156827926636,
      "kl": 0.01055145263671875,
      "learning_rate": 7.416006812042827e-07,
      "loss": 0.0004,
      "reward": 0.17801955621689558,
      "reward_std": 0.10441821068525314,
      "rewards/cosine_scaled_reward": 0.1050565280020237,
      "rewards/format_reward": 0.8333333414047956,
      "step": 212
    },
    {
      "completion_length": 1114.354190826416,
      "epoch": 0.24342857142857144,
      "grad_norm": 0.4188780188560486,
      "kl": 0.011600494384765625,
      "learning_rate": 7.387534371007797e-07,
      "loss": 0.0005,
      "reward": 0.17088149162009358,
      "reward_std": 0.11700731026940048,
      "rewards/cosine_scaled_reward": 0.030954405665397644,
      "rewards/format_reward": 0.9375000074505806,
      "step": 213
    },
    {
      "completion_length": 1921.0833587646484,
      "epoch": 0.24457142857142858,
      "grad_norm": 0.22250115871429443,
      "kl": 0.010768890380859375,
      "learning_rate": 7.358969934210438e-07,
      "loss": 0.0004,
      "reward": 0.165664148516953,
      "reward_std": 0.15507009578868747,
      "rewards/cosine_scaled_reward": 0.08362742932513356,
      "rewards/format_reward": 0.791666679084301,
      "step": 214
    },
    {
      "completion_length": 1409.3333587646484,
      "epoch": 0.24571428571428572,
      "grad_norm": 0.26721227169036865,
      "kl": 0.007991790771484375,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0003,
      "reward": 0.11459713708609343,
      "reward_std": 0.10593406949192286,
      "rewards/cosine_scaled_reward": -0.08912589284591377,
      "rewards/format_reward": 0.8333333432674408,
      "step": 215
    },
    {
      "completion_length": 1280.6042175292969,
      "epoch": 0.24685714285714286,
      "grad_norm": 0.31878677010536194,
      "kl": 0.010250091552734375,
      "learning_rate": 7.301570646506027e-07,
      "loss": 0.0004,
      "reward": 0.19101236946880817,
      "reward_std": 0.11184050468727946,
      "rewards/cosine_scaled_reward": 0.12221446633338928,
      "rewards/format_reward": 0.8750000111758709,
      "step": 216
    },
    {
      "completion_length": 1485.3541870117188,
      "epoch": 0.248,
      "grad_norm": 0.24682621657848358,
      "kl": 0.00894927978515625,
      "learning_rate": 7.27273859315928e-07,
      "loss": 0.0004,
      "reward": 0.1820166790857911,
      "reward_std": 0.16682014428079128,
      "rewards/cosine_scaled_reward": 0.11264388589188457,
      "rewards/format_reward": 0.8333333395421505,
      "step": 217
    },
    {
      "completion_length": 1584.9583740234375,
      "epoch": 0.24914285714285714,
      "grad_norm": 0.28223466873168945,
      "kl": 0.010005950927734375,
      "learning_rate": 7.243820139034464e-07,
      "loss": 0.0004,
      "reward": 0.11426169364131056,
      "reward_std": 0.1477383803576231,
      "rewards/cosine_scaled_reward": -0.10390518826898187,
      "rewards/format_reward": 0.8750000149011612,
      "step": 218
    },
    {
      "completion_length": 1366.0625305175781,
      "epoch": 0.2502857142857143,
      "grad_norm": 0.3087660074234009,
      "kl": 0.0101165771484375,
      "learning_rate": 7.214816693576234e-07,
      "loss": 0.0004,
      "reward": 0.162561041302979,
      "reward_std": 0.15994944656267762,
      "rewards/cosine_scaled_reward": 0.03411710192449391,
      "rewards/format_reward": 0.8750000055879354,
      "step": 219
    },
    {
      "completion_length": 1414.1666717529297,
      "epoch": 0.25142857142857145,
      "grad_norm": 0.30027467012405396,
      "kl": 0.007183074951171875,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0003,
      "reward": 0.04776344425044954,
      "reward_std": 0.06931147351861,
      "rewards/cosine_scaled_reward": -0.2924546115100384,
      "rewards/format_reward": 0.8541666772216558,
      "step": 220
    },
    {
      "completion_length": 1363.2292022705078,
      "epoch": 0.25257142857142856,
      "grad_norm": 0.32602623105049133,
      "kl": 0.00916290283203125,
      "learning_rate": 7.156560487081051e-07,
      "loss": 0.0004,
      "reward": 0.17115566816937644,
      "reward_std": 0.10846680961549282,
      "rewards/cosine_scaled_reward": 0.0650689210742712,
      "rewards/format_reward": 0.8750000149011612,
      "step": 221
    },
    {
      "completion_length": 1461.8542022705078,
      "epoch": 0.2537142857142857,
      "grad_norm": 0.2890350818634033,
      "kl": 0.009174346923828125,
      "learning_rate": 7.127310565369415e-07,
      "loss": 0.0004,
      "reward": 0.1370222427067347,
      "reward_std": 0.13589494908228517,
      "rewards/cosine_scaled_reward": -0.022166259586811066,
      "rewards/format_reward": 0.8333333395421505,
      "step": 222
    },
    {
      "completion_length": 1474.7500381469727,
      "epoch": 0.25485714285714284,
      "grad_norm": 0.22688108682632446,
      "kl": 0.008312225341796875,
      "learning_rate": 7.097981330836616e-07,
      "loss": 0.0003,
      "reward": 0.1485723494552076,
      "reward_std": 0.10952452756464481,
      "rewards/cosine_scaled_reward": 0.029805000871419907,
      "rewards/format_reward": 0.8125000074505806,
      "step": 223
    },
    {
      "completion_length": 1974.1458892822266,
      "epoch": 0.256,
      "grad_norm": 0.24594879150390625,
      "kl": 0.008819580078125,
      "learning_rate": 7.068574212948169e-07,
      "loss": 0.0004,
      "reward": 0.11885308753699064,
      "reward_std": 0.1231303745880723,
      "rewards/cosine_scaled_reward": -0.07305323181208223,
      "rewards/format_reward": 0.8333333432674408,
      "step": 224
    },
    {
      "completion_length": 1713.7500305175781,
      "epoch": 0.2571428571428571,
      "grad_norm": 0.2284773290157318,
      "kl": 0.014232635498046875,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0006,
      "reward": 0.10374407912604511,
      "reward_std": 0.14682644978165627,
      "rewards/cosine_scaled_reward": -0.10095677326899022,
      "rewards/format_reward": 0.8125000111758709,
      "step": 225
    },
    {
      "completion_length": 1503.7708740234375,
      "epoch": 0.2582857142857143,
      "grad_norm": 0.23106862604618073,
      "kl": 0.00830841064453125,
      "learning_rate": 7.009532063876148e-07,
      "loss": 0.0003,
      "reward": 0.18740362441167235,
      "reward_std": 0.08802895061671734,
      "rewards/cosine_scaled_reward": 0.10540169104933739,
      "rewards/format_reward": 0.8958333395421505,
      "step": 226
    },
    {
      "completion_length": 1244.1875534057617,
      "epoch": 0.25942857142857145,
      "grad_norm": 0.2707884907722473,
      "kl": 0.013683319091796875,
      "learning_rate": 6.979899910323624e-07,
      "loss": 0.0005,
      "reward": 0.15233041066676378,
      "reward_std": 0.11700486252084374,
      "rewards/cosine_scaled_reward": -0.04158254712820053,
      "rewards/format_reward": 0.9583333432674408,
      "step": 227
    },
    {
      "completion_length": 1151.6666870117188,
      "epoch": 0.26057142857142856,
      "grad_norm": 0.31404733657836914,
      "kl": 0.00904083251953125,
      "learning_rate": 6.950195628537299e-07,
      "loss": 0.0004,
      "reward": 0.1969418814405799,
      "reward_std": 0.11795947467908263,
      "rewards/cosine_scaled_reward": 0.13468213769374415,
      "rewards/format_reward": 0.8958333395421505,
      "step": 228
    },
    {
      "completion_length": 1607.1875610351562,
      "epoch": 0.26171428571428573,
      "grad_norm": 0.3030518889427185,
      "kl": 0.011165618896484375,
      "learning_rate": 6.920420666261961e-07,
      "loss": 0.0004,
      "reward": 0.11387707642279565,
      "reward_std": 0.0794507262762636,
      "rewards/cosine_scaled_reward": -0.06163225881755352,
      "rewards/format_reward": 0.7916666828095913,
      "step": 229
    },
    {
      "completion_length": 1865.7292022705078,
      "epoch": 0.26285714285714284,
      "grad_norm": 0.26728665828704834,
      "kl": 0.009771347045898438,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.0004,
      "reward": 0.06328508502338082,
      "reward_std": 0.11280694883316755,
      "rewards/cosine_scaled_reward": -0.1895492672920227,
      "rewards/format_reward": 0.7500000074505806,
      "step": 230
    },
    {
      "completion_length": 1559.5625610351562,
      "epoch": 0.264,
      "grad_norm": 0.25874829292297363,
      "kl": 0.009349822998046875,
      "learning_rate": 6.860664508377001e-07,
      "loss": 0.0004,
      "reward": 0.22241820394992828,
      "reward_std": 0.11675288947299123,
      "rewards/cosine_scaled_reward": 0.19621967896819115,
      "rewards/format_reward": 0.9166666716337204,
      "step": 231
    },
    {
      "completion_length": 1832.2291870117188,
      "epoch": 0.2651428571428571,
      "grad_norm": 0.2535684406757355,
      "kl": 0.011371612548828125,
      "learning_rate": 6.83068622519821e-07,
      "loss": 0.0005,
      "reward": 0.04174727539066225,
      "reward_std": 0.08498770324513316,
      "rewards/cosine_scaled_reward": -0.2521397266536951,
      "rewards/format_reward": 0.7500000111758709,
      "step": 232
    },
    {
      "completion_length": 1139.8958587646484,
      "epoch": 0.2662857142857143,
      "grad_norm": 0.24799145758152008,
      "kl": 0.00756072998046875,
      "learning_rate": 6.800643086250121e-07,
      "loss": 0.0003,
      "reward": 0.10831367457285523,
      "reward_std": 0.10008358396589756,
      "rewards/cosine_scaled_reward": -0.16862575709819794,
      "rewards/format_reward": 0.9583333432674408,
      "step": 233
    },
    {
      "completion_length": 1544.5000305175781,
      "epoch": 0.2674285714285714,
      "grad_norm": 0.2703922986984253,
      "kl": 0.0122222900390625,
      "learning_rate": 6.770536555792944e-07,
      "loss": 0.0005,
      "reward": 0.13072135020047426,
      "reward_std": 0.12703408766537905,
      "rewards/cosine_scaled_reward": -0.010831212624907494,
      "rewards/format_reward": 0.791666679084301,
      "step": 234
    },
    {
      "completion_length": 1198.8125267028809,
      "epoch": 0.26857142857142857,
      "grad_norm": 0.33897149562835693,
      "kl": 0.01023101806640625,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0004,
      "reward": 0.22266705462243408,
      "reward_std": 0.13369846408022568,
      "rewards/cosine_scaled_reward": 0.17654232122004032,
      "rewards/format_reward": 0.9375000074505806,
      "step": 235
    },
    {
      "completion_length": 1820.2500457763672,
      "epoch": 0.26971428571428574,
      "grad_norm": 0.22755910456180573,
      "kl": 0.008855819702148438,
      "learning_rate": 6.710139192768694e-07,
      "loss": 0.0004,
      "reward": 0.10096835857257247,
      "reward_std": 0.12764764530584216,
      "rewards/cosine_scaled_reward": -0.10865432699210942,
      "rewards/format_reward": 0.8125000055879354,
      "step": 236
    },
    {
      "completion_length": 1567.0208587646484,
      "epoch": 0.27085714285714285,
      "grad_norm": 0.23869018256664276,
      "kl": 0.010875701904296875,
      "learning_rate": 6.679851303883891e-07,
      "loss": 0.0004,
      "reward": 0.1583275799639523,
      "reward_std": 0.07935558445751667,
      "rewards/cosine_scaled_reward": 0.06267662812024355,
      "rewards/format_reward": 0.8125000074505806,
      "step": 237
    },
    {
      "completion_length": 1344.5625305175781,
      "epoch": 0.272,
      "grad_norm": 0.23950256407260895,
      "kl": 0.01043701171875,
      "learning_rate": 6.649505910711058e-07,
      "loss": 0.0004,
      "reward": 0.2100483477115631,
      "reward_std": 0.10991842532530427,
      "rewards/cosine_scaled_reward": 0.15677512856200337,
      "rewards/format_reward": 0.9166666716337204,
      "step": 238
    },
    {
      "completion_length": 1426.041690826416,
      "epoch": 0.27314285714285713,
      "grad_norm": 0.2463754415512085,
      "kl": 0.0074310302734375,
      "learning_rate": 6.619104492241847e-07,
      "loss": 0.0003,
      "reward": 0.21384657500311732,
      "reward_std": 0.08286083268467337,
      "rewards/cosine_scaled_reward": 0.23679617792367935,
      "rewards/format_reward": 0.770833333954215,
      "step": 239
    },
    {
      "completion_length": 1488.8958587646484,
      "epoch": 0.2742857142857143,
      "grad_norm": 0.4307219088077545,
      "kl": 0.0166473388671875,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0007,
      "reward": 0.043208114642766304,
      "reward_std": 0.10311023378744721,
      "rewards/cosine_scaled_reward": -0.24894601851701736,
      "rewards/format_reward": 0.7500000149011612,
      "step": 240
    },
    {
      "completion_length": 1405.2916870117188,
      "epoch": 0.2754285714285714,
      "grad_norm": 0.42230215668678284,
      "kl": 0.013263702392578125,
      "learning_rate": 6.558139508961654e-07,
      "loss": 0.0005,
      "reward": 0.05663964038831182,
      "reward_std": 0.10991012584418058,
      "rewards/cosine_scaled_reward": -0.2536990698426962,
      "rewards/format_reward": 0.8333333432674408,
      "step": 241
    },
    {
      "completion_length": 1111.4375457763672,
      "epoch": 0.2765714285714286,
      "grad_norm": 0.365119606256485,
      "kl": 0.018003463745117188,
      "learning_rate": 6.527578915497951e-07,
      "loss": 0.0007,
      "reward": 0.12744820569059812,
      "reward_std": 0.1108874985948205,
      "rewards/cosine_scaled_reward": -0.08098506298847497,
      "rewards/format_reward": 0.8958333432674408,
      "step": 242
    },
    {
      "completion_length": 1557.8541870117188,
      "epoch": 0.2777142857142857,
      "grad_norm": 0.22312189638614655,
      "kl": 0.0106353759765625,
      "learning_rate": 6.496968239287603e-07,
      "loss": 0.0004,
      "reward": 0.1432976769283414,
      "reward_std": 0.11755505437031388,
      "rewards/cosine_scaled_reward": 0.002848621690645814,
      "rewards/format_reward": 0.8333333432674408,
      "step": 243
    },
    {
      "completion_length": 1817.9167022705078,
      "epoch": 0.27885714285714286,
      "grad_norm": 0.284006804227829,
      "kl": 0.01305389404296875,
      "learning_rate": 6.466308972251785e-07,
      "loss": 0.0005,
      "reward": 0.16071391198784113,
      "reward_std": 0.11166157713159919,
      "rewards/cosine_scaled_reward": 0.0663528572767973,
      "rewards/format_reward": 0.8125000074505806,
      "step": 244
    },
    {
      "completion_length": 1842.479232788086,
      "epoch": 0.28,
      "grad_norm": 0.26421064138412476,
      "kl": 0.01108551025390625,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0004,
      "reward": 0.15495485439896584,
      "reward_std": 0.14859340619295835,
      "rewards/cosine_scaled_reward": 0.060377851128578186,
      "rewards/format_reward": 0.7916666772216558,
      "step": 245
    },
    {
      "completion_length": 1367.7292098999023,
      "epoch": 0.28114285714285714,
      "grad_norm": 0.257500022649765,
      "kl": 0.01171112060546875,
      "learning_rate": 6.404850645156841e-07,
      "loss": 0.0005,
      "reward": 0.1178839597851038,
      "reward_std": 0.11170466430485249,
      "rewards/cosine_scaled_reward": -0.12124981544911861,
      "rewards/format_reward": 0.9375,
      "step": 246
    },
    {
      "completion_length": 2025.4167175292969,
      "epoch": 0.2822857142857143,
      "grad_norm": 0.2877947986125946,
      "kl": 0.014371871948242188,
      "learning_rate": 6.374054580489873e-07,
      "loss": 0.0006,
      "reward": 0.041426120849791914,
      "reward_std": 0.10642113536596298,
      "rewards/cosine_scaled_reward": -0.20261837355792522,
      "rewards/format_reward": 0.6458333432674408,
      "step": 247
    },
    {
      "completion_length": 1317.4167022705078,
      "epoch": 0.2834285714285714,
      "grad_norm": 0.3394499123096466,
      "kl": 0.012571334838867188,
      "learning_rate": 6.343215915635761e-07,
      "loss": 0.0005,
      "reward": 0.16258022841066122,
      "reward_std": 0.09610571060329676,
      "rewards/cosine_scaled_reward": 0.07010693056508899,
      "rewards/format_reward": 0.8125000074505806,
      "step": 248
    },
    {
      "completion_length": 1280.3125305175781,
      "epoch": 0.2845714285714286,
      "grad_norm": 0.27582064270973206,
      "kl": 0.011753082275390625,
      "learning_rate": 6.31233615362752e-07,
      "loss": 0.0005,
      "reward": 0.21597624802961946,
      "reward_std": 0.11335421586409211,
      "rewards/cosine_scaled_reward": 0.18638860061764717,
      "rewards/format_reward": 0.8958333358168602,
      "step": 249
    },
    {
      "completion_length": 1112.1875381469727,
      "epoch": 0.2857142857142857,
      "grad_norm": 0.3718562424182892,
      "kl": 0.01335906982421875,
      "learning_rate": 6.281416799501187e-07,
      "loss": 0.0005,
      "reward": 0.1376856635324657,
      "reward_std": 0.06630115583539009,
      "rewards/cosine_scaled_reward": -0.09413989027962089,
      "rewards/format_reward": 0.9791666716337204,
      "step": 250
    },
    {
      "completion_length": 969.0625152587891,
      "epoch": 0.28685714285714287,
      "grad_norm": 0.4498876631259918,
      "kl": 0.01354217529296875,
      "learning_rate": 6.25045936022246e-07,
      "loss": 0.0005,
      "reward": 0.13784294662764296,
      "reward_std": 0.11408048821613193,
      "rewards/cosine_scaled_reward": -0.029557042755186558,
      "rewards/format_reward": 0.8541666828095913,
      "step": 251
    },
    {
      "completion_length": 1556.2500534057617,
      "epoch": 0.288,
      "grad_norm": 0.35213375091552734,
      "kl": 0.01563262939453125,
      "learning_rate": 6.219465344613258e-07,
      "loss": 0.0006,
      "reward": 0.10068062460049987,
      "reward_std": 0.08584798872470856,
      "rewards/cosine_scaled_reward": -0.10943282302469015,
      "rewards/format_reward": 0.812500013038516,
      "step": 252
    },
    {
      "completion_length": 1594.7292175292969,
      "epoch": 0.28914285714285715,
      "grad_norm": 0.36279869079589844,
      "kl": 0.019073486328125,
      "learning_rate": 6.188436263278172e-07,
      "loss": 0.0008,
      "reward": 0.10339808277785778,
      "reward_std": 0.14174744859337807,
      "rewards/cosine_scaled_reward": -0.09498709812760353,
      "rewards/format_reward": 0.7916666734963655,
      "step": 253
    },
    {
      "completion_length": 1617.0417022705078,
      "epoch": 0.29028571428571426,
      "grad_norm": 0.37625786662101746,
      "kl": 0.016139984130859375,
      "learning_rate": 6.157373628530852e-07,
      "loss": 0.0006,
      "reward": 0.06562280771322548,
      "reward_std": 0.09647063678130507,
      "rewards/cosine_scaled_reward": -0.20810320507735014,
      "rewards/format_reward": 0.791666679084301,
      "step": 254
    },
    {
      "completion_length": 1775.500015258789,
      "epoch": 0.2914285714285714,
      "grad_norm": 0.30141302943229675,
      "kl": 0.012451171875,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0005,
      "reward": 0.03216266352683306,
      "reward_std": 0.09485536953434348,
      "rewards/cosine_scaled_reward": -0.2940612696111202,
      "rewards/format_reward": 0.7708333432674408,
      "step": 255
    },
    {
      "completion_length": 1419.5833740234375,
      "epoch": 0.2925714285714286,
      "grad_norm": 0.3471844494342804,
      "kl": 0.0146484375,
      "learning_rate": 6.095153756157051e-07,
      "loss": 0.0006,
      "reward": 0.15079816803336143,
      "reward_std": 0.10951651586219668,
      "rewards/cosine_scaled_reward": -0.006679622456431389,
      "rewards/format_reward": 0.8958333432674408,
      "step": 256
    },
    {
      "completion_length": 1932.9583892822266,
      "epoch": 0.2937142857142857,
      "grad_norm": 0.22311758995056152,
      "kl": 0.01297760009765625,
      "learning_rate": 6.06399955103937e-07,
      "loss": 0.0005,
      "reward": 0.20792145188897848,
      "reward_std": 0.12192836869508028,
      "rewards/cosine_scaled_reward": 0.19739503040909767,
      "rewards/format_reward": 0.8333333395421505,
      "step": 257
    },
    {
      "completion_length": 1696.458366394043,
      "epoch": 0.2948571428571429,
      "grad_norm": 0.3503570556640625,
      "kl": 0.012058258056640625,
      "learning_rate": 6.032817857379256e-07,
      "loss": 0.0005,
      "reward": 0.111656179651618,
      "reward_std": 0.14000094681978226,
      "rewards/cosine_scaled_reward": -0.06675757747143507,
      "rewards/format_reward": 0.7916666772216558,
      "step": 258
    },
    {
      "completion_length": 1482.2292022705078,
      "epoch": 0.296,
      "grad_norm": 0.38044440746307373,
      "kl": 0.016082763671875,
      "learning_rate": 6.001610194928464e-07,
      "loss": 0.0006,
      "reward": 0.14132701233029366,
      "reward_std": 0.09021098469384015,
      "rewards/cosine_scaled_reward": 0.016738089732825756,
      "rewards/format_reward": 0.791666679084301,
      "step": 259
    },
    {
      "completion_length": 1308.8958930969238,
      "epoch": 0.29714285714285715,
      "grad_norm": 0.3778381943702698,
      "kl": 0.0111083984375,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.0004,
      "reward": 0.20748187974095345,
      "reward_std": 0.1426714597037062,
      "rewards/cosine_scaled_reward": 0.14869027212262154,
      "rewards/format_reward": 0.916666679084301,
      "step": 260
    },
    {
      "completion_length": 2185.395866394043,
      "epoch": 0.29828571428571427,
      "grad_norm": 0.29314514994621277,
      "kl": 0.019428253173828125,
      "learning_rate": 5.939123048916173e-07,
      "loss": 0.0008,
      "reward": 0.05902714841067791,
      "reward_std": 0.09425672655925155,
      "rewards/cosine_scaled_reward": -0.1403571031987667,
      "rewards/format_reward": 0.6250000111758709,
      "step": 261
    },
    {
      "completion_length": 1879.7917175292969,
      "epoch": 0.29942857142857143,
      "grad_norm": 0.46697157621383667,
      "kl": 0.020751953125,
      "learning_rate": 5.907846610890011e-07,
      "loss": 0.0008,
      "reward": 0.04076826642267406,
      "reward_std": 0.10802473686635494,
      "rewards/cosine_scaled_reward": -0.2183469645678997,
      "rewards/format_reward": 0.6666666828095913,
      "step": 262
    },
    {
      "completion_length": 1427.145881652832,
      "epoch": 0.30057142857142854,
      "grad_norm": 0.2928243577480316,
      "kl": 0.0118255615234375,
      "learning_rate": 5.87655029499542e-07,
      "loss": 0.0005,
      "reward": 0.05615242966450751,
      "reward_std": 0.06438650703057647,
      "rewards/cosine_scaled_reward": -0.2520607812330127,
      "rewards/format_reward": 0.8333333488553762,
      "step": 263
    },
    {
      "completion_length": 1357.7292251586914,
      "epoch": 0.3017142857142857,
      "grad_norm": 0.3987843096256256,
      "kl": 0.012592315673828125,
      "learning_rate": 5.845235626570683e-07,
      "loss": 0.0005,
      "reward": 0.11286137904971838,
      "reward_std": 0.13379723951220512,
      "rewards/cosine_scaled_reward": -0.10757828690111637,
      "rewards/format_reward": 0.8750000223517418,
      "step": 264
    },
    {
      "completion_length": 1381.5000381469727,
      "epoch": 0.3028571428571429,
      "grad_norm": 0.3415836989879608,
      "kl": 0.0178070068359375,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.0007,
      "reward": 0.1456056940369308,
      "reward_std": 0.14075131434947252,
      "rewards/cosine_scaled_reward": -0.009397267829626799,
      "rewards/format_reward": 0.8541666716337204,
      "step": 265
    },
    {
      "completion_length": 1806.1250610351562,
      "epoch": 0.304,
      "grad_norm": 0.38759687542915344,
      "kl": 0.01422119140625,
      "learning_rate": 5.78255733788191e-07,
      "loss": 0.0006,
      "reward": 0.1108898997772485,
      "reward_std": 0.11440710537135601,
      "rewards/cosine_scaled_reward": -0.08643979020416737,
      "rewards/format_reward": 0.8125000074505806,
      "step": 266
    },
    {
      "completion_length": 2056.729202270508,
      "epoch": 0.30514285714285716,
      "grad_norm": 0.42212945222854614,
      "kl": 0.026676177978515625,
      "learning_rate": 5.751196772469237e-07,
      "loss": 0.0011,
      "reward": 0.022361958224792033,
      "reward_std": 0.08537128940224648,
      "rewards/cosine_scaled_reward": -0.24759646970778704,
      "rewards/format_reward": 0.6250000167638063,
      "step": 267
    },
    {
      "completion_length": 1231.9166870117188,
      "epoch": 0.3062857142857143,
      "grad_norm": 0.5157231092453003,
      "kl": 0.02033233642578125,
      "learning_rate": 5.71982396408026e-07,
      "loss": 0.0008,
      "reward": 0.0841333303033025,
      "reward_std": 0.10015194956213236,
      "rewards/cosine_scaled_reward": -0.18771476298570633,
      "rewards/format_reward": 0.854166679084301,
      "step": 268
    },
    {
      "completion_length": 1421.8541870117188,
      "epoch": 0.30742857142857144,
      "grad_norm": 0.3175782859325409,
      "kl": 0.015338897705078125,
      "learning_rate": 5.688440441781398e-07,
      "loss": 0.0006,
      "reward": 0.0848088227212429,
      "reward_std": 0.10533672664314508,
      "rewards/cosine_scaled_reward": -0.1579919238574803,
      "rewards/format_reward": 0.8125,
      "step": 269
    },
    {
      "completion_length": 1852.9792251586914,
      "epoch": 0.30857142857142855,
      "grad_norm": 0.31283366680145264,
      "kl": 0.020725250244140625,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0008,
      "reward": 0.1707673908676952,
      "reward_std": 0.14836188638582826,
      "rewards/cosine_scaled_reward": 0.11553415982052684,
      "rewards/format_reward": 0.7708333432674408,
      "step": 270
    },
    {
      "completion_length": 1270.8333625793457,
      "epoch": 0.3097142857142857,
      "grad_norm": 0.4045168459415436,
      "kl": 0.020198822021484375,
      "learning_rate": 5.625647374256061e-07,
      "loss": 0.0008,
      "reward": 0.2056138776242733,
      "reward_std": 0.1318775275722146,
      "rewards/cosine_scaled_reward": 0.1646625578578096,
      "rewards/format_reward": 0.8750000111758709,
      "step": 271
    },
    {
      "completion_length": 1756.8542098999023,
      "epoch": 0.31085714285714283,
      "grad_norm": 0.34620898962020874,
      "kl": 0.02004241943359375,
      "learning_rate": 5.594240889475106e-07,
      "loss": 0.0008,
      "reward": 0.12157785186354886,
      "reward_std": 0.1287962575443089,
      "rewards/cosine_scaled_reward": -0.04085011733695865,
      "rewards/format_reward": 0.791666679084301,
      "step": 272
    },
    {
      "completion_length": 1281.5000381469727,
      "epoch": 0.312,
      "grad_norm": 0.4477139115333557,
      "kl": 0.0211029052734375,
      "learning_rate": 5.562829811526154e-07,
      "loss": 0.0008,
      "reward": 0.1495568435639143,
      "reward_std": 0.1047550356015563,
      "rewards/cosine_scaled_reward": 0.03370687458664179,
      "rewards/format_reward": 0.8125000111758709,
      "step": 273
    },
    {
      "completion_length": 1047.6250457763672,
      "epoch": 0.31314285714285717,
      "grad_norm": 0.38608211278915405,
      "kl": 0.015705108642578125,
      "learning_rate": 5.531415671340826e-07,
      "loss": 0.0006,
      "reward": 0.16697457217378542,
      "reward_std": 0.1347856274805963,
      "rewards/cosine_scaled_reward": 0.032155007666005986,
      "rewards/format_reward": 0.8958333432674408,
      "step": 274
    },
    {
      "completion_length": 1519.5625534057617,
      "epoch": 0.3142857142857143,
      "grad_norm": 0.41990184783935547,
      "kl": 0.02295684814453125,
      "learning_rate": 5.5e-07,
      "loss": 0.0009,
      "reward": 0.19299401948228478,
      "reward_std": 0.1319624213501811,
      "rewards/cosine_scaled_reward": 0.15465315023902804,
      "rewards/format_reward": 0.8125000074505806,
      "step": 275
    },
    {
      "completion_length": 1206.583351135254,
      "epoch": 0.31542857142857145,
      "grad_norm": 0.38451164960861206,
      "kl": 0.020355224609375,
      "learning_rate": 5.468584328659172e-07,
      "loss": 0.0008,
      "reward": 0.14626443712040782,
      "reward_std": 0.13966891495510936,
      "rewards/cosine_scaled_reward": 0.004510253042099066,
      "rewards/format_reward": 0.8333333432674408,
      "step": 276
    },
    {
      "completion_length": 1248.2917022705078,
      "epoch": 0.31657142857142856,
      "grad_norm": 0.5081427097320557,
      "kl": 0.02585601806640625,
      "learning_rate": 5.437170188473847e-07,
      "loss": 0.001,
      "reward": 0.14069768914487213,
      "reward_std": 0.12813527416437864,
      "rewards/cosine_scaled_reward": -0.05632503447122872,
      "rewards/format_reward": 0.916666679084301,
      "step": 277
    },
    {
      "completion_length": 1363.895866394043,
      "epoch": 0.3177142857142857,
      "grad_norm": 0.5567955374717712,
      "kl": 0.02330780029296875,
      "learning_rate": 5.405759110524894e-07,
      "loss": 0.0009,
      "reward": 0.1985365085711237,
      "reward_std": 0.08099143509753048,
      "rewards/cosine_scaled_reward": 0.12419202888850123,
      "rewards/format_reward": 0.875,
      "step": 278
    },
    {
      "completion_length": 1633.9583740234375,
      "epoch": 0.31885714285714284,
      "grad_norm": 0.42431047558784485,
      "kl": 0.035366058349609375,
      "learning_rate": 5.37435262574394e-07,
      "loss": 0.0014,
      "reward": 0.10419091582298279,
      "reward_std": 0.15187329379841685,
      "rewards/cosine_scaled_reward": -0.07383611425757408,
      "rewards/format_reward": 0.7500000093132257,
      "step": 279
    },
    {
      "completion_length": 1602.2292251586914,
      "epoch": 0.32,
      "grad_norm": 0.3582462668418884,
      "kl": 0.042247772216796875,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0017,
      "reward": 0.20170635590329766,
      "reward_std": 0.15067243855446577,
      "rewards/cosine_scaled_reward": 0.19395790994167328,
      "rewards/format_reward": 0.7708333395421505,
      "step": 280
    },
    {
      "completion_length": 2467.0416717529297,
      "epoch": 0.3211428571428571,
      "grad_norm": 0.29816770553588867,
      "kl": 0.05840301513671875,
      "learning_rate": 5.311559558218603e-07,
      "loss": 0.0023,
      "reward": 0.05792845832183957,
      "reward_std": 0.14697478245943785,
      "rewards/cosine_scaled_reward": -0.10388723947107792,
      "rewards/format_reward": 0.541666679084301,
      "step": 281
    },
    {
      "completion_length": 1241.5416946411133,
      "epoch": 0.3222857142857143,
      "grad_norm": 0.46031466126441956,
      "kl": 0.0266265869140625,
      "learning_rate": 5.28017603591974e-07,
      "loss": 0.0011,
      "reward": 0.13118357677012682,
      "reward_std": 0.13791329925879836,
      "rewards/cosine_scaled_reward": -0.02069989638403058,
      "rewards/format_reward": 0.8125000074505806,
      "step": 282
    },
    {
      "completion_length": 1868.6250228881836,
      "epoch": 0.32342857142857145,
      "grad_norm": 0.32365959882736206,
      "kl": 0.052989959716796875,
      "learning_rate": 5.248803227530763e-07,
      "loss": 0.0021,
      "reward": 0.18105492927134037,
      "reward_std": 0.13572940416634083,
      "rewards/cosine_scaled_reward": 0.16773403156548738,
      "rewards/format_reward": 0.7291666828095913,
      "step": 283
    },
    {
      "completion_length": 1280.8542098999023,
      "epoch": 0.32457142857142857,
      "grad_norm": 0.4519507586956024,
      "kl": 0.019351959228515625,
      "learning_rate": 5.21744266211809e-07,
      "loss": 0.0008,
      "reward": 0.11721091519575566,
      "reward_std": 0.15023711137473583,
      "rewards/cosine_scaled_reward": -0.09081541141495109,
      "rewards/format_reward": 0.8541666865348816,
      "step": 284
    },
    {
      "completion_length": 1140.1250305175781,
      "epoch": 0.32571428571428573,
      "grad_norm": 0.7309654951095581,
      "kl": 0.0377349853515625,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0015,
      "reward": 0.09533977252431214,
      "reward_std": 0.11678969115018845,
      "rewards/cosine_scaled_reward": -0.12558545544743538,
      "rewards/format_reward": 0.8125000186264515,
      "step": 285
    },
    {
      "completion_length": 1449.3958740234375,
      "epoch": 0.32685714285714285,
      "grad_norm": 0.49403443932533264,
      "kl": 0.0523681640625,
      "learning_rate": 5.154764373429315e-07,
      "loss": 0.0021,
      "reward": 0.1131226432044059,
      "reward_std": 0.12507515028119087,
      "rewards/cosine_scaled_reward": -0.08685349836014211,
      "rewards/format_reward": 0.8125000149011612,
      "step": 286
    },
    {
      "completion_length": 1521.6250457763672,
      "epoch": 0.328,
      "grad_norm": 0.858707070350647,
      "kl": 0.069488525390625,
      "learning_rate": 5.123449705004581e-07,
      "loss": 0.0028,
      "reward": 0.11133736907504499,
      "reward_std": 0.11151464702561498,
      "rewards/cosine_scaled_reward": -0.03888722602277994,
      "rewards/format_reward": 0.708333345130086,
      "step": 287
    },
    {
      "completion_length": 1589.6458892822266,
      "epoch": 0.3291428571428571,
      "grad_norm": 0.5665430426597595,
      "kl": 0.059162139892578125,
      "learning_rate": 5.09215338910999e-07,
      "loss": 0.0024,
      "reward": 0.11726528691360727,
      "reward_std": 0.11230942467227578,
      "rewards/cosine_scaled_reward": -0.06808646989520639,
      "rewards/format_reward": 0.8125000111758709,
      "step": 288
    },
    {
      "completion_length": 1320.7916831970215,
      "epoch": 0.3302857142857143,
      "grad_norm": 0.8541005849838257,
      "kl": 0.035400390625,
      "learning_rate": 5.060876951083828e-07,
      "loss": 0.0014,
      "reward": 0.12517158885020763,
      "reward_std": 0.07708289241418242,
      "rewards/cosine_scaled_reward": -0.030706046149134636,
      "rewards/format_reward": 0.7916666772216558,
      "step": 289
    },
    {
      "completion_length": 1165.520866394043,
      "epoch": 0.3314285714285714,
      "grad_norm": 0.45539847016334534,
      "kl": 0.050296783447265625,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.002,
      "reward": 0.1758332857862115,
      "reward_std": 0.14369960315525532,
      "rewards/cosine_scaled_reward": 0.06662235781550407,
      "rewards/format_reward": 0.895833358168602,
      "step": 290
    },
    {
      "completion_length": 1888.7292556762695,
      "epoch": 0.3325714285714286,
      "grad_norm": 0.4208749234676361,
      "kl": 0.10372543334960938,
      "learning_rate": 4.998389805071536e-07,
      "loss": 0.0041,
      "reward": 0.15313149709254503,
      "reward_std": 0.14885053224861622,
      "rewards/cosine_scaled_reward": 0.030359832802787423,
      "rewards/format_reward": 0.8333333414047956,
      "step": 291
    },
    {
      "completion_length": 1906.958381652832,
      "epoch": 0.33371428571428574,
      "grad_norm": 1.015468716621399,
      "kl": 0.11415481567382812,
      "learning_rate": 4.967182142620745e-07,
      "loss": 0.0046,
      "reward": 0.08511171862483025,
      "reward_std": 0.11772025120444596,
      "rewards/cosine_scaled_reward": -0.146739911288023,
      "rewards/format_reward": 0.7916666865348816,
      "step": 292
    },
    {
      "completion_length": 1338.8541946411133,
      "epoch": 0.33485714285714285,
      "grad_norm": 0.5531266331672668,
      "kl": 0.0963287353515625,
      "learning_rate": 4.93600044896063e-07,
      "loss": 0.0039,
      "reward": 0.14025127002969384,
      "reward_std": 0.12792299408465624,
      "rewards/cosine_scaled_reward": -0.01351697463542223,
      "rewards/format_reward": 0.8541666772216558,
      "step": 293
    },
    {
      "completion_length": 1772.7291870117188,
      "epoch": 0.336,
      "grad_norm": 0.9273716807365417,
      "kl": 0.05928802490234375,
      "learning_rate": 4.904846243842949e-07,
      "loss": 0.0024,
      "reward": 0.14557143417187035,
      "reward_std": 0.13822759315371513,
      "rewards/cosine_scaled_reward": 0.04577969899401069,
      "rewards/format_reward": 0.7500000149011612,
      "step": 294
    },
    {
      "completion_length": 1621.5833587646484,
      "epoch": 0.33714285714285713,
      "grad_norm": 0.7160328030586243,
      "kl": 0.06582260131835938,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0026,
      "reward": 0.15534777799621224,
      "reward_std": 0.12278164038434625,
      "rewards/cosine_scaled_reward": 0.053630582988262177,
      "rewards/format_reward": 0.7916666753590107,
      "step": 295
    },
    {
      "completion_length": 1988.916732788086,
      "epoch": 0.3382857142857143,
      "grad_norm": 1.5293620824813843,
      "kl": 0.130126953125,
      "learning_rate": 4.842626371469149e-07,
      "loss": 0.0052,
      "reward": 0.07497212127782404,
      "reward_std": 0.15273468242958188,
      "rewards/cosine_scaled_reward": -0.14556175749748945,
      "rewards/format_reward": 0.7291666902601719,
      "step": 296
    },
    {
      "completion_length": 2342.8959197998047,
      "epoch": 0.3394285714285714,
      "grad_norm": 1.5157654285430908,
      "kl": 0.10904693603515625,
      "learning_rate": 4.811563736721829e-07,
      "loss": 0.0044,
      "reward": 0.08573226723819971,
      "reward_std": 0.14976172288879752,
      "rewards/cosine_scaled_reward": -0.05238574789837003,
      "rewards/format_reward": 0.6041666753590107,
      "step": 297
    },
    {
      "completion_length": 1915.416732788086,
      "epoch": 0.3405714285714286,
      "grad_norm": 0.8163977861404419,
      "kl": 0.1429595947265625,
      "learning_rate": 4.780534655386743e-07,
      "loss": 0.0057,
      "reward": 0.10191577160730958,
      "reward_std": 0.10441284067928791,
      "rewards/cosine_scaled_reward": -0.054123382084071636,
      "rewards/format_reward": 0.7083333469927311,
      "step": 298
    },
    {
      "completion_length": 1942.7084045410156,
      "epoch": 0.3417142857142857,
      "grad_norm": 1.951476812362671,
      "kl": 0.17220306396484375,
      "learning_rate": 4.749540639777539e-07,
      "loss": 0.0069,
      "reward": 0.09106689637701493,
      "reward_std": 0.11453963397070765,
      "rewards/cosine_scaled_reward": -0.08032999746501446,
      "rewards/format_reward": 0.6875000186264515,
      "step": 299
    },
    {
      "completion_length": 2121.520881652832,
      "epoch": 0.34285714285714286,
      "grad_norm": 0.9551680684089661,
      "kl": 0.1997222900390625,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.008,
      "reward": 0.09341209102421999,
      "reward_std": 0.11039053509011865,
      "rewards/cosine_scaled_reward": -0.07323687896132469,
      "rewards/format_reward": 0.6875000149011612,
      "step": 300
    },
    {
      "completion_length": 2006.5209007263184,
      "epoch": 0.344,
      "grad_norm": 1.1690226793289185,
      "kl": 0.244354248046875,
      "learning_rate": 4.68766384637248e-07,
      "loss": 0.0098,
      "reward": 0.07932836120016873,
      "reward_std": 0.13190083391964436,
      "rewards/cosine_scaled_reward": -0.11513608321547508,
      "rewards/format_reward": 0.6875000093132257,
      "step": 301
    },
    {
      "completion_length": 1556.2917137145996,
      "epoch": 0.34514285714285714,
      "grad_norm": 1.1125872135162354,
      "kl": 0.10612106323242188,
      "learning_rate": 4.656784084364238e-07,
      "loss": 0.0042,
      "reward": 0.15471726842224598,
      "reward_std": 0.10859864950180054,
      "rewards/cosine_scaled_reward": 0.06700704153627157,
      "rewards/format_reward": 0.7708333414047956,
      "step": 302
    },
    {
      "completion_length": 1470.5625267028809,
      "epoch": 0.3462857142857143,
      "grad_norm": 0.939318060874939,
      "kl": 0.0955810546875,
      "learning_rate": 4.6259454195101267e-07,
      "loss": 0.0038,
      "reward": 0.16825311817228794,
      "reward_std": 0.13590177986770868,
      "rewards/cosine_scaled_reward": 0.02981522586196661,
      "rewards/format_reward": 0.9166666865348816,
      "step": 303
    },
    {
      "completion_length": 1612.6458740234375,
      "epoch": 0.3474285714285714,
      "grad_norm": 1.2758152484893799,
      "kl": 0.13702392578125,
      "learning_rate": 4.59514935484316e-07,
      "loss": 0.0055,
      "reward": 0.08096132357604802,
      "reward_std": 0.10056991688907146,
      "rewards/cosine_scaled_reward": -0.1730497945100069,
      "rewards/format_reward": 0.812500013038516,
      "step": 304
    },
    {
      "completion_length": 1444.104232788086,
      "epoch": 0.3485714285714286,
      "grad_norm": 1.4721267223358154,
      "kl": 0.09059906005859375,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.0036,
      "reward": 0.10656621214002371,
      "reward_std": 0.13656832091510296,
      "rewards/cosine_scaled_reward": -0.12406391743570566,
      "rewards/format_reward": 0.8750000223517418,
      "step": 305
    },
    {
      "completion_length": 1328.6458740234375,
      "epoch": 0.3497142857142857,
      "grad_norm": 1.2772217988967896,
      "kl": 0.14186859130859375,
      "learning_rate": 4.5336910277482155e-07,
      "loss": 0.0057,
      "reward": 0.19892889651237056,
      "reward_std": 0.11784483585506678,
      "rewards/cosine_scaled_reward": 0.14353829622268677,
      "rewards/format_reward": 0.8750000074505806,
      "step": 306
    },
    {
      "completion_length": 1276.645866394043,
      "epoch": 0.35085714285714287,
      "grad_norm": 1.0841631889343262,
      "kl": 0.130828857421875,
      "learning_rate": 4.503031760712397e-07,
      "loss": 0.0052,
      "reward": 0.12190989218652248,
      "reward_std": 0.12431731820106506,
      "rewards/cosine_scaled_reward": -0.08158679166808724,
      "rewards/format_reward": 0.8750000074505806,
      "step": 307
    },
    {
      "completion_length": 2021.0000305175781,
      "epoch": 0.352,
      "grad_norm": 1.4621251821517944,
      "kl": 0.2753448486328125,
      "learning_rate": 4.4724210845020494e-07,
      "loss": 0.011,
      "reward": 0.09099714574404061,
      "reward_std": 0.14960693335160613,
      "rewards/cosine_scaled_reward": -0.09915194474160671,
      "rewards/format_reward": 0.7291666716337204,
      "step": 308
    },
    {
      "completion_length": 2146.7083740234375,
      "epoch": 0.35314285714285715,
      "grad_norm": 2.272526502609253,
      "kl": 0.4404144287109375,
      "learning_rate": 4.441860491038345e-07,
      "loss": 0.0176,
      "reward": 0.09269980387762189,
      "reward_std": 0.14243671763688326,
      "rewards/cosine_scaled_reward": -0.06128171645104885,
      "rewards/format_reward": 0.666666679084301,
      "step": 309
    },
    {
      "completion_length": 1203.4791946411133,
      "epoch": 0.35428571428571426,
      "grad_norm": 1.225544810295105,
      "kl": 0.16626739501953125,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.0066,
      "reward": 0.10261391778476536,
      "reward_std": 0.08983314875513315,
      "rewards/cosine_scaled_reward": -0.16803579218685627,
      "rewards/format_reward": 0.916666679084301,
      "step": 310
    },
    {
      "completion_length": 1146.8125343322754,
      "epoch": 0.3554285714285714,
      "grad_norm": 1.5572764873504639,
      "kl": 0.07305145263671875,
      "learning_rate": 4.3808955077581546e-07,
      "loss": 0.0029,
      "reward": 0.19523340463638306,
      "reward_std": 0.14635545574128628,
      "rewards/cosine_scaled_reward": 0.08061425480991602,
      "rewards/format_reward": 0.9791666716337204,
      "step": 311
    },
    {
      "completion_length": 1333.0625228881836,
      "epoch": 0.3565714285714286,
      "grad_norm": 1.1539429426193237,
      "kl": 0.34932708740234375,
      "learning_rate": 4.350494089288943e-07,
      "loss": 0.014,
      "reward": 0.1985101569443941,
      "reward_std": 0.07966499030590057,
      "rewards/cosine_scaled_reward": 0.1561545841395855,
      "rewards/format_reward": 0.8541666697710752,
      "step": 312
    },
    {
      "completion_length": 2031.31254196167,
      "epoch": 0.3577142857142857,
      "grad_norm": 1.5145570039749146,
      "kl": 0.6246414184570312,
      "learning_rate": 4.3201486961161093e-07,
      "loss": 0.0249,
      "reward": 0.11477423517499119,
      "reward_std": 0.11296425701584667,
      "rewards/cosine_scaled_reward": 0.012727040797472,
      "rewards/format_reward": 0.6458333432674408,
      "step": 313
    },
    {
      "completion_length": 1317.7083625793457,
      "epoch": 0.3588571428571429,
      "grad_norm": 2.5971078872680664,
      "kl": 0.4033660888671875,
      "learning_rate": 4.2898608072313045e-07,
      "loss": 0.0162,
      "reward": 0.18170432932674885,
      "reward_std": 0.13025694666430354,
      "rewards/cosine_scaled_reward": 0.10399175062775612,
      "rewards/format_reward": 0.854166679084301,
      "step": 314
    },
    {
      "completion_length": 2108.854248046875,
      "epoch": 0.36,
      "grad_norm": 2.6029951572418213,
      "kl": 0.965606689453125,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.0386,
      "reward": 0.09199583710869774,
      "reward_std": 0.1252267644740641,
      "rewards/cosine_scaled_reward": -0.07803997304290533,
      "rewards/format_reward": 0.6875000186264515,
      "step": 315
    },
    {
      "completion_length": 1711.2917366027832,
      "epoch": 0.36114285714285715,
      "grad_norm": 2.844820499420166,
      "kl": 0.6187896728515625,
      "learning_rate": 4.2294634442070553e-07,
      "loss": 0.0247,
      "reward": 0.04600826557725668,
      "reward_std": 0.08856615889817476,
      "rewards/cosine_scaled_reward": -0.20388950034976006,
      "rewards/format_reward": 0.6666666846722364,
      "step": 316
    },
    {
      "completion_length": 1576.0625076293945,
      "epoch": 0.36228571428571427,
      "grad_norm": 2.0948660373687744,
      "kl": 0.4916229248046875,
      "learning_rate": 4.1993569137498776e-07,
      "loss": 0.0197,
      "reward": 0.11145175900310278,
      "reward_std": 0.11253520660102367,
      "rewards/cosine_scaled_reward": -0.05809229984879494,
      "rewards/format_reward": 0.7500000186264515,
      "step": 317
    },
    {
      "completion_length": 1069.9166831970215,
      "epoch": 0.36342857142857143,
      "grad_norm": 2.052123785018921,
      "kl": 0.205841064453125,
      "learning_rate": 4.1693137748017915e-07,
      "loss": 0.0082,
      "reward": 0.1357195656746626,
      "reward_std": 0.11260256776586175,
      "rewards/cosine_scaled_reward": -0.10857605282217264,
      "rewards/format_reward": 1.0,
      "step": 318
    },
    {
      "completion_length": 1185.145866394043,
      "epoch": 0.36457142857142855,
      "grad_norm": 3.6290059089660645,
      "kl": 0.23030853271484375,
      "learning_rate": 4.1393354916230005e-07,
      "loss": 0.0092,
      "reward": 0.039664710406214,
      "reward_std": 0.09799355687573552,
      "rewards/cosine_scaled_reward": -0.29441852401942015,
      "rewards/format_reward": 0.8125000074505806,
      "step": 319
    },
    {
      "completion_length": 822.8125267028809,
      "epoch": 0.3657142857142857,
      "grad_norm": 10.01123046875,
      "kl": 0.42021942138671875,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0169,
      "reward": 0.14526783768087626,
      "reward_std": 0.1066361116245389,
      "rewards/cosine_scaled_reward": -0.05672251805663109,
      "rewards/format_reward": 0.9583333432674408,
      "step": 320
    },
    {
      "completion_length": 951.9166946411133,
      "epoch": 0.3668571428571429,
      "grad_norm": 1.7355072498321533,
      "kl": 0.166778564453125,
      "learning_rate": 4.079579333738039e-07,
      "loss": 0.0067,
      "reward": 0.16883038450032473,
      "reward_std": 0.10923763830214739,
      "rewards/cosine_scaled_reward": 0.03077949397265911,
      "rewards/format_reward": 0.9166666716337204,
      "step": 321
    },
    {
      "completion_length": 1182.9167137145996,
      "epoch": 0.368,
      "grad_norm": 3.371811628341675,
      "kl": 0.32440185546875,
      "learning_rate": 4.0498043714627006e-07,
      "loss": 0.013,
      "reward": 0.10157357528805733,
      "reward_std": 0.09876813879236579,
      "rewards/cosine_scaled_reward": -0.07484898250550032,
      "rewards/format_reward": 0.7291666679084301,
      "step": 322
    },
    {
      "completion_length": 1447.1458587646484,
      "epoch": 0.36914285714285716,
      "grad_norm": 2.1031486988067627,
      "kl": 0.4143524169921875,
      "learning_rate": 4.020100089676376e-07,
      "loss": 0.0166,
      "reward": 0.0939664258621633,
      "reward_std": 0.11301013454794884,
      "rewards/cosine_scaled_reward": -0.08118974138051271,
      "rewards/format_reward": 0.7083333376795053,
      "step": 323
    },
    {
      "completion_length": 1413.3542251586914,
      "epoch": 0.3702857142857143,
      "grad_norm": 1.4308388233184814,
      "kl": 0.449249267578125,
      "learning_rate": 3.9904679361238526e-07,
      "loss": 0.018,
      "reward": 0.054550049535464495,
      "reward_std": 0.1179595545399934,
      "rewards/cosine_scaled_reward": -0.20703712804242969,
      "rewards/format_reward": 0.7291666697710752,
      "step": 324
    },
    {
      "completion_length": 1824.2708702087402,
      "epoch": 0.37142857142857144,
      "grad_norm": 2.757559299468994,
      "kl": 1.072296142578125,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0429,
      "reward": 0.08934608940035105,
      "reward_std": 0.12084951438009739,
      "rewards/cosine_scaled_reward": -0.030909400433301926,
      "rewards/format_reward": 0.5833333432674408,
      "step": 325
    },
    {
      "completion_length": 1115.270866394043,
      "epoch": 0.37257142857142855,
      "grad_norm": 2.2729392051696777,
      "kl": 0.2136383056640625,
      "learning_rate": 3.931425787051832e-07,
      "loss": 0.0085,
      "reward": 0.16146898362785578,
      "reward_std": 0.14930668845772743,
      "rewards/cosine_scaled_reward": 0.023914188146591187,
      "rewards/format_reward": 0.8958333432674408,
      "step": 326
    },
    {
      "completion_length": 1361.3542022705078,
      "epoch": 0.3737142857142857,
      "grad_norm": 0.9111059904098511,
      "kl": 0.24196624755859375,
      "learning_rate": 3.902018669163384e-07,
      "loss": 0.0097,
      "reward": 0.18883195845410228,
      "reward_std": 0.12315699364989996,
      "rewards/cosine_scaled_reward": 0.10393881611526012,
      "rewards/format_reward": 0.895833333954215,
      "step": 327
    },
    {
      "completion_length": 1545.9375381469727,
      "epoch": 0.37485714285714283,
      "grad_norm": 2.3425772190093994,
      "kl": 0.56195068359375,
      "learning_rate": 3.872689434630585e-07,
      "loss": 0.0225,
      "reward": 0.0559651258517988,
      "reward_std": 0.11655561625957489,
      "rewards/cosine_scaled_reward": -0.21696932520717382,
      "rewards/format_reward": 0.7500000149011612,
      "step": 328
    },
    {
      "completion_length": 979.2708511352539,
      "epoch": 0.376,
      "grad_norm": 3.092437744140625,
      "kl": 0.1307373046875,
      "learning_rate": 3.843439512918949e-07,
      "loss": 0.0052,
      "reward": 0.16340086178388447,
      "reward_std": 0.10502722533419728,
      "rewards/cosine_scaled_reward": 0.060619720607064664,
      "rewards/format_reward": 0.8125000111758709,
      "step": 329
    },
    {
      "completion_length": 1155.9375305175781,
      "epoch": 0.37714285714285717,
      "grad_norm": 2.5028579235076904,
      "kl": 0.5152435302734375,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0206,
      "reward": 0.0951260298024863,
      "reward_std": 0.10306549491360784,
      "rewards/cosine_scaled_reward": -0.17680510133504868,
      "rewards/format_reward": 0.8958333432674408,
      "step": 330
    },
    {
      "completion_length": 1770.0208892822266,
      "epoch": 0.3782857142857143,
      "grad_norm": 1.7597198486328125,
      "kl": 0.6001739501953125,
      "learning_rate": 3.785183306423767e-07,
      "loss": 0.024,
      "reward": 0.10013224184513092,
      "reward_std": 0.11445806687697768,
      "rewards/cosine_scaled_reward": -0.0341799296438694,
      "rewards/format_reward": 0.6458333395421505,
      "step": 331
    },
    {
      "completion_length": 1468.270881652832,
      "epoch": 0.37942857142857145,
      "grad_norm": 2.4370524883270264,
      "kl": 0.4661865234375,
      "learning_rate": 3.7561798609655373e-07,
      "loss": 0.0187,
      "reward": 0.0839388279709965,
      "reward_std": 0.08322880789637566,
      "rewards/cosine_scaled_reward": -0.1475684866309166,
      "rewards/format_reward": 0.7708333544433117,
      "step": 332
    },
    {
      "completion_length": 1169.4166946411133,
      "epoch": 0.38057142857142856,
      "grad_norm": 1.17229163646698,
      "kl": 0.29107666015625,
      "learning_rate": 3.72726140684072e-07,
      "loss": 0.0117,
      "reward": 0.12938493536785245,
      "reward_std": 0.13117663795128465,
      "rewards/cosine_scaled_reward": -0.09842105722054839,
      "rewards/format_reward": 0.9375000149011612,
      "step": 333
    },
    {
      "completion_length": 2001.9375305175781,
      "epoch": 0.38171428571428573,
      "grad_norm": 2.2998111248016357,
      "kl": 1.0654296875,
      "learning_rate": 3.6984293534939737e-07,
      "loss": 0.0426,
      "reward": 0.03344674052641494,
      "reward_std": 0.12521635321900249,
      "rewards/cosine_scaled_reward": -0.2376094851642847,
      "rewards/format_reward": 0.666666679084301,
      "step": 334
    },
    {
      "completion_length": 1584.0625534057617,
      "epoch": 0.38285714285714284,
      "grad_norm": 2.906867504119873,
      "kl": 0.5097579956054688,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.0204,
      "reward": 0.11645684402901679,
      "reward_std": 0.12058440665714443,
      "rewards/cosine_scaled_reward": -0.04469235986471176,
      "rewards/format_reward": 0.7708333432674408,
      "step": 335
    },
    {
      "completion_length": 1674.354206085205,
      "epoch": 0.384,
      "grad_norm": 1.6613434553146362,
      "kl": 0.6746101379394531,
      "learning_rate": 3.641030065789562e-07,
      "loss": 0.027,
      "reward": 0.10816401499323547,
      "reward_std": 0.13928860798478127,
      "rewards/cosine_scaled_reward": 0.030387197621166706,
      "rewards/format_reward": 0.5625000074505806,
      "step": 336
    },
    {
      "completion_length": 1494.8334045410156,
      "epoch": 0.3851428571428571,
      "grad_norm": 2.075910806655884,
      "kl": 0.5845260620117188,
      "learning_rate": 3.612465628992203e-07,
      "loss": 0.0234,
      "reward": 0.09280223221867345,
      "reward_std": 0.1204152749851346,
      "rewards/cosine_scaled_reward": -0.17956852912902832,
      "rewards/format_reward": 0.8958333432674408,
      "step": 337
    },
    {
      "completion_length": 1031.0625076293945,
      "epoch": 0.3862857142857143,
      "grad_norm": 1.3386896848678589,
      "kl": 0.3839874267578125,
      "learning_rate": 3.5839931879571725e-07,
      "loss": 0.0154,
      "reward": 0.12651887256652117,
      "reward_std": 0.11517677642405033,
      "rewards/cosine_scaled_reward": -0.06461506709456444,
      "rewards/format_reward": 0.8541666772216558,
      "step": 338
    },
    {
      "completion_length": 1663.1250305175781,
      "epoch": 0.38742857142857146,
      "grad_norm": 2.8500008583068848,
      "kl": 0.616424560546875,
      "learning_rate": 3.555614130391079e-07,
      "loss": 0.0247,
      "reward": 0.09466042937128805,
      "reward_std": 0.09753736667335033,
      "rewards/cosine_scaled_reward": -0.1472719321027398,
      "rewards/format_reward": 0.8333333544433117,
      "step": 339
    },
    {
      "completion_length": 1387.4375381469727,
      "epoch": 0.38857142857142857,
      "grad_norm": 2.0746471881866455,
      "kl": 0.4688873291015625,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.0188,
      "reward": 0.1363295007031411,
      "reward_std": 0.12295715417712927,
      "rewards/cosine_scaled_reward": -0.040779574774205685,
      "rewards/format_reward": 0.875,
      "step": 340
    },
    {
      "completion_length": 1612.5208740234375,
      "epoch": 0.38971428571428574,
      "grad_norm": 2.2794735431671143,
      "kl": 0.814697265625,
      "learning_rate": 3.4991416936678276e-07,
      "loss": 0.0326,
      "reward": 0.1513365504797548,
      "reward_std": 0.15308804996311665,
      "rewards/cosine_scaled_reward": 0.11684986762702465,
      "rewards/format_reward": 0.6250000149011612,
      "step": 341
    },
    {
      "completion_length": 1542.833381652832,
      "epoch": 0.39085714285714285,
      "grad_norm": 2.162214756011963,
      "kl": 0.7668609619140625,
      "learning_rate": 3.471051066897562e-07,
      "loss": 0.0307,
      "reward": 0.07486888614948839,
      "reward_std": 0.08258337597362697,
      "rewards/cosine_scaled_reward": -0.19311499642208219,
      "rewards/format_reward": 0.8125000111758709,
      "step": 342
    },
    {
      "completion_length": 1584.8750457763672,
      "epoch": 0.392,
      "grad_norm": 2.241345167160034,
      "kl": 0.691986083984375,
      "learning_rate": 3.4430593282358777e-07,
      "loss": 0.0277,
      "reward": 0.13888886122731492,
      "reward_std": 0.1267297170124948,
      "rewards/cosine_scaled_reward": 0.026948151644319296,
      "rewards/format_reward": 0.7500000186264515,
      "step": 343
    },
    {
      "completion_length": 1414.5833587646484,
      "epoch": 0.3931428571428571,
      "grad_norm": 4.952951908111572,
      "kl": 0.78118896484375,
      "learning_rate": 3.4151678419606233e-07,
      "loss": 0.0312,
      "reward": 0.1714508015429601,
      "reward_std": 0.1391323572024703,
      "rewards/cosine_scaled_reward": 0.07935434021055698,
      "rewards/format_reward": 0.833333358168602,
      "step": 344
    },
    {
      "completion_length": 1180.333366394043,
      "epoch": 0.3942857142857143,
      "grad_norm": 1.540398120880127,
      "kl": 0.5207366943359375,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0209,
      "reward": 0.13360386714339256,
      "reward_std": 0.14522417169064283,
      "rewards/cosine_scaled_reward": -0.034749194979667664,
      "rewards/format_reward": 0.8541666865348816,
      "step": 345
    },
    {
      "completion_length": 1519.6875305175781,
      "epoch": 0.3954285714285714,
      "grad_norm": 2.235276699066162,
      "kl": 0.7563095092773438,
      "learning_rate": 3.359691059183761e-07,
      "loss": 0.0302,
      "reward": 0.06512028211727738,
      "reward_std": 0.11110542109236121,
      "rewards/cosine_scaled_reward": -0.17456839326769114,
      "rewards/format_reward": 0.7291666865348816,
      "step": 346
    },
    {
      "completion_length": 1307.8750305175781,
      "epoch": 0.3965714285714286,
      "grad_norm": 4.259415626525879,
      "kl": 0.570068359375,
      "learning_rate": 3.3321084665422803e-07,
      "loss": 0.0228,
      "reward": 0.08080136496573687,
      "reward_std": 0.10251538408920169,
      "rewards/cosine_scaled_reward": -0.22406550850064377,
      "rewards/format_reward": 0.9166666865348816,
      "step": 347
    },
    {
      "completion_length": 1316.1250381469727,
      "epoch": 0.3977142857142857,
      "grad_norm": 1.8449828624725342,
      "kl": 0.8987274169921875,
      "learning_rate": 3.3046315338757026e-07,
      "loss": 0.0359,
      "reward": 0.11601153435185552,
      "reward_std": 0.12534239329397678,
      "rewards/cosine_scaled_reward": -0.09255390800535679,
      "rewards/format_reward": 0.8541666939854622,
      "step": 348
    },
    {
      "completion_length": 1266.5000305175781,
      "epoch": 0.39885714285714285,
      "grad_norm": 2.301177740097046,
      "kl": 0.484771728515625,
      "learning_rate": 3.2772616003709616e-07,
      "loss": 0.0194,
      "reward": 0.11150891752913594,
      "reward_std": 0.10962018929421902,
      "rewards/cosine_scaled_reward": -0.07765534892678261,
      "rewards/format_reward": 0.8125000037252903,
      "step": 349
    },
    {
      "completion_length": 1148.3125343322754,
      "epoch": 0.4,
      "grad_norm": 2.0247983932495117,
      "kl": 0.4848480224609375,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.0194,
      "reward": 0.15430114115588367,
      "reward_std": 0.14347478654235601,
      "rewards/cosine_scaled_reward": 0.002343377098441124,
      "rewards/format_reward": 0.8958333507180214,
      "step": 350
    },
    {
      "completion_length": 1485.7500457763672,
      "epoch": 0.40114285714285713,
      "grad_norm": 2.1637814044952393,
      "kl": 0.888519287109375,
      "learning_rate": 3.222848061454764e-07,
      "loss": 0.0356,
      "reward": 0.12535253415990155,
      "reward_std": 0.15357061475515366,
      "rewards/cosine_scaled_reward": -0.002679265569895506,
      "rewards/format_reward": 0.7291666865348816,
      "step": 351
    },
    {
      "completion_length": 1839.8125534057617,
      "epoch": 0.4022857142857143,
      "grad_norm": 3.5423998832702637,
      "kl": 1.449371337890625,
      "learning_rate": 3.195807108082429e-07,
      "loss": 0.058,
      "reward": 0.09909449616679922,
      "reward_std": 0.15971363987773657,
      "rewards/cosine_scaled_reward": -0.0335394795256434,
      "rewards/format_reward": 0.6458333507180214,
      "step": 352
    },
    {
      "completion_length": 1224.6875305175781,
      "epoch": 0.4034285714285714,
      "grad_norm": 3.0395312309265137,
      "kl": 0.6749420166015625,
      "learning_rate": 3.168878457820915e-07,
      "loss": 0.027,
      "reward": 0.2176575637422502,
      "reward_std": 0.1126766725210473,
      "rewards/cosine_scaled_reward": 0.20828061178326607,
      "rewards/format_reward": 0.8541666753590107,
      "step": 353
    },
    {
      "completion_length": 1273.708381652832,
      "epoch": 0.4045714285714286,
      "grad_norm": 2.6028831005096436,
      "kl": 0.7369384765625,
      "learning_rate": 3.142063423134644e-07,
      "loss": 0.0296,
      "reward": 0.12516162917017937,
      "reward_std": 0.09799297200515866,
      "rewards/cosine_scaled_reward": -0.0536028565838933,
      "rewards/format_reward": 0.8333333432674408,
      "step": 354
    },
    {
      "completion_length": 1033.4166946411133,
      "epoch": 0.4057142857142857,
      "grad_norm": 8.765235900878906,
      "kl": 0.76263427734375,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0305,
      "reward": 0.2028479753062129,
      "reward_std": 0.13977694138884544,
      "rewards/cosine_scaled_reward": 0.11576181277632713,
      "rewards/format_reward": 0.9583333432674408,
      "step": 355
    },
    {
      "completion_length": 1755.3959197998047,
      "epoch": 0.40685714285714286,
      "grad_norm": 8.445033073425293,
      "kl": 1.7131805419921875,
      "learning_rate": 3.0887794225945143e-07,
      "loss": 0.0685,
      "reward": 0.06232760299462825,
      "reward_std": 0.15073465276509523,
      "rewards/cosine_scaled_reward": -0.1625305749475956,
      "rewards/format_reward": 0.6875000223517418,
      "step": 356
    },
    {
      "completion_length": 1907.8750686645508,
      "epoch": 0.408,
      "grad_norm": 2.7435877323150635,
      "kl": 1.49609375,
      "learning_rate": 3.062313053727671e-07,
      "loss": 0.0599,
      "reward": 0.045256637153215706,
      "reward_std": 0.10663844272494316,
      "rewards/cosine_scaled_reward": -0.23459493229165673,
      "rewards/format_reward": 0.7291666846722364,
      "step": 357
    },
    {
      "completion_length": 1480.333366394043,
      "epoch": 0.40914285714285714,
      "grad_norm": 2.5645945072174072,
      "kl": 0.68377685546875,
      "learning_rate": 3.0359654942835247e-07,
      "loss": 0.0273,
      "reward": 0.1269954121671617,
      "reward_std": 0.12238077353686094,
      "rewards/cosine_scaled_reward": -0.08821228123269975,
      "rewards/format_reward": 0.9166666865348816,
      "step": 358
    },
    {
      "completion_length": 1268.8750381469727,
      "epoch": 0.4102857142857143,
      "grad_norm": 9.816429138183594,
      "kl": 1.206329345703125,
      "learning_rate": 3.0097380284049523e-07,
      "loss": 0.0483,
      "reward": 0.0941942217759788,
      "reward_std": 0.09892157511785626,
      "rewards/cosine_scaled_reward": -0.17397763207554817,
      "rewards/format_reward": 0.8958333507180214,
      "step": 359
    },
    {
      "completion_length": 1494.9583587646484,
      "epoch": 0.4114285714285714,
      "grad_norm": 2.043757915496826,
      "kl": 0.8025665283203125,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0321,
      "reward": 0.14919199608266354,
      "reward_std": 0.16920089721679688,
      "rewards/cosine_scaled_reward": 0.01944921351969242,
      "rewards/format_reward": 0.833333358168602,
      "step": 360
    },
    {
      "completion_length": 1387.5000457763672,
      "epoch": 0.4125714285714286,
      "grad_norm": 3.627060890197754,
      "kl": 0.604827880859375,
      "learning_rate": 2.9576484845877793e-07,
      "loss": 0.0243,
      "reward": 0.0996842200984247,
      "reward_std": 0.12515971716493368,
      "rewards/cosine_scaled_reward": -0.14535773918032646,
      "rewards/format_reward": 0.8750000149011612,
      "step": 361
    },
    {
      "completion_length": 1282.7291946411133,
      "epoch": 0.4137142857142857,
      "grad_norm": 2.7785682678222656,
      "kl": 0.9784774780273438,
      "learning_rate": 2.931788945420058e-07,
      "loss": 0.0392,
      "reward": 0.15694484941195697,
      "reward_std": 0.08703827066347003,
      "rewards/cosine_scaled_reward": 0.005612561479210854,
      "rewards/format_reward": 0.8958333432674408,
      "step": 362
    },
    {
      "completion_length": 981.1875267028809,
      "epoch": 0.41485714285714287,
      "grad_norm": 2.8391177654266357,
      "kl": 0.4031524658203125,
      "learning_rate": 2.9060545772359305e-07,
      "loss": 0.0162,
      "reward": 0.20889083773363382,
      "reward_std": 0.09412059234455228,
      "rewards/cosine_scaled_reward": 0.14475639257580042,
      "rewards/format_reward": 0.8958333432674408,
      "step": 363
    },
    {
      "completion_length": 1328.2500457763672,
      "epoch": 0.416,
      "grad_norm": 1.4150444269180298,
      "kl": 0.5859603881835938,
      "learning_rate": 2.8804466342921987e-07,
      "loss": 0.0235,
      "reward": 0.05011219787411392,
      "reward_std": 0.09909166162833571,
      "rewards/cosine_scaled_reward": -0.26201771944761276,
      "rewards/format_reward": 0.8125000260770321,
      "step": 364
    },
    {
      "completion_length": 1732.4375457763672,
      "epoch": 0.41714285714285715,
      "grad_norm": 3.007594585418701,
      "kl": 1.2857818603515625,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.0514,
      "reward": 0.13636061176657677,
      "reward_std": 0.1195504111237824,
      "rewards/cosine_scaled_reward": 0.015712120453827083,
      "rewards/format_reward": 0.750000013038516,
      "step": 365
    },
    {
      "completion_length": 1108.0000495910645,
      "epoch": 0.41828571428571426,
      "grad_norm": 4.287596702575684,
      "kl": 0.290069580078125,
      "learning_rate": 2.829615010283344e-07,
      "loss": 0.0116,
      "reward": 0.15384384151548147,
      "reward_std": 0.1040445901453495,
      "rewards/cosine_scaled_reward": 0.009631453547626734,
      "rewards/format_reward": 0.8750000149011612,
      "step": 366
    },
    {
      "completion_length": 1712.8542022705078,
      "epoch": 0.41942857142857143,
      "grad_norm": 2.6099181175231934,
      "kl": 0.9123382568359375,
      "learning_rate": 2.8043938066798645e-07,
      "loss": 0.0365,
      "reward": 0.0756345079280436,
      "reward_std": 0.11609864910133183,
      "rewards/cosine_scaled_reward": -0.12630227487534285,
      "rewards/format_reward": 0.6875000111758709,
      "step": 367
    },
    {
      "completion_length": 1932.4375381469727,
      "epoch": 0.4205714285714286,
      "grad_norm": 4.4302215576171875,
      "kl": 1.157470703125,
      "learning_rate": 2.7793039831193133e-07,
      "loss": 0.0463,
      "reward": 0.05102244240697473,
      "reward_std": 0.14196251472458243,
      "rewards/cosine_scaled_reward": -0.1656601596623659,
      "rewards/format_reward": 0.6250000074505806,
      "step": 368
    },
    {
      "completion_length": 1801.0000686645508,
      "epoch": 0.4217142857142857,
      "grad_norm": 2.231231451034546,
      "kl": 1.43145751953125,
      "learning_rate": 2.7543467624442956e-07,
      "loss": 0.0572,
      "reward": 0.13401565965614282,
      "reward_std": 0.1620243601500988,
      "rewards/cosine_scaled_reward": 0.007201282307505608,
      "rewards/format_reward": 0.7500000204890966,
      "step": 369
    },
    {
      "completion_length": 1558.6250267028809,
      "epoch": 0.4228571428571429,
      "grad_norm": 2.5228145122528076,
      "kl": 1.020751953125,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0408,
      "reward": 0.09310736267070752,
      "reward_std": 0.09267222543712705,
      "rewards/cosine_scaled_reward": -0.09523181803524494,
      "rewards/format_reward": 0.7291666865348816,
      "step": 370
    },
    {
      "completion_length": 933.7292060852051,
      "epoch": 0.424,
      "grad_norm": 1.8327668905258179,
      "kl": 0.6253280639648438,
      "learning_rate": 2.7048349887476037e-07,
      "loss": 0.025,
      "reward": 0.1391057469882071,
      "reward_std": 0.09823393146507442,
      "rewards/cosine_scaled_reward": 0.023010117933154106,
      "rewards/format_reward": 0.7708333469927311,
      "step": 371
    },
    {
      "completion_length": 1648.6458740234375,
      "epoch": 0.42514285714285716,
      "grad_norm": 3.7653868198394775,
      "kl": 0.7486572265625,
      "learning_rate": 2.6802828488599294e-07,
      "loss": 0.0299,
      "reward": 0.11916764298803173,
      "reward_std": 0.16286169085651636,
      "rewards/cosine_scaled_reward": -0.006715672556310892,
      "rewards/format_reward": 0.7083333544433117,
      "step": 372
    },
    {
      "completion_length": 899.7083587646484,
      "epoch": 0.42628571428571427,
      "grad_norm": 2.0490570068359375,
      "kl": 0.393463134765625,
      "learning_rate": 2.655868138008171e-07,
      "loss": 0.0157,
      "reward": 0.11241465236525983,
      "reward_std": 0.1083641320001334,
      "rewards/cosine_scaled_reward": -0.0999544644728303,
      "rewards/format_reward": 0.8541666828095913,
      "step": 373
    },
    {
      "completion_length": 1088.1042022705078,
      "epoch": 0.42742857142857144,
      "grad_norm": 2.575913667678833,
      "kl": 0.3724212646484375,
      "learning_rate": 2.631592046130896e-07,
      "loss": 0.0149,
      "reward": 0.1537247821688652,
      "reward_std": 0.144410849083215,
      "rewards/cosine_scaled_reward": 0.0019068364053964615,
      "rewards/format_reward": 0.8958333432674408,
      "step": 374
    },
    {
      "completion_length": 1255.9375457763672,
      "epoch": 0.42857142857142855,
      "grad_norm": 4.075099468231201,
      "kl": 1.0,
      "learning_rate": 2.6074557564105724e-07,
      "loss": 0.04,
      "reward": 0.11958665121346712,
      "reward_std": 0.17436719313263893,
      "rewards/cosine_scaled_reward": -0.02865603007376194,
      "rewards/format_reward": 0.7500000149011612,
      "step": 375
    },
    {
      "completion_length": 1406.4583740234375,
      "epoch": 0.4297142857142857,
      "grad_norm": 4.192153453826904,
      "kl": 1.0216064453125,
      "learning_rate": 2.583460445215911e-07,
      "loss": 0.0409,
      "reward": 0.11533228470943868,
      "reward_std": 0.1220639725215733,
      "rewards/cosine_scaled_reward": -0.10069759003818035,
      "rewards/format_reward": 0.8750000149011612,
      "step": 376
    },
    {
      "completion_length": 1517.8542175292969,
      "epoch": 0.4308571428571429,
      "grad_norm": 3.5561301708221436,
      "kl": 0.7474365234375,
      "learning_rate": 2.5596072820445254e-07,
      "loss": 0.0299,
      "reward": 0.06857469491660595,
      "reward_std": 0.12185074761509895,
      "rewards/cosine_scaled_reward": -0.20741149224340916,
      "rewards/format_reward": 0.8125000298023224,
      "step": 377
    },
    {
      "completion_length": 1371.6458797454834,
      "epoch": 0.432,
      "grad_norm": 2.8652353286743164,
      "kl": 0.5273590087890625,
      "learning_rate": 2.5358974294659373e-07,
      "loss": 0.0211,
      "reward": 0.15830194216687232,
      "reward_std": 0.12496394384652376,
      "rewards/cosine_scaled_reward": 0.02524256706237793,
      "rewards/format_reward": 0.8333333432674408,
      "step": 378
    },
    {
      "completion_length": 1719.6666946411133,
      "epoch": 0.43314285714285716,
      "grad_norm": 1.9505208730697632,
      "kl": 0.86883544921875,
      "learning_rate": 2.512332043064913e-07,
      "loss": 0.0347,
      "reward": 0.07183210924267769,
      "reward_std": 0.08735731011256576,
      "rewards/cosine_scaled_reward": -0.19765841774642467,
      "rewards/format_reward": 0.812500013038516,
      "step": 379
    },
    {
      "completion_length": 1268.458366394043,
      "epoch": 0.4342857142857143,
      "grad_norm": 2.074549436569214,
      "kl": 0.7683868408203125,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.0308,
      "reward": 0.09168538171797991,
      "reward_std": 0.1286314483731985,
      "rewards/cosine_scaled_reward": -0.10583510436117649,
      "rewards/format_reward": 0.7500000055879354,
      "step": 380
    },
    {
      "completion_length": 1851.1042251586914,
      "epoch": 0.43542857142857144,
      "grad_norm": 3.475543975830078,
      "kl": 1.308349609375,
      "learning_rate": 2.465639255873246e-07,
      "loss": 0.0523,
      "reward": 0.017921562888659537,
      "reward_std": 0.10069577465765178,
      "rewards/cosine_scaled_reward": -0.26895514875650406,
      "rewards/format_reward": 0.6458333525806665,
      "step": 381
    },
    {
      "completion_length": 1056.208351135254,
      "epoch": 0.43657142857142855,
      "grad_norm": 3.336723566055298,
      "kl": 0.65924072265625,
      "learning_rate": 2.4425141308231765e-07,
      "loss": 0.0264,
      "reward": 0.0698289682622999,
      "reward_std": 0.10450809169560671,
      "rewards/cosine_scaled_reward": -0.20110327936708927,
      "rewards/format_reward": 0.8125000149011612,
      "step": 382
    },
    {
      "completion_length": 1278.520881652832,
      "epoch": 0.4377142857142857,
      "grad_norm": 1.7827008962631226,
      "kl": 1.0176849365234375,
      "learning_rate": 2.4195380233209006e-07,
      "loss": 0.0407,
      "reward": 0.15380790340714157,
      "reward_std": 0.1372139612212777,
      "rewards/cosine_scaled_reward": 0.03767992998473346,
      "rewards/format_reward": 0.8125000074505806,
      "step": 383
    },
    {
      "completion_length": 1171.3541946411133,
      "epoch": 0.43885714285714283,
      "grad_norm": 2.2583577632904053,
      "kl": 0.81756591796875,
      "learning_rate": 2.3967120531894857e-07,
      "loss": 0.0327,
      "reward": 0.21438380563631654,
      "reward_std": 0.18565166369080544,
      "rewards/cosine_scaled_reward": 0.19361148471944034,
      "rewards/format_reward": 0.8541666865348816,
      "step": 384
    },
    {
      "completion_length": 1468.4584121704102,
      "epoch": 0.44,
      "grad_norm": 3.9643666744232178,
      "kl": 0.9686126708984375,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0388,
      "reward": 0.09368883771821856,
      "reward_std": 0.12416216172277927,
      "rewards/cosine_scaled_reward": -0.13376147765666246,
      "rewards/format_reward": 0.8125000298023224,
      "step": 385
    },
    {
      "completion_length": 1320.7292022705078,
      "epoch": 0.44114285714285717,
      "grad_norm": 2.8084425926208496,
      "kl": 0.906951904296875,
      "learning_rate": 2.3515149676898552e-07,
      "loss": 0.0363,
      "reward": 0.18752056313678622,
      "reward_std": 0.09679444809444249,
      "rewards/cosine_scaled_reward": 0.0695192702114582,
      "rewards/format_reward": 0.9375000149011612,
      "step": 386
    },
    {
      "completion_length": 1446.6667175292969,
      "epoch": 0.4422857142857143,
      "grad_norm": 2.2158679962158203,
      "kl": 0.94366455078125,
      "learning_rate": 2.3291460551638237e-07,
      "loss": 0.0378,
      "reward": 0.13372905366122723,
      "reward_std": 0.09293439192697406,
      "rewards/cosine_scaled_reward": -0.017829248681664467,
      "rewards/format_reward": 0.8125000204890966,
      "step": 387
    },
    {
      "completion_length": 1416.458366394043,
      "epoch": 0.44342857142857145,
      "grad_norm": 3.4314982891082764,
      "kl": 1.20458984375,
      "learning_rate": 2.306931685585657e-07,
      "loss": 0.0482,
      "reward": 0.10979634639807045,
      "reward_std": 0.14348772866651416,
      "rewards/cosine_scaled_reward": -0.09102598764002323,
      "rewards/format_reward": 0.8125000074505806,
      "step": 388
    },
    {
      "completion_length": 1301.3750381469727,
      "epoch": 0.44457142857142856,
      "grad_norm": 2.1517300605773926,
      "kl": 0.8640289306640625,
      "learning_rate": 2.2848729416523859e-07,
      "loss": 0.0345,
      "reward": 0.12174346391111612,
      "reward_std": 0.1113532236777246,
      "rewards/cosine_scaled_reward": -0.11768272100016475,
      "rewards/format_reward": 0.9375000149011612,
      "step": 389
    },
    {
      "completion_length": 1864.270881652832,
      "epoch": 0.44571428571428573,
      "grad_norm": 2.703826665878296,
      "kl": 1.3235321044921875,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.053,
      "reward": 0.08123930124565959,
      "reward_std": 0.12660485692322254,
      "rewards/cosine_scaled_reward": -0.06472943164408207,
      "rewards/format_reward": 0.6041666753590107,
      "step": 390
    },
    {
      "completion_length": 1732.4167137145996,
      "epoch": 0.44685714285714284,
      "grad_norm": 3.1918063163757324,
      "kl": 1.76708984375,
      "learning_rate": 2.2412266235313973e-07,
      "loss": 0.0708,
      "reward": 0.14286700636148453,
      "reward_std": 0.18649776838719845,
      "rewards/cosine_scaled_reward": 0.028382533695548773,
      "rewards/format_reward": 0.7500000111758709,
      "step": 391
    },
    {
      "completion_length": 1371.7916946411133,
      "epoch": 0.448,
      "grad_norm": 3.6219229698181152,
      "kl": 0.9765167236328125,
      "learning_rate": 2.2196411766036487e-07,
      "loss": 0.039,
      "reward": 0.06799169280566275,
      "reward_std": 0.09887422667816281,
      "rewards/cosine_scaled_reward": -0.19014379568398,
      "rewards/format_reward": 0.7708333488553762,
      "step": 392
    },
    {
      "completion_length": 1691.1666870117188,
      "epoch": 0.4491428571428571,
      "grad_norm": 2.618884801864624,
      "kl": 1.6240234375,
      "learning_rate": 2.1982156097370557e-07,
      "loss": 0.0649,
      "reward": 0.06132141873240471,
      "reward_std": 0.10926410043612123,
      "rewards/cosine_scaled_reward": -0.12576089892536402,
      "rewards/format_reward": 0.6041666809469461,
      "step": 393
    },
    {
      "completion_length": 1537.270866394043,
      "epoch": 0.4502857142857143,
      "grad_norm": 6.063929557800293,
      "kl": 1.8046112060546875,
      "learning_rate": 2.1769509671835223e-07,
      "loss": 0.0722,
      "reward": 0.0685962769202888,
      "reward_std": 0.09430173807777464,
      "rewards/cosine_scaled_reward": -0.1982439812272787,
      "rewards/format_reward": 0.7916666772216558,
      "step": 394
    },
    {
      "completion_length": 1333.958396911621,
      "epoch": 0.4514285714285714,
      "grad_norm": 4.921578884124756,
      "kl": 1.4940185546875,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.0598,
      "reward": 0.1565821710973978,
      "reward_std": 0.11654932564124465,
      "rewards/cosine_scaled_reward": 0.03457294497638941,
      "rewards/format_reward": 0.8125000111758709,
      "step": 395
    },
    {
      "completion_length": 1335.645866394043,
      "epoch": 0.45257142857142857,
      "grad_norm": 2.5358009338378906,
      "kl": 0.828582763671875,
      "learning_rate": 2.134908592756607e-07,
      "loss": 0.0332,
      "reward": 0.143600944429636,
      "reward_std": 0.12354162661358714,
      "rewards/cosine_scaled_reward": -0.04586084187030792,
      "rewards/format_reward": 0.9166666865348816,
      "step": 396
    },
    {
      "completion_length": 1311.5625381469727,
      "epoch": 0.45371428571428574,
      "grad_norm": 3.427694082260132,
      "kl": 0.9869384765625,
      "learning_rate": 2.1141329099692406e-07,
      "loss": 0.0395,
      "reward": 0.08522352996078553,
      "reward_std": 0.11573155457153916,
      "rewards/cosine_scaled_reward": -0.11667108163237572,
      "rewards/format_reward": 0.7291666883975267,
      "step": 397
    },
    {
      "completion_length": 1535.7500457763672,
      "epoch": 0.45485714285714285,
      "grad_norm": 1.9625768661499023,
      "kl": 1.721282958984375,
      "learning_rate": 2.0935222495670968e-07,
      "loss": 0.0688,
      "reward": 0.10138712753541768,
      "reward_std": 0.12253602081909776,
      "rewards/cosine_scaled_reward": -0.0959820756688714,
      "rewards/format_reward": 0.7708333432674408,
      "step": 398
    },
    {
      "completion_length": 1052.1875305175781,
      "epoch": 0.456,
      "grad_norm": 2.016707181930542,
      "kl": 0.39638519287109375,
      "learning_rate": 2.0730776160846853e-07,
      "loss": 0.0159,
      "reward": 0.20977385994046926,
      "reward_std": 0.17129726987332106,
      "rewards/cosine_scaled_reward": 0.13206836581230164,
      "rewards/format_reward": 0.9375000149011612,
      "step": 399
    },
    {
      "completion_length": 998.1458587646484,
      "epoch": 0.45714285714285713,
      "grad_norm": 1.5737963914871216,
      "kl": 0.47527313232421875,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.019,
      "reward": 0.20247652614489198,
      "reward_std": 0.11905267764814198,
      "rewards/cosine_scaled_reward": 0.13497311808168888,
      "rewards/format_reward": 0.9166666716337204,
      "step": 400
    },
    {
      "completion_length": 1659.1250305175781,
      "epoch": 0.4582857142857143,
      "grad_norm": 2.9050798416137695,
      "kl": 1.2333984375,
      "learning_rate": 2.032690407508949e-07,
      "loss": 0.0493,
      "reward": 0.13014382123947144,
      "reward_std": 0.12995108915492892,
      "rewards/cosine_scaled_reward": -0.019444716162979603,
      "rewards/format_reward": 0.791666679084301,
      "step": 401
    },
    {
      "completion_length": 1586.0417175292969,
      "epoch": 0.4594285714285714,
      "grad_norm": 2.764214277267456,
      "kl": 1.1788330078125,
      "learning_rate": 2.0127498008311922e-07,
      "loss": 0.0472,
      "reward": 0.11509460117667913,
      "reward_std": 0.12621240504086018,
      "rewards/cosine_scaled_reward": -0.03602955490350723,
      "rewards/format_reward": 0.7291666939854622,
      "step": 402
    },
    {
      "completion_length": 1443.87504196167,
      "epoch": 0.4605714285714286,
      "grad_norm": 2.6378586292266846,
      "kl": 1.0952301025390625,
      "learning_rate": 1.9929791578083655e-07,
      "loss": 0.0438,
      "reward": 0.14914248324930668,
      "reward_std": 0.08150437835138291,
      "rewards/cosine_scaled_reward": 0.02718578651547432,
      "rewards/format_reward": 0.8125000204890966,
      "step": 403
    },
    {
      "completion_length": 1235.4792098999023,
      "epoch": 0.4617142857142857,
      "grad_norm": 2.0312211513519287,
      "kl": 0.6242218017578125,
      "learning_rate": 1.9733794420337213e-07,
      "loss": 0.025,
      "reward": 0.15180272003635764,
      "reward_std": 0.10177731700241566,
      "rewards/cosine_scaled_reward": -0.007191255688667297,
      "rewards/format_reward": 0.8958333507180214,
      "step": 404
    },
    {
      "completion_length": 1179.3958587646484,
      "epoch": 0.46285714285714286,
      "grad_norm": 2.201613664627075,
      "kl": 0.848846435546875,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.0339,
      "reward": 0.21597139816731215,
      "reward_std": 0.16427310602739453,
      "rewards/cosine_scaled_reward": 0.154250493273139,
      "rewards/format_reward": 0.9375000074505806,
      "step": 405
    },
    {
      "completion_length": 1454.9166946411133,
      "epoch": 0.464,
      "grad_norm": 3.2176761627197266,
      "kl": 1.196868896484375,
      "learning_rate": 1.934696604901642e-07,
      "loss": 0.0477,
      "reward": 0.12235562037676573,
      "reward_std": 0.12256110971793532,
      "rewards/cosine_scaled_reward": -0.045554774114862084,
      "rewards/format_reward": 0.791666679084301,
      "step": 406
    },
    {
      "completion_length": 1515.4792289733887,
      "epoch": 0.46514285714285714,
      "grad_norm": 3.040894031524658,
      "kl": 0.8402099609375,
      "learning_rate": 1.915615368891117e-07,
      "loss": 0.0336,
      "reward": 0.15898172045126557,
      "reward_std": 0.10987926088273525,
      "rewards/cosine_scaled_reward": 0.03327514789998531,
      "rewards/format_reward": 0.8541666939854622,
      "step": 407
    },
    {
      "completion_length": 1372.5625534057617,
      "epoch": 0.4662857142857143,
      "grad_norm": 1.9494367837905884,
      "kl": 0.5256500244140625,
      "learning_rate": 1.8967088307307e-07,
      "loss": 0.021,
      "reward": 0.18118306156247854,
      "reward_std": 0.1542639322578907,
      "rewards/cosine_scaled_reward": 0.08791719190776348,
      "rewards/format_reward": 0.8750000149011612,
      "step": 408
    },
    {
      "completion_length": 2067.58341217041,
      "epoch": 0.4674285714285714,
      "grad_norm": 2.919564962387085,
      "kl": 1.49102783203125,
      "learning_rate": 1.8779779118983867e-07,
      "loss": 0.0597,
      "reward": 0.07456145505420864,
      "reward_std": 0.12785195047035813,
      "rewards/cosine_scaled_reward": -0.12900145258754492,
      "rewards/format_reward": 0.6875000186264515,
      "step": 409
    },
    {
      "completion_length": 1623.2500228881836,
      "epoch": 0.4685714285714286,
      "grad_norm": 2.359740972518921,
      "kl": 1.49200439453125,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0597,
      "reward": 0.12223792565055192,
      "reward_std": 0.13907577726058662,
      "rewards/cosine_scaled_reward": -0.026431459933519363,
      "rewards/format_reward": 0.7500000167638063,
      "step": 410
    },
    {
      "completion_length": 1614.6250534057617,
      "epoch": 0.4697142857142857,
      "grad_norm": 2.7134463787078857,
      "kl": 0.764923095703125,
      "learning_rate": 1.8410465752883758e-07,
      "loss": 0.0306,
      "reward": 0.07999165914952755,
      "reward_std": 0.12184700695797801,
      "rewards/cosine_scaled_reward": -0.16527405753731728,
      "rewards/format_reward": 0.7916666865348816,
      "step": 411
    },
    {
      "completion_length": 1176.7291946411133,
      "epoch": 0.47085714285714286,
      "grad_norm": 3.365088701248169,
      "kl": 0.94677734375,
      "learning_rate": 1.822847957491922e-07,
      "loss": 0.0378,
      "reward": 0.10302653594408184,
      "reward_std": 0.13904161704704165,
      "rewards/cosine_scaled_reward": -0.11917255260050297,
      "rewards/format_reward": 0.8333333432674408,
      "step": 412
    },
    {
      "completion_length": 1315.0833854675293,
      "epoch": 0.472,
      "grad_norm": 3.6166837215423584,
      "kl": 0.6557846069335938,
      "learning_rate": 1.804828558898332e-07,
      "loss": 0.0263,
      "reward": 0.13768333243206143,
      "reward_std": 0.12120772805064917,
      "rewards/cosine_scaled_reward": -0.05770981824025512,
      "rewards/format_reward": 0.9166666865348816,
      "step": 413
    },
    {
      "completion_length": 2013.416748046875,
      "epoch": 0.47314285714285714,
      "grad_norm": 2.6909852027893066,
      "kl": 1.70458984375,
      "learning_rate": 1.7869892577476722e-07,
      "loss": 0.0682,
      "reward": 0.04394434345886111,
      "reward_std": 0.12014323053881526,
      "rewards/cosine_scaled_reward": -0.24098782893270254,
      "rewards/format_reward": 0.7291666772216558,
      "step": 414
    },
    {
      "completion_length": 1488.583366394043,
      "epoch": 0.4742857142857143,
      "grad_norm": 2.07572340965271,
      "kl": 0.9158935546875,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0366,
      "reward": 0.09156056097708642,
      "reward_std": 0.1220615403726697,
      "rewards/cosine_scaled_reward": -0.16554791200906038,
      "rewards/format_reward": 0.8541666939854622,
      "step": 415
    },
    {
      "completion_length": 1202.708366394043,
      "epoch": 0.4754285714285714,
      "grad_norm": 2.5329267978668213,
      "kl": 0.7248611450195312,
      "learning_rate": 1.7518544168045524e-07,
      "loss": 0.029,
      "reward": 0.20120010571554303,
      "reward_std": 0.16196386714000255,
      "rewards/cosine_scaled_reward": 0.1876464392989874,
      "rewards/format_reward": 0.7916666772216558,
      "step": 416
    },
    {
      "completion_length": 1595.958366394043,
      "epoch": 0.4765714285714286,
      "grad_norm": 2.1365303993225098,
      "kl": 0.925079345703125,
      "learning_rate": 1.7345605894346726e-07,
      "loss": 0.037,
      "reward": 0.13611510070040822,
      "reward_std": 0.10463207168504596,
      "rewards/cosine_scaled_reward": -0.014729505404829979,
      "rewards/format_reward": 0.8125000074505806,
      "step": 417
    },
    {
      "completion_length": 1091.2292137145996,
      "epoch": 0.4777142857142857,
      "grad_norm": 3.3795764446258545,
      "kl": 0.7227020263671875,
      "learning_rate": 1.7174502842694212e-07,
      "loss": 0.0289,
      "reward": 0.21031097415834665,
      "reward_std": 0.12943580746650696,
      "rewards/cosine_scaled_reward": 0.15948706772178411,
      "rewards/format_reward": 0.8958333432674408,
      "step": 418
    },
    {
      "completion_length": 1551.8750610351562,
      "epoch": 0.47885714285714287,
      "grad_norm": 2.417349100112915,
      "kl": 0.9874191284179688,
      "learning_rate": 1.7005243352409333e-07,
      "loss": 0.0395,
      "reward": 0.1355010142287938,
      "reward_std": 0.15785679733380675,
      "rewards/cosine_scaled_reward": -0.02094801003113389,
      "rewards/format_reward": 0.8125000149011612,
      "step": 419
    },
    {
      "completion_length": 945.6041870117188,
      "epoch": 0.48,
      "grad_norm": 1.6603590250015259,
      "kl": 0.618865966796875,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0248,
      "reward": 0.10752899164799601,
      "reward_std": 0.09324449067935348,
      "rewards/cosine_scaled_reward": -0.155439174734056,
      "rewards/format_reward": 0.9166666716337204,
      "step": 420
    },
    {
      "completion_length": 1413.1042251586914,
      "epoch": 0.48114285714285715,
      "grad_norm": 2.401996374130249,
      "kl": 1.11029052734375,
      "learning_rate": 1.6672287963562852e-07,
      "loss": 0.0444,
      "reward": 0.08400759304640815,
      "reward_std": 0.13007194455713034,
      "rewards/cosine_scaled_reward": -0.18536985479295254,
      "rewards/format_reward": 0.854166679084301,
      "step": 421
    },
    {
      "completion_length": 1390.4375305175781,
      "epoch": 0.48228571428571426,
      "grad_norm": 2.419326066970825,
      "kl": 1.081390380859375,
      "learning_rate": 1.6508608292777203e-07,
      "loss": 0.0433,
      "reward": 0.13678289717063308,
      "reward_std": 0.11409326584544033,
      "rewards/cosine_scaled_reward": -0.019857976818457246,
      "rewards/format_reward": 0.8333333469927311,
      "step": 422
    },
    {
      "completion_length": 1838.7917022705078,
      "epoch": 0.48342857142857143,
      "grad_norm": 1.9876563549041748,
      "kl": 1.2999267578125,
      "learning_rate": 1.6346804638120098e-07,
      "loss": 0.052,
      "reward": 0.05111281352583319,
      "reward_std": 0.11355760088190436,
      "rewards/cosine_scaled_reward": -0.21695283614099026,
      "rewards/format_reward": 0.729166679084301,
      "step": 423
    },
    {
      "completion_length": 1542.2292251586914,
      "epoch": 0.4845714285714286,
      "grad_norm": 2.6723501682281494,
      "kl": 1.047271728515625,
      "learning_rate": 1.6186884885673413e-07,
      "loss": 0.0418,
      "reward": 0.09121810318902135,
      "reward_std": 0.11254043271765113,
      "rewards/cosine_scaled_reward": -0.18606012500822544,
      "rewards/format_reward": 0.8958333507180214,
      "step": 424
    },
    {
      "completion_length": 1012.3541717529297,
      "epoch": 0.4857142857142857,
      "grad_norm": 2.142672061920166,
      "kl": 0.32970428466796875,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0132,
      "reward": 0.27949744602665305,
      "reward_std": 0.13918299926444888,
      "rewards/cosine_scaled_reward": 0.35947928391397,
      "rewards/format_reward": 0.916666679084301,
      "step": 425
    },
    {
      "completion_length": 1376.7917022705078,
      "epoch": 0.4868571428571429,
      "grad_norm": 2.2903144359588623,
      "kl": 1.4896697998046875,
      "learning_rate": 1.5872728172265146e-07,
      "loss": 0.0595,
      "reward": 0.10234351572580636,
      "reward_std": 0.1212971555069089,
      "rewards/cosine_scaled_reward": -0.1176013108342886,
      "rewards/format_reward": 0.8333333507180214,
      "step": 426
    },
    {
      "completion_length": 1822.0208892822266,
      "epoch": 0.488,
      "grad_norm": 3.0833256244659424,
      "kl": 0.966033935546875,
      "learning_rate": 1.5718506522858572e-07,
      "loss": 0.0387,
      "reward": 0.1447278270497918,
      "reward_std": 0.1653783330693841,
      "rewards/cosine_scaled_reward": 0.011570073664188385,
      "rewards/format_reward": 0.8125000149011612,
      "step": 427
    },
    {
      "completion_length": 1313.0625228881836,
      "epoch": 0.48914285714285716,
      "grad_norm": 1.966538906097412,
      "kl": 0.5592117309570312,
      "learning_rate": 1.5566199398026147e-07,
      "loss": 0.0223,
      "reward": 0.12443019635975361,
      "reward_std": 0.11524406261742115,
      "rewards/cosine_scaled_reward": -0.076734006870538,
      "rewards/format_reward": 0.8750000111758709,
      "step": 428
    },
    {
      "completion_length": 1229.2916870117188,
      "epoch": 0.49028571428571427,
      "grad_norm": 1.638811707496643,
      "kl": 1.2337493896484375,
      "learning_rate": 1.5415814221002265e-07,
      "loss": 0.0494,
      "reward": 0.147400954447221,
      "reward_std": 0.1352155078202486,
      "rewards/cosine_scaled_reward": -0.012214789167046547,
      "rewards/format_reward": 0.8750000074505806,
      "step": 429
    },
    {
      "completion_length": 979.6458587646484,
      "epoch": 0.49142857142857144,
      "grad_norm": 1.6275877952575684,
      "kl": 0.34814453125,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0139,
      "reward": 0.1619871830334887,
      "reward_std": 0.11312393890693784,
      "rewards/cosine_scaled_reward": 0.02661276888102293,
      "rewards/format_reward": 0.8958333432674408,
      "step": 430
    },
    {
      "completion_length": 1446.2292175292969,
      "epoch": 0.49257142857142855,
      "grad_norm": 3.2351579666137695,
      "kl": 1.28387451171875,
      "learning_rate": 1.5120838934595337e-07,
      "loss": 0.0514,
      "reward": 0.07184328138828278,
      "reward_std": 0.0726720115635544,
      "rewards/cosine_scaled_reward": -0.19962184969335794,
      "rewards/format_reward": 0.8125000111758709,
      "step": 431
    },
    {
      "completion_length": 1804.979232788086,
      "epoch": 0.4937142857142857,
      "grad_norm": 3.7058115005493164,
      "kl": 1.610595703125,
      "learning_rate": 1.4976263201891613e-07,
      "loss": 0.0645,
      "reward": 0.05518326349556446,
      "reward_std": 0.10842809174209833,
      "rewards/cosine_scaled_reward": -0.19627886731177568,
      "rewards/format_reward": 0.7083333525806665,
      "step": 432
    },
    {
      "completion_length": 1190.7292022705078,
      "epoch": 0.4948571428571429,
      "grad_norm": 1.7561228275299072,
      "kl": 0.5131759643554688,
      "learning_rate": 1.483363816965435e-07,
      "loss": 0.0205,
      "reward": 0.1619516264181584,
      "reward_std": 0.10530700022354722,
      "rewards/cosine_scaled_reward": 0.022388019686331972,
      "rewards/format_reward": 0.895833358168602,
      "step": 433
    },
    {
      "completion_length": 1296.0417175292969,
      "epoch": 0.496,
      "grad_norm": 1.8874526023864746,
      "kl": 0.7233428955078125,
      "learning_rate": 1.469297078922642e-07,
      "loss": 0.0289,
      "reward": 0.06844042587908916,
      "reward_std": 0.08203518786467612,
      "rewards/cosine_scaled_reward": -0.23772956430912018,
      "rewards/format_reward": 0.8750000149011612,
      "step": 434
    },
    {
      "completion_length": 1286.6042213439941,
      "epoch": 0.49714285714285716,
      "grad_norm": 2.515587568283081,
      "kl": 1.2847747802734375,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.0513,
      "reward": 0.06948329764418304,
      "reward_std": 0.09834526525810361,
      "rewards/cosine_scaled_reward": -0.22741128038614988,
      "rewards/format_reward": 0.8541666865348816,
      "step": 435
    },
    {
      "completion_length": 1111.458366394043,
      "epoch": 0.4982857142857143,
      "grad_norm": 1.83876633644104,
      "kl": 0.725067138671875,
      "learning_rate": 1.4417536311769885e-07,
      "loss": 0.029,
      "reward": 0.18536534893792123,
      "reward_std": 0.12061038403771818,
      "rewards/cosine_scaled_reward": 0.11224743165075779,
      "rewards/format_reward": 0.8541666865348816,
      "step": 436
    },
    {
      "completion_length": 1064.020866394043,
      "epoch": 0.49942857142857144,
      "grad_norm": 2.1831390857696533,
      "kl": 0.4850616455078125,
      "learning_rate": 1.4282782639029128e-07,
      "loss": 0.0194,
      "reward": 0.1590421856380999,
      "reward_std": 0.09123086743056774,
      "rewards/cosine_scaled_reward": -0.028785159811377525,
      "rewards/format_reward": 0.9583333432674408,
      "step": 437
    },
    {
      "completion_length": 1594.8750610351562,
      "epoch": 0.5005714285714286,
      "grad_norm": 2.209979295730591,
      "kl": 0.8747100830078125,
      "learning_rate": 1.4150013466019114e-07,
      "loss": 0.035,
      "reward": 0.08124704580404796,
      "reward_std": 0.10059291700599715,
      "rewards/cosine_scaled_reward": -0.1798710956936702,
      "rewards/format_reward": 0.8333333507180214,
      "step": 438
    },
    {
      "completion_length": 1178.1875305175781,
      "epoch": 0.5017142857142857,
      "grad_norm": 1.6698002815246582,
      "kl": 0.3531494140625,
      "learning_rate": 1.4019235263722034e-07,
      "loss": 0.0141,
      "reward": 0.07717898802366108,
      "reward_std": 0.06667181965894997,
      "rewards/cosine_scaled_reward": -0.24372687563300133,
      "rewards/format_reward": 0.9375000149011612,
      "step": 439
    },
    {
      "completion_length": 1250.7083549499512,
      "epoch": 0.5028571428571429,
      "grad_norm": 3.8609611988067627,
      "kl": 0.908416748046875,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0363,
      "reward": 0.08707587665412575,
      "reward_std": 0.0901845304761082,
      "rewards/cosine_scaled_reward": -0.19678178988397121,
      "rewards/format_reward": 0.8958333432674408,
      "step": 440
    },
    {
      "completion_length": 1613.2292098999023,
      "epoch": 0.504,
      "grad_norm": 2.8707664012908936,
      "kl": 1.0423583984375,
      "learning_rate": 1.3763677169699217e-07,
      "loss": 0.0417,
      "reward": 0.1019732168642804,
      "reward_std": 0.11968619748950005,
      "rewards/cosine_scaled_reward": -0.07690610364079475,
      "rewards/format_reward": 0.7500000111758709,
      "step": 441
    },
    {
      "completion_length": 1124.3125228881836,
      "epoch": 0.5051428571428571,
      "grad_norm": 3.18782377243042,
      "kl": 0.6033172607421875,
      "learning_rate": 1.3638909733514452e-07,
      "loss": 0.0241,
      "reward": 0.1847122572362423,
      "reward_std": 0.11424465058371425,
      "rewards/cosine_scaled_reward": 0.06125026382505894,
      "rewards/format_reward": 0.9375000149011612,
      "step": 442
    },
    {
      "completion_length": 1618.208396911621,
      "epoch": 0.5062857142857143,
      "grad_norm": 3.1647167205810547,
      "kl": 0.8673095703125,
      "learning_rate": 1.351615817851748e-07,
      "loss": 0.0347,
      "reward": 0.1230522379046306,
      "reward_std": 0.08929805480875075,
      "rewards/cosine_scaled_reward": -0.06928862258791924,
      "rewards/format_reward": 0.8541666939854622,
      "step": 443
    },
    {
      "completion_length": 1356.6458740234375,
      "epoch": 0.5074285714285715,
      "grad_norm": 2.186877489089966,
      "kl": 0.634979248046875,
      "learning_rate": 1.3395428487445914e-07,
      "loss": 0.0254,
      "reward": 0.11778642190620303,
      "reward_std": 0.10000652819871902,
      "rewards/cosine_scaled_reward": -0.13054069224745035,
      "rewards/format_reward": 0.9375000149011612,
      "step": 444
    },
    {
      "completion_length": 1302.6042022705078,
      "epoch": 0.5085714285714286,
      "grad_norm": 1.7782143354415894,
      "kl": 0.7170562744140625,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0287,
      "reward": 0.09130575158633292,
      "reward_std": 0.09221122646704316,
      "rewards/cosine_scaled_reward": -0.1979788908502087,
      "rewards/format_reward": 0.9166666865348816,
      "step": 445
    },
    {
      "completion_length": 1449.1667022705078,
      "epoch": 0.5097142857142857,
      "grad_norm": 2.79809308052063,
      "kl": 0.5194091796875,
      "learning_rate": 1.316005813502869e-07,
      "loss": 0.0207,
      "reward": 0.10592652973718941,
      "reward_std": 0.0978604587726295,
      "rewards/cosine_scaled_reward": -0.12308421358466148,
      "rewards/format_reward": 0.854166679084301,
      "step": 446
    },
    {
      "completion_length": 1229.6667137145996,
      "epoch": 0.5108571428571429,
      "grad_norm": 1.967965006828308,
      "kl": 0.792236328125,
      "learning_rate": 1.3045428945301953e-07,
      "loss": 0.0317,
      "reward": 0.12869366817176342,
      "reward_std": 0.09020477347075939,
      "rewards/cosine_scaled_reward": -0.07901583984494209,
      "rewards/format_reward": 0.895833358168602,
      "step": 447
    },
    {
      "completion_length": 1233.4167022705078,
      "epoch": 0.512,
      "grad_norm": 5.0800299644470215,
      "kl": 0.8968582153320312,
      "learning_rate": 1.2932844562179352e-07,
      "loss": 0.0358,
      "reward": 0.12176637991797179,
      "reward_std": 0.07025636686012149,
      "rewards/cosine_scaled_reward": -0.10369249619543552,
      "rewards/format_reward": 0.916666679084301,
      "step": 448
    },
    {
      "completion_length": 1115.104206085205,
      "epoch": 0.5131428571428571,
      "grad_norm": 5.752594947814941,
      "kl": 0.5258560180664062,
      "learning_rate": 1.2822310472864885e-07,
      "loss": 0.021,
      "reward": 0.09505507163703442,
      "reward_std": 0.08659794740378857,
      "rewards/cosine_scaled_reward": -0.18167979642748833,
      "rewards/format_reward": 0.916666679084301,
      "step": 449
    },
    {
      "completion_length": 1027.0416946411133,
      "epoch": 0.5142857142857142,
      "grad_norm": 1.9456526041030884,
      "kl": 0.33565521240234375,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.0134,
      "reward": 0.1392190819606185,
      "reward_std": 0.08027261192910373,
      "rewards/cosine_scaled_reward": -0.06442757230252028,
      "rewards/format_reward": 0.9375000074505806,
      "step": 450
    },
    {
      "completion_length": 1365.3125495910645,
      "epoch": 0.5154285714285715,
      "grad_norm": 2.0280954837799072,
      "kl": 0.9598846435546875,
      "learning_rate": 1.260741462457165e-07,
      "loss": 0.0384,
      "reward": 0.14097292395308614,
      "reward_std": 0.1067453664727509,
      "rewards/cosine_scaled_reward": -0.06067850440740585,
      "rewards/format_reward": 0.9375000074505806,
      "step": 451
    },
    {
      "completion_length": 1629.979248046875,
      "epoch": 0.5165714285714286,
      "grad_norm": 2.833832025527954,
      "kl": 1.052734375,
      "learning_rate": 1.2503063339313356e-07,
      "loss": 0.0421,
      "reward": 0.1490084226243198,
      "reward_std": 0.1209279503673315,
      "rewards/cosine_scaled_reward": 0.01308462768793106,
      "rewards/format_reward": 0.8333333507180214,
      "step": 452
    },
    {
      "completion_length": 1423.2083702087402,
      "epoch": 0.5177142857142857,
      "grad_norm": 2.2027335166931152,
      "kl": 0.909820556640625,
      "learning_rate": 1.2400783294793668e-07,
      "loss": 0.0364,
      "reward": 0.12369064858648926,
      "reward_std": 0.12611138448119164,
      "rewards/cosine_scaled_reward": -0.08292348496615887,
      "rewards/format_reward": 0.8750000223517418,
      "step": 453
    },
    {
      "completion_length": 1188.7500381469727,
      "epoch": 0.5188571428571429,
      "grad_norm": 2.987781524658203,
      "kl": 0.386993408203125,
      "learning_rate": 1.2300579475997657e-07,
      "loss": 0.0155,
      "reward": 0.11641452787443995,
      "reward_std": 0.0906132124364376,
      "rewards/cosine_scaled_reward": -0.12823456013575196,
      "rewards/format_reward": 0.9375000074505806,
      "step": 454
    },
    {
      "completion_length": 1531.6875457763672,
      "epoch": 0.52,
      "grad_norm": 2.0451388359069824,
      "kl": 0.9672393798828125,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.0387,
      "reward": 0.08102269377559423,
      "reward_std": 0.10998783679679036,
      "rewards/cosine_scaled_reward": -0.17945297434926033,
      "rewards/format_reward": 0.8333333507180214,
      "step": 455
    },
    {
      "completion_length": 1520.6458587646484,
      "epoch": 0.5211428571428571,
      "grad_norm": 1.6886857748031616,
      "kl": 0.6373214721679688,
      "learning_rate": 1.2106419949317388e-07,
      "loss": 0.0255,
      "reward": 0.04929352249018848,
      "reward_std": 0.10546231491025537,
      "rewards/cosine_scaled_reward": -0.23321845568716526,
      "rewards/format_reward": 0.7500000111758709,
      "step": 456
    },
    {
      "completion_length": 1495.3750534057617,
      "epoch": 0.5222857142857142,
      "grad_norm": 3.0165514945983887,
      "kl": 0.941650390625,
      "learning_rate": 1.2012473704494537e-07,
      "loss": 0.0377,
      "reward": 0.09918425139039755,
      "reward_std": 0.12838075123727322,
      "rewards/cosine_scaled_reward": -0.124738659709692,
      "rewards/format_reward": 0.8125000111758709,
      "step": 457
    },
    {
      "completion_length": 1388.5208740234375,
      "epoch": 0.5234285714285715,
      "grad_norm": 1.8302446603775024,
      "kl": 0.6633834838867188,
      "learning_rate": 1.1920622611056974e-07,
      "loss": 0.0266,
      "reward": 0.07702152655110694,
      "reward_std": 0.08541670115664601,
      "rewards/cosine_scaled_reward": -0.21506773598957807,
      "rewards/format_reward": 0.875,
      "step": 458
    },
    {
      "completion_length": 1343.5208473205566,
      "epoch": 0.5245714285714286,
      "grad_norm": 2.486727714538574,
      "kl": 0.926055908203125,
      "learning_rate": 1.1830871145697412e-07,
      "loss": 0.0371,
      "reward": 0.1338568499777466,
      "reward_std": 0.14767975080758333,
      "rewards/cosine_scaled_reward": -0.04322970123030245,
      "rewards/format_reward": 0.8541666865348816,
      "step": 459
    },
    {
      "completion_length": 1559.1250762939453,
      "epoch": 0.5257142857142857,
      "grad_norm": 1.7466343641281128,
      "kl": 0.7754058837890625,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.031,
      "reward": 0.11281750063062645,
      "reward_std": 0.09168540453538299,
      "rewards/cosine_scaled_reward": -0.10402092151343822,
      "rewards/format_reward": 0.8541666865348816,
      "step": 460
    },
    {
      "completion_length": 1441.5417098999023,
      "epoch": 0.5268571428571428,
      "grad_norm": 2.377612829208374,
      "kl": 0.6788558959960938,
      "learning_rate": 1.1657684494105386e-07,
      "loss": 0.0271,
      "reward": 0.12857838673517108,
      "reward_std": 0.1296999854966998,
      "rewards/cosine_scaled_reward": -0.040342007763683796,
      "rewards/format_reward": 0.8333333395421505,
      "step": 461
    },
    {
      "completion_length": 1260.895866394043,
      "epoch": 0.528,
      "grad_norm": 5.065842151641846,
      "kl": 0.7448501586914062,
      "learning_rate": 1.1574257748745986e-07,
      "loss": 0.0298,
      "reward": 0.08250287733972073,
      "reward_std": 0.11116986721754074,
      "rewards/cosine_scaled_reward": -0.15972350211814046,
      "rewards/format_reward": 0.7916666865348816,
      "step": 462
    },
    {
      "completion_length": 1197.4166946411133,
      "epoch": 0.5291428571428571,
      "grad_norm": 2.4315950870513916,
      "kl": 0.316741943359375,
      "learning_rate": 1.1492947512799328e-07,
      "loss": 0.0126,
      "reward": 0.17255353461951017,
      "reward_std": 0.14624354103580117,
      "rewards/cosine_scaled_reward": 0.10790145858481992,
      "rewards/format_reward": 0.7916666753590107,
      "step": 463
    },
    {
      "completion_length": 1026.7292098999023,
      "epoch": 0.5302857142857142,
      "grad_norm": 1.8613412380218506,
      "kl": 0.590972900390625,
      "learning_rate": 1.1413757749211602e-07,
      "loss": 0.0236,
      "reward": 0.17181929713115096,
      "reward_std": 0.12013115221634507,
      "rewards/cosine_scaled_reward": 0.02979774959385395,
      "rewards/format_reward": 0.9375000149011612,
      "step": 464
    },
    {
      "completion_length": 1346.9375305175781,
      "epoch": 0.5314285714285715,
      "grad_norm": 1.951102614402771,
      "kl": 0.7594528198242188,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0304,
      "reward": 0.12367417407222092,
      "reward_std": 0.13810825487598777,
      "rewards/cosine_scaled_reward": -0.10004201903939247,
      "rewards/format_reward": 0.9166666716337204,
      "step": 465
    },
    {
      "completion_length": 1361.2708702087402,
      "epoch": 0.5325714285714286,
      "grad_norm": 4.891315937042236,
      "kl": 0.6375732421875,
      "learning_rate": 1.1261754973965422e-07,
      "loss": 0.0255,
      "reward": 0.1684982028673403,
      "reward_std": 0.13606750033795834,
      "rewards/cosine_scaled_reward": 0.051418120972812176,
      "rewards/format_reward": 0.875,
      "step": 466
    },
    {
      "completion_length": 1727.2708740234375,
      "epoch": 0.5337142857142857,
      "grad_norm": 3.141331434249878,
      "kl": 0.8231964111328125,
      "learning_rate": 1.1188949370707787e-07,
      "loss": 0.0329,
      "reward": 0.0738575104624033,
      "reward_std": 0.11307883076369762,
      "rewards/cosine_scaled_reward": -0.20494843367487192,
      "rewards/format_reward": 0.8333333358168602,
      "step": 467
    },
    {
      "completion_length": 1474.2708930969238,
      "epoch": 0.5348571428571428,
      "grad_norm": 2.9029293060302734,
      "kl": 0.9022445678710938,
      "learning_rate": 1.1118279056249653e-07,
      "loss": 0.0361,
      "reward": 0.11058385335491039,
      "reward_std": 0.14076802507042885,
      "rewards/cosine_scaled_reward": -0.07730268314480782,
      "rewards/format_reward": 0.7916666846722364,
      "step": 468
    },
    {
      "completion_length": 1365.81254196167,
      "epoch": 0.536,
      "grad_norm": 2.5009942054748535,
      "kl": 0.7577972412109375,
      "learning_rate": 1.1049747474962444e-07,
      "loss": 0.0303,
      "reward": 0.14495484717190266,
      "reward_std": 0.13738009426742792,
      "rewards/cosine_scaled_reward": 0.003895143046975136,
      "rewards/format_reward": 0.8333333395421505,
      "step": 469
    },
    {
      "completion_length": 1764.3333740234375,
      "epoch": 0.5371428571428571,
      "grad_norm": 3.4764883518218994,
      "kl": 1.04931640625,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.042,
      "reward": 0.12439248080772813,
      "reward_std": 0.1104172533378005,
      "rewards/cosine_scaled_reward": -0.042109834030270576,
      "rewards/format_reward": 0.791666679084301,
      "step": 470
    },
    {
      "completion_length": 1697.2083587646484,
      "epoch": 0.5382857142857143,
      "grad_norm": 3.593810558319092,
      "kl": 1.015869140625,
      "learning_rate": 1.0919113768029517e-07,
      "loss": 0.0406,
      "reward": 0.16313168196938932,
      "reward_std": 0.1711752563714981,
      "rewards/cosine_scaled_reward": 0.08633797615766525,
      "rewards/format_reward": 0.7708333544433117,
      "step": 471
    },
    {
      "completion_length": 1437.3125457763672,
      "epoch": 0.5394285714285715,
      "grad_norm": 2.227132558822632,
      "kl": 0.584442138671875,
      "learning_rate": 1.0857018009286381e-07,
      "loss": 0.0234,
      "reward": 0.049232515739277005,
      "reward_std": 0.10539399227127433,
      "rewards/cosine_scaled_reward": -0.2601514309644699,
      "rewards/format_reward": 0.8125000186264515,
      "step": 472
    },
    {
      "completion_length": 1427.6667175292969,
      "epoch": 0.5405714285714286,
      "grad_norm": 2.773785352706909,
      "kl": 0.7134475708007812,
      "learning_rate": 1.0797073717209013e-07,
      "loss": 0.0286,
      "reward": 0.06174903141800314,
      "reward_std": 0.0896931691095233,
      "rewards/cosine_scaled_reward": -0.2288785793352872,
      "rewards/format_reward": 0.8125000055879354,
      "step": 473
    },
    {
      "completion_length": 1671.583366394043,
      "epoch": 0.5417142857142857,
      "grad_norm": 3.813199520111084,
      "kl": 1.1416015625,
      "learning_rate": 1.0739283813397639e-07,
      "loss": 0.0456,
      "reward": 0.22307377692777663,
      "reward_std": 0.1535223526880145,
      "rewards/cosine_scaled_reward": 0.24020265229046345,
      "rewards/format_reward": 0.8125000149011612,
      "step": 474
    },
    {
      "completion_length": 1295.3333587646484,
      "epoch": 0.5428571428571428,
      "grad_norm": 1.257153868675232,
      "kl": 0.5095977783203125,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.0204,
      "reward": 0.11992977559566498,
      "reward_std": 0.13077597226947546,
      "rewards/cosine_scaled_reward": -0.1030871132388711,
      "rewards/format_reward": 0.8958333432674408,
      "step": 475
    },
    {
      "completion_length": 1405.2500457763672,
      "epoch": 0.544,
      "grad_norm": 1.7341384887695312,
      "kl": 0.8976593017578125,
      "learning_rate": 1.063017833182728e-07,
      "loss": 0.0359,
      "reward": 0.11575727723538876,
      "reward_std": 0.13892074767500162,
      "rewards/cosine_scaled_reward": -0.09050496038980782,
      "rewards/format_reward": 0.8541666828095913,
      "step": 476
    },
    {
      "completion_length": 1334.520866394043,
      "epoch": 0.5451428571428572,
      "grad_norm": 2.1544551849365234,
      "kl": 0.790008544921875,
      "learning_rate": 1.0578868071715544e-07,
      "loss": 0.0317,
      "reward": 0.1808297468814999,
      "reward_std": 0.15548726078122854,
      "rewards/cosine_scaled_reward": 0.09577041026204824,
      "rewards/format_reward": 0.8541666865348816,
      "step": 477
    },
    {
      "completion_length": 1640.0417098999023,
      "epoch": 0.5462857142857143,
      "grad_norm": 2.7642574310302734,
      "kl": 1.0536422729492188,
      "learning_rate": 1.0529722834905125e-07,
      "loss": 0.0421,
      "reward": 0.12267802411224693,
      "reward_std": 0.11104590399190784,
      "rewards/cosine_scaled_reward": -0.03174232318997383,
      "rewards/format_reward": 0.7708333469927311,
      "step": 478
    },
    {
      "completion_length": 1397.3750457763672,
      "epoch": 0.5474285714285714,
      "grad_norm": 3.118372678756714,
      "kl": 0.7571868896484375,
      "learning_rate": 1.0482745016665526e-07,
      "loss": 0.0302,
      "reward": 0.11781717138364911,
      "reward_std": 0.11601145751774311,
      "rewards/cosine_scaled_reward": -0.06828589458018541,
      "rewards/format_reward": 0.8125000186264515,
      "step": 479
    },
    {
      "completion_length": 1380.4375381469727,
      "epoch": 0.5485714285714286,
      "grad_norm": 2.790198802947998,
      "kl": 1.3089599609375,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0523,
      "reward": 0.09327662736177444,
      "reward_std": 0.12549111153930426,
      "rewards/cosine_scaled_reward": -0.14611217193305492,
      "rewards/format_reward": 0.8333333507180214,
      "step": 480
    },
    {
      "completion_length": 1502.9375457763672,
      "epoch": 0.5497142857142857,
      "grad_norm": 3.935354232788086,
      "kl": 0.792388916015625,
      "learning_rate": 1.0395300688680625e-07,
      "loss": 0.0317,
      "reward": 0.06033009593375027,
      "reward_std": 0.12011839542537928,
      "rewards/cosine_scaled_reward": -0.16942069120705128,
      "rewards/format_reward": 0.6875000186264515,
      "step": 481
    },
    {
      "completion_length": 1320.3541984558105,
      "epoch": 0.5508571428571428,
      "grad_norm": 3.4273793697357178,
      "kl": 0.8202667236328125,
      "learning_rate": 1.0354838440848501e-07,
      "loss": 0.0328,
      "reward": 0.145433189580217,
      "reward_std": 0.13374328007921576,
      "rewards/cosine_scaled_reward": -0.03734024800360203,
      "rewards/format_reward": 0.916666679084301,
      "step": 482
    },
    {
      "completion_length": 1709.2500534057617,
      "epoch": 0.552,
      "grad_norm": 3.1256120204925537,
      "kl": 1.09405517578125,
      "learning_rate": 1.0316552135205837e-07,
      "loss": 0.0438,
      "reward": 0.10491634625941515,
      "reward_std": 0.13583884108811617,
      "rewards/cosine_scaled_reward": -0.11350399069488049,
      "rewards/format_reward": 0.8333333432674408,
      "step": 483
    },
    {
      "completion_length": 1060.3125457763672,
      "epoch": 0.5531428571428572,
      "grad_norm": 2.7416789531707764,
      "kl": 0.593719482421875,
      "learning_rate": 1.0280443637773163e-07,
      "loss": 0.0237,
      "reward": 0.13031714130192995,
      "reward_std": 0.12379100965335965,
      "rewards/cosine_scaled_reward": -0.09740070005500456,
      "rewards/format_reward": 0.9375000074505806,
      "step": 484
    },
    {
      "completion_length": 1706.8333740234375,
      "epoch": 0.5542857142857143,
      "grad_norm": 2.7037065029144287,
      "kl": 1.599609375,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.064,
      "reward": 0.07579714641906321,
      "reward_std": 0.1142753018066287,
      "rewards/cosine_scaled_reward": -0.1662511508911848,
      "rewards/format_reward": 0.7708333469927311,
      "step": 485
    },
    {
      "completion_length": 909.7500267028809,
      "epoch": 0.5554285714285714,
      "grad_norm": 2.9626100063323975,
      "kl": 0.5513458251953125,
      "learning_rate": 1.0214767000817596e-07,
      "loss": 0.0221,
      "reward": 0.16371853230521083,
      "reward_std": 0.08725554682314396,
      "rewards/cosine_scaled_reward": -0.0292919734492898,
      "rewards/format_reward": 1.0,
      "step": 486
    },
    {
      "completion_length": 977.083366394043,
      "epoch": 0.5565714285714286,
      "grad_norm": 1.2269593477249146,
      "kl": 0.354217529296875,
      "learning_rate": 1.0185202062281336e-07,
      "loss": 0.0142,
      "reward": 0.18861438240855932,
      "reward_std": 0.12223522993735969,
      "rewards/cosine_scaled_reward": 0.09728494752198458,
      "rewards/format_reward": 0.9166666716337204,
      "step": 487
    },
    {
      "completion_length": 1098.43754196167,
      "epoch": 0.5577142857142857,
      "grad_norm": 3.2133853435516357,
      "kl": 0.5051727294921875,
      "learning_rate": 1.0157821333772304e-07,
      "loss": 0.0202,
      "reward": 0.13033967884257436,
      "reward_std": 0.08723578602075577,
      "rewards/cosine_scaled_reward": -0.08787534758448601,
      "rewards/format_reward": 0.9375000074505806,
      "step": 488
    },
    {
      "completion_length": 1465.5000457763672,
      "epoch": 0.5588571428571428,
      "grad_norm": 4.108081817626953,
      "kl": 0.8738555908203125,
      "learning_rate": 1.013262614978859e-07,
      "loss": 0.035,
      "reward": 0.0360484067350626,
      "reward_std": 0.08054300001822412,
      "rewards/cosine_scaled_reward": -0.2918727397918701,
      "rewards/format_reward": 0.7916666865348816,
      "step": 489
    },
    {
      "completion_length": 1255.0416984558105,
      "epoch": 0.56,
      "grad_norm": 1.202156662940979,
      "kl": 0.7819290161132812,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.0313,
      "reward": 0.15311120147816837,
      "reward_std": 0.10377770848572254,
      "rewards/cosine_scaled_reward": -0.02998751401901245,
      "rewards/format_reward": 0.9583333432674408,
      "step": 490
    },
    {
      "completion_length": 1578.0625381469727,
      "epoch": 0.5611428571428572,
      "grad_norm": 1.941928505897522,
      "kl": 0.84698486328125,
      "learning_rate": 1.0088797220727779e-07,
      "loss": 0.0339,
      "reward": 0.13159164262469858,
      "reward_std": 0.13246915489435196,
      "rewards/cosine_scaled_reward": -0.05548815353540704,
      "rewards/format_reward": 0.8750000074505806,
      "step": 491
    },
    {
      "completion_length": 1292.3333587646484,
      "epoch": 0.5622857142857143,
      "grad_norm": 2.9671390056610107,
      "kl": 0.797515869140625,
      "learning_rate": 1.0070165611810855e-07,
      "loss": 0.0319,
      "reward": 0.13928497838787735,
      "reward_std": 0.10173067264258862,
      "rewards/cosine_scaled_reward": -0.08179534692317247,
      "rewards/format_reward": 0.9583333432674408,
      "step": 492
    },
    {
      "completion_length": 1064.8125228881836,
      "epoch": 0.5634285714285714,
      "grad_norm": 2.1849365234375,
      "kl": 0.4040985107421875,
      "learning_rate": 1.005372381963547e-07,
      "loss": 0.0162,
      "reward": 0.13772490341216326,
      "reward_std": 0.14829062833450735,
      "rewards/cosine_scaled_reward": -0.07700726587790996,
      "rewards/format_reward": 0.9583333432674408,
      "step": 493
    },
    {
      "completion_length": 1061.7291870117188,
      "epoch": 0.5645714285714286,
      "grad_norm": 1.5284849405288696,
      "kl": 0.540435791015625,
      "learning_rate": 1.0039472645551372e-07,
      "loss": 0.0217,
      "reward": 0.12483312236145139,
      "reward_std": 0.1254338538274169,
      "rewards/cosine_scaled_reward": -0.08674540685024112,
      "rewards/format_reward": 0.8958333432674408,
      "step": 494
    },
    {
      "completion_length": 1627.0625305175781,
      "epoch": 0.5657142857142857,
      "grad_norm": 5.17227029800415,
      "kl": 1.3586196899414062,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.0544,
      "reward": 0.11015307196066715,
      "reward_std": 0.13126440905034542,
      "rewards/cosine_scaled_reward": -0.08530823234468699,
      "rewards/format_reward": 0.8125000149011612,
      "step": 495
    },
    {
      "completion_length": 1575.4167098999023,
      "epoch": 0.5668571428571428,
      "grad_norm": 3.1292481422424316,
      "kl": 1.2574386596679688,
      "learning_rate": 1.0017544823184055e-07,
      "loss": 0.0503,
      "reward": 0.1306152348406613,
      "reward_std": 0.08260433259420097,
      "rewards/cosine_scaled_reward": -0.013811783166602254,
      "rewards/format_reward": 0.7916666716337204,
      "step": 496
    },
    {
      "completion_length": 1092.5208587646484,
      "epoch": 0.568,
      "grad_norm": 3.035909414291382,
      "kl": 0.6652145385742188,
      "learning_rate": 1.0009869243631952e-07,
      "loss": 0.0266,
      "reward": 0.17385427234694362,
      "reward_std": 0.09661010140553117,
      "rewards/cosine_scaled_reward": 0.058778489008545876,
      "rewards/format_reward": 0.8958333358168602,
      "step": 497
    },
    {
      "completion_length": 1320.1250457763672,
      "epoch": 0.5691428571428572,
      "grad_norm": 1.9357889890670776,
      "kl": 0.88330078125,
      "learning_rate": 1.000438641958131e-07,
      "loss": 0.0354,
      "reward": 0.13382654823362827,
      "reward_std": 0.11791369551792741,
      "rewards/cosine_scaled_reward": -0.06090674642473459,
      "rewards/format_reward": 0.8958333432674408,
      "step": 498
    },
    {
      "completion_length": 1220.7292022705078,
      "epoch": 0.5702857142857143,
      "grad_norm": 2.041574239730835,
      "kl": 0.31233978271484375,
      "learning_rate": 1.0001096618257236e-07,
      "loss": 0.0125,
      "reward": 0.15456983912736177,
      "reward_std": 0.130745030939579,
      "rewards/cosine_scaled_reward": -0.008478153496980667,
      "rewards/format_reward": 0.9166666865348816,
      "step": 499
    },
    {
      "completion_length": 1320.8750457763672,
      "epoch": 0.5714285714285714,
      "grad_norm": 3.362833023071289,
      "kl": 0.72802734375,
      "learning_rate": 1e-07,
      "loss": 0.0292,
      "reward": 0.11918663769029081,
      "reward_std": 0.1335056396201253,
      "rewards/cosine_scaled_reward": -0.1146287601441145,
      "rewards/format_reward": 0.9166666865348816,
      "step": 500
    },
    {
      "epoch": 0.5714285714285714,
      "step": 500,
      "total_flos": 0.0,
      "train_loss": 0.012724974361105416,
      "train_runtime": 59499.213,
      "train_samples_per_second": 0.403,
      "train_steps_per_second": 0.008
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}