{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2857142857142857,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 3023.0,
      "epoch": 0.0005714285714285715,
      "grad_norm": 0.2460898458957672,
      "kl": 0.0,
      "learning_rate": 2e-08,
      "loss": -0.0314,
      "num_tokens": 151404.0,
      "reward": -0.17859874665737152,
      "reward_std": 0.18563616648316383,
      "rewards/cosine_scaled_reward": -0.08929938450455666,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2731.3958740234375,
      "epoch": 0.001142857142857143,
      "grad_norm": 0.22445940971374512,
      "kl": 0.0,
      "learning_rate": 4e-08,
      "loss": 0.0519,
      "num_tokens": 288319.0,
      "reward": -0.535461600869894,
      "reward_std": 0.16202664375305176,
      "rewards/cosine_scaled_reward": -0.2677307929843664,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2233.5416717529297,
      "epoch": 0.0017142857142857142,
      "grad_norm": 0.24249283969402313,
      "kl": 0.00036716461181640625,
      "learning_rate": 6e-08,
      "loss": 0.0576,
      "num_tokens": 401025.0,
      "reward": 0.33875009417533875,
      "reward_std": 0.6338529586791992,
      "rewards/cosine_scaled_reward": 0.16937505267560482,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2376.2500915527344,
      "epoch": 0.002285714285714286,
      "grad_norm": 0.3028571605682373,
      "kl": 0.0006170272827148438,
      "learning_rate": 8e-08,
      "loss": 0.0773,
      "num_tokens": 521001.0,
      "reward": -0.4882083088159561,
      "reward_std": 0.4496277957223356,
      "rewards/cosine_scaled_reward": -0.24410414695739746,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3372.3750610351562,
      "epoch": 0.002857142857142857,
      "grad_norm": 0.21095335483551025,
      "kl": 0.0006723403930664062,
      "learning_rate": 1e-07,
      "loss": 0.007,
      "num_tokens": 689631.0,
      "reward": -0.5746253430843353,
      "reward_std": 0.23810617998242378,
      "rewards/cosine_scaled_reward": -0.28731267154216766,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2820.375030517578,
      "epoch": 0.0034285714285714284,
      "grad_norm": 0.33144864439964294,
      "kl": 0.0005979537963867188,
      "learning_rate": 1.2e-07,
      "loss": -0.0916,
      "num_tokens": 831357.0,
      "reward": -0.4876829609274864,
      "reward_std": 0.20684241317212582,
      "rewards/cosine_scaled_reward": -0.24384147115051746,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3243.8750610351562,
      "epoch": 0.004,
      "grad_norm": 0.20102664828300476,
      "kl": 0.0005273818969726562,
      "learning_rate": 1.4e-07,
      "loss": -0.0443,
      "num_tokens": 992799.0,
      "reward": -0.30651520285755396,
      "reward_std": 0.3921822514384985,
      "rewards/cosine_scaled_reward": -0.15325760166160762,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3092.625,
      "epoch": 0.004571428571428572,
      "grad_norm": 0.23749692738056183,
      "kl": 0.0005922317504882812,
      "learning_rate": 1.6e-07,
      "loss": -0.0614,
      "num_tokens": 1146921.0,
      "reward": -0.8370707631111145,
      "reward_std": 0.16819308325648308,
      "rewards/cosine_scaled_reward": -0.41853538155555725,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2227.9166717529297,
      "epoch": 0.005142857142857143,
      "grad_norm": 0.3016290068626404,
      "kl": 0.0005288124084472656,
      "learning_rate": 1.8e-07,
      "loss": -0.0113,
      "num_tokens": 1259777.0,
      "reward": -0.15989744663238525,
      "reward_std": 0.25975861586630344,
      "rewards/cosine_scaled_reward": -0.07994873821735382,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3047.4376220703125,
      "epoch": 0.005714285714285714,
      "grad_norm": 0.21191374957561493,
      "kl": 0.000568389892578125,
      "learning_rate": 2e-07,
      "loss": -0.0152,
      "num_tokens": 1411430.0,
      "reward": -0.019763831049203873,
      "reward_std": 0.8465264737606049,
      "rewards/cosine_scaled_reward": -0.00988190807402134,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2482.4375,
      "epoch": 0.006285714285714286,
      "grad_norm": 0.330398291349411,
      "kl": 0.0006580352783203125,
      "learning_rate": 2.1999999999999998e-07,
      "loss": 0.053,
      "num_tokens": 1536179.0,
      "reward": -0.4834251650609076,
      "reward_std": 0.5014891251921654,
      "rewards/cosine_scaled_reward": -0.2417125750798732,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2487.2500610351562,
      "epoch": 0.006857142857142857,
      "grad_norm": 0.30102092027664185,
      "kl": 0.000583648681640625,
      "learning_rate": 2.4e-07,
      "loss": -0.0912,
      "num_tokens": 1661063.0,
      "reward": -0.14752477407455444,
      "reward_std": 0.5868879407644272,
      "rewards/cosine_scaled_reward": -0.07376237958669662,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2876.229217529297,
      "epoch": 0.0074285714285714285,
      "grad_norm": 0.25117188692092896,
      "kl": 0.0005583763122558594,
      "learning_rate": 2.6e-07,
      "loss": -0.0352,
      "num_tokens": 1804702.0,
      "reward": -0.08496717864181846,
      "reward_std": 0.5994590483605862,
      "rewards/cosine_scaled_reward": -0.042483578145038337,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2376.166717529297,
      "epoch": 0.008,
      "grad_norm": 0.34051504731178284,
      "kl": 0.0006012916564941406,
      "learning_rate": 2.8e-07,
      "loss": 0.1844,
      "num_tokens": 1924458.0,
      "reward": 0.5019294954836369,
      "reward_std": 0.4014289937913418,
      "rewards/cosine_scaled_reward": 0.2509647514671087,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2104.3541870117188,
      "epoch": 0.008571428571428572,
      "grad_norm": 0.27839282155036926,
      "kl": 0.0004687309265136719,
      "learning_rate": 3e-07,
      "loss": 0.0602,
      "num_tokens": 2031107.0,
      "reward": -0.2739022574387491,
      "reward_std": 0.5232805069535971,
      "rewards/cosine_scaled_reward": -0.13695112499408424,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3022.2709350585938,
      "epoch": 0.009142857142857144,
      "grad_norm": 0.23329903185367584,
      "kl": 0.0007066726684570312,
      "learning_rate": 3.2e-07,
      "loss": -0.0235,
      "num_tokens": 2182272.0,
      "reward": -0.28543997276574373,
      "reward_std": 0.5580427274107933,
      "rewards/cosine_scaled_reward": -0.14271997893229127,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2995.8959350585938,
      "epoch": 0.009714285714285713,
      "grad_norm": 0.20035724341869354,
      "kl": 0.00064849853515625,
      "learning_rate": 3.4000000000000003e-07,
      "loss": 0.0584,
      "num_tokens": 2331607.0,
      "reward": -0.49751752614974976,
      "reward_std": 0.40262408554553986,
      "rewards/cosine_scaled_reward": -0.24875876307487488,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3225.3125,
      "epoch": 0.010285714285714285,
      "grad_norm": 0.20705807209014893,
      "kl": 0.0006122589111328125,
      "learning_rate": 3.6e-07,
      "loss": 0.0237,
      "num_tokens": 2492782.0,
      "reward": -0.5619450844824314,
      "reward_std": 0.47893428802490234,
      "rewards/cosine_scaled_reward": -0.2809725347906351,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2767.8541717529297,
      "epoch": 0.010857142857142857,
      "grad_norm": 0.25546160340309143,
      "kl": 0.0007276535034179688,
      "learning_rate": 3.7999999999999996e-07,
      "loss": -0.0882,
      "num_tokens": 2631327.0,
      "reward": -0.18378404527902603,
      "reward_std": 0.5495752617716789,
      "rewards/cosine_scaled_reward": -0.09189202263951302,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.000015258789,
      "epoch": 0.011428571428571429,
      "grad_norm": 0.3158569931983948,
      "kl": 0.0004367828369140625,
      "learning_rate": 4e-07,
      "loss": 0.0439,
      "num_tokens": 2708367.0,
      "reward": -0.025459617376327515,
      "reward_std": 0.57894092425704,
      "rewards/cosine_scaled_reward": -0.01272980123758316,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.2500457763672,
      "epoch": 0.012,
      "grad_norm": 0.4832952618598938,
      "kl": 0.00046253204345703125,
      "learning_rate": 4.1999999999999995e-07,
      "loss": 0.0893,
      "num_tokens": 2803731.0,
      "reward": -0.20306236669421196,
      "reward_std": 0.7009828165173531,
      "rewards/cosine_scaled_reward": -0.10153118334710598,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3143.604248046875,
      "epoch": 0.012571428571428572,
      "grad_norm": 0.20443888008594513,
      "kl": 0.0005559921264648438,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0298,
      "num_tokens": 2960960.0,
      "reward": 0.664834626019001,
      "reward_std": 1.0612835884094238,
      "rewards/cosine_scaled_reward": 0.3324173092842102,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2191.9791870117188,
      "epoch": 0.013142857142857144,
      "grad_norm": 0.23772495985031128,
      "kl": 0.00048279762268066406,
      "learning_rate": 4.6e-07,
      "loss": 0.0419,
      "num_tokens": 3071671.0,
      "reward": -0.08595703169703484,
      "reward_std": 0.7537456881254911,
      "rewards/cosine_scaled_reward": -0.042978519573807716,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2681.1458740234375,
      "epoch": 0.013714285714285714,
      "grad_norm": 0.2397710382938385,
      "kl": 0.0005970001220703125,
      "learning_rate": 4.8e-07,
      "loss": 0.0064,
      "num_tokens": 3206774.0,
      "reward": 0.038678646087646484,
      "reward_std": 0.3931765630841255,
      "rewards/cosine_scaled_reward": 0.01933930814266205,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.250015258789,
      "epoch": 0.014285714285714285,
      "grad_norm": 0.3846415579319,
      "kl": 0.0004100799560546875,
      "learning_rate": 5e-07,
      "loss": 0.0557,
      "num_tokens": 3287126.0,
      "reward": -0.13053925335407257,
      "reward_std": 0.4432575963437557,
      "rewards/cosine_scaled_reward": -0.06526962295174599,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2773.6250610351562,
      "epoch": 0.014857142857142857,
      "grad_norm": 0.23315444588661194,
      "kl": 0.0005998611450195312,
      "learning_rate": 5.2e-07,
      "loss": 0.0573,
      "num_tokens": 3426272.0,
      "reward": -0.14446274191141129,
      "reward_std": 0.7020265012979507,
      "rewards/cosine_scaled_reward": -0.07223137095570564,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3283.5,
      "epoch": 0.015428571428571429,
      "grad_norm": 0.21181795001029968,
      "kl": 0.0006422996520996094,
      "learning_rate": 5.4e-07,
      "loss": -0.0769,
      "num_tokens": 3590060.0,
      "reward": -0.299600200727582,
      "reward_std": 0.4262968748807907,
      "rewards/cosine_scaled_reward": -0.14980009896680713,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2844.0833740234375,
      "epoch": 0.016,
      "grad_norm": 0.21920780837535858,
      "kl": 0.000614166259765625,
      "learning_rate": 5.6e-07,
      "loss": 0.0034,
      "num_tokens": 3733524.0,
      "reward": -0.4807719439268112,
      "reward_std": 0.42509571835398674,
      "rewards/cosine_scaled_reward": -0.24038597010076046,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2722.250030517578,
      "epoch": 0.01657142857142857,
      "grad_norm": 0.31188592314720154,
      "kl": 0.0006227493286132812,
      "learning_rate": 5.8e-07,
      "loss": 0.0756,
      "num_tokens": 3871068.0,
      "reward": 0.08422036468982697,
      "reward_std": 0.6395114436745644,
      "rewards/cosine_scaled_reward": 0.04211018607020378,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3481.2709350585938,
      "epoch": 0.017142857142857144,
      "grad_norm": 0.20324808359146118,
      "kl": 0.000644683837890625,
      "learning_rate": 6e-07,
      "loss": 0.0018,
      "num_tokens": 4044145.0,
      "reward": -0.18417476117610931,
      "reward_std": 0.6154340840876102,
      "rewards/cosine_scaled_reward": -0.0920873824506998,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2657.166793823242,
      "epoch": 0.017714285714285714,
      "grad_norm": 0.3732287287712097,
      "kl": 0.0005855560302734375,
      "learning_rate": 6.2e-07,
      "loss": 0.1126,
      "num_tokens": 4176981.0,
      "reward": -0.34596723690629005,
      "reward_std": 0.6994314044713974,
      "rewards/cosine_scaled_reward": -0.17298361286520958,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.4583587646484,
      "epoch": 0.018285714285714287,
      "grad_norm": 0.34063395857810974,
      "kl": 0.0004658699035644531,
      "learning_rate": 6.4e-07,
      "loss": 0.0435,
      "num_tokens": 4271395.0,
      "reward": 0.1516597867012024,
      "reward_std": 0.7864086776971817,
      "rewards/cosine_scaled_reward": 0.0758299008011818,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.0,
      "epoch": 0.018857142857142857,
      "grad_norm": 0.22958050668239594,
      "kl": 0.000720977783203125,
      "learning_rate": 6.6e-07,
      "loss": 0.0,
      "num_tokens": 4450063.0,
      "reward": -0.6370590478181839,
      "reward_std": 0.20538340508937836,
      "rewards/cosine_scaled_reward": -0.31852949783205986,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2935.3959350585938,
      "epoch": 0.019428571428571427,
      "grad_norm": 0.2298842966556549,
      "kl": 0.0005502700805664062,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0374,
      "num_tokens": 4596926.0,
      "reward": -0.41714829951524734,
      "reward_std": 0.39900972694158554,
      "rewards/cosine_scaled_reward": -0.20857414416968822,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3036.9166870117188,
      "epoch": 0.02,
      "grad_norm": 0.23063404858112335,
      "kl": 0.0006365776062011719,
      "learning_rate": 7e-07,
      "loss": 0.0008,
      "num_tokens": 4749586.0,
      "reward": -0.7142433375120163,
      "reward_std": 0.3739009462296963,
      "rewards/cosine_scaled_reward": -0.35712166875600815,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2749.8958740234375,
      "epoch": 0.02057142857142857,
      "grad_norm": 0.27252835035324097,
      "kl": 0.000637054443359375,
      "learning_rate": 7.2e-07,
      "loss": -0.0058,
      "num_tokens": 4887449.0,
      "reward": -0.10459958261344582,
      "reward_std": 0.6155130080878735,
      "rewards/cosine_scaled_reward": -0.052299798757303506,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2935.562530517578,
      "epoch": 0.021142857142857144,
      "grad_norm": 0.2475793957710266,
      "kl": 0.000568389892578125,
      "learning_rate": 7.4e-07,
      "loss": -0.088,
      "num_tokens": 5034704.0,
      "reward": -0.29862387478351593,
      "reward_std": 0.39744907803833485,
      "rewards/cosine_scaled_reward": -0.14931193552911282,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3025.7083740234375,
      "epoch": 0.021714285714285714,
      "grad_norm": 0.22514301538467407,
      "kl": 0.0006685256958007812,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0347,
      "num_tokens": 5186814.0,
      "reward": -0.2246699258685112,
      "reward_std": 0.6279377490282059,
      "rewards/cosine_scaled_reward": -0.11233496479690075,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1896.0208435058594,
      "epoch": 0.022285714285714287,
      "grad_norm": 0.31844407320022583,
      "kl": 0.00075531005859375,
      "learning_rate": 7.799999999999999e-07,
      "loss": -0.0971,
      "num_tokens": 5282791.0,
      "reward": -0.26946142315864563,
      "reward_std": 0.6844599097967148,
      "rewards/cosine_scaled_reward": -0.13473070412874222,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3461.6458740234375,
      "epoch": 0.022857142857142857,
      "grad_norm": 0.1988898515701294,
      "kl": 0.0005855560302734375,
      "learning_rate": 8e-07,
      "loss": 0.0348,
      "num_tokens": 5455178.0,
      "reward": -0.3771579749882221,
      "reward_std": 0.43913378193974495,
      "rewards/cosine_scaled_reward": -0.18857897631824017,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2864.0625,
      "epoch": 0.023428571428571427,
      "grad_norm": 0.2541484534740448,
      "kl": 0.0006170272827148438,
      "learning_rate": 8.199999999999999e-07,
      "loss": 0.0332,
      "num_tokens": 5599517.0,
      "reward": -0.15848201513290405,
      "reward_std": 0.28519516810774803,
      "rewards/cosine_scaled_reward": -0.07924101501703262,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2103.375045776367,
      "epoch": 0.024,
      "grad_norm": 0.4904050827026367,
      "kl": 0.0005826950073242188,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.1226,
      "num_tokens": 5706251.0,
      "reward": 0.05192290246486664,
      "reward_std": 0.5277432054281235,
      "rewards/cosine_scaled_reward": 0.02596145309507847,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3375.9583740234375,
      "epoch": 0.02457142857142857,
      "grad_norm": 0.20466555655002594,
      "kl": 0.0007343292236328125,
      "learning_rate": 8.599999999999999e-07,
      "loss": 0.0472,
      "num_tokens": 5873601.0,
      "reward": -0.1930120848119259,
      "reward_std": 0.7160178981721401,
      "rewards/cosine_scaled_reward": -0.0965060293674469,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3035.7708740234375,
      "epoch": 0.025142857142857144,
      "grad_norm": 0.19474832713603973,
      "kl": 0.0006237030029296875,
      "learning_rate": 8.799999999999999e-07,
      "loss": -0.0035,
      "num_tokens": 6025522.0,
      "reward": -0.16648699529469013,
      "reward_std": 0.6652341857552528,
      "rewards/cosine_scaled_reward": -0.08324349066242576,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3335.8750610351562,
      "epoch": 0.025714285714285714,
      "grad_norm": 0.21470214426517487,
      "kl": 0.0007848739624023438,
      "learning_rate": 9e-07,
      "loss": 0.0771,
      "num_tokens": 6192064.0,
      "reward": -0.6564953848719597,
      "reward_std": 0.22902014665305614,
      "rewards/cosine_scaled_reward": -0.32824768498539925,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2241.8541870117188,
      "epoch": 0.026285714285714287,
      "grad_norm": 0.29658329486846924,
      "kl": 0.00080108642578125,
      "learning_rate": 9.2e-07,
      "loss": -0.0078,
      "num_tokens": 6305085.0,
      "reward": 0.3887103348970413,
      "reward_std": 1.0468786805868149,
      "rewards/cosine_scaled_reward": 0.19435517117381096,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2734.7083740234375,
      "epoch": 0.026857142857142857,
      "grad_norm": 0.28740641474723816,
      "kl": 0.0004706382751464844,
      "learning_rate": 9.399999999999999e-07,
      "loss": 0.049,
      "num_tokens": 6441355.0,
      "reward": 0.15308012068271637,
      "reward_std": 0.4208949161693454,
      "rewards/cosine_scaled_reward": 0.07654005661606789,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.8958740234375,
      "epoch": 0.027428571428571427,
      "grad_norm": 0.39340028166770935,
      "kl": 0.0004849433898925781,
      "learning_rate": 9.6e-07,
      "loss": 0.0855,
      "num_tokens": 6520190.0,
      "reward": 0.44709211960434914,
      "reward_std": 0.7460008524358273,
      "rewards/cosine_scaled_reward": 0.22354605607688427,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2304.1458435058594,
      "epoch": 0.028,
      "grad_norm": 0.35876935720443726,
      "kl": 0.0008916854858398438,
      "learning_rate": 9.8e-07,
      "loss": 0.0348,
      "num_tokens": 6636501.0,
      "reward": -0.05522707849740982,
      "reward_std": 0.5208401791751385,
      "rewards/cosine_scaled_reward": -0.02761353738605976,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3012.8541870117188,
      "epoch": 0.02857142857142857,
      "grad_norm": 0.2135663479566574,
      "kl": 0.0007343292236328125,
      "learning_rate": 1e-06,
      "loss": 0.0216,
      "num_tokens": 6786926.0,
      "reward": -0.08169351518154144,
      "reward_std": 0.6610444337129593,
      "rewards/cosine_scaled_reward": -0.040846746414899826,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2833.375,
      "epoch": 0.029142857142857144,
      "grad_norm": 0.24645818769931793,
      "kl": 0.0005292892456054688,
      "learning_rate": 9.999890338174275e-07,
      "loss": 0.0353,
      "num_tokens": 6929624.0,
      "reward": 0.0020843185484409332,
      "reward_std": 0.12999659916386008,
      "rewards/cosine_scaled_reward": 0.001042170450091362,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3556.0416870117188,
      "epoch": 0.029714285714285714,
      "grad_norm": 0.18516220152378082,
      "kl": 0.00058746337890625,
      "learning_rate": 9.999561358041868e-07,
      "loss": 0.0009,
      "num_tokens": 7107826.0,
      "reward": -0.3568333759903908,
      "reward_std": 0.5824087001383305,
      "rewards/cosine_scaled_reward": -0.1784166805446148,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2816.1666717529297,
      "epoch": 0.030285714285714287,
      "grad_norm": 0.3401743173599243,
      "kl": 0.0006856918334960938,
      "learning_rate": 9.999013075636804e-07,
      "loss": 0.06,
      "num_tokens": 7250142.0,
      "reward": -0.00018092244863510132,
      "reward_std": 0.16538633033633232,
      "rewards/cosine_scaled_reward": -9.047612547874451e-05,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3037.3958435058594,
      "epoch": 0.030857142857142857,
      "grad_norm": 0.39248475432395935,
      "kl": 0.000926971435546875,
      "learning_rate": 9.998245517681593e-07,
      "loss": -0.0691,
      "num_tokens": 7402561.0,
      "reward": -0.5911240540444851,
      "reward_std": 0.18204397335648537,
      "rewards/cosine_scaled_reward": -0.29556202609091997,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3278.8958740234375,
      "epoch": 0.03142857142857143,
      "grad_norm": 0.22106441855430603,
      "kl": 0.000995635986328125,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0338,
      "num_tokens": 7566500.0,
      "reward": -0.07510977238416672,
      "reward_std": 0.608110748231411,
      "rewards/cosine_scaled_reward": -0.03755488805472851,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2919.937530517578,
      "epoch": 0.032,
      "grad_norm": 0.27260297536849976,
      "kl": 0.0011653900146484375,
      "learning_rate": 9.996052735444862e-07,
      "loss": 0.0375,
      "num_tokens": 7712429.0,
      "reward": -0.1868463009595871,
      "reward_std": 0.5689870864152908,
      "rewards/cosine_scaled_reward": -0.09342315793037415,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2649.8751220703125,
      "epoch": 0.03257142857142857,
      "grad_norm": 0.21306385099887848,
      "kl": 0.0008172988891601562,
      "learning_rate": 9.994627618036452e-07,
      "loss": -0.0171,
      "num_tokens": 7845899.0,
      "reward": -0.43829748034477234,
      "reward_std": 0.6765277907252312,
      "rewards/cosine_scaled_reward": -0.21914873644709587,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3418.3958740234375,
      "epoch": 0.03314285714285714,
      "grad_norm": 0.19094346463680267,
      "kl": 0.00072479248046875,
      "learning_rate": 9.992983438818915e-07,
      "loss": 0.0116,
      "num_tokens": 8016522.0,
      "reward": -0.2600528746843338,
      "reward_std": 0.41667389310896397,
      "rewards/cosine_scaled_reward": -0.1300264373421669,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2712.9583740234375,
      "epoch": 0.03371428571428572,
      "grad_norm": 0.27561813592910767,
      "kl": 0.0011749267578125,
      "learning_rate": 9.991120277927223e-07,
      "loss": -0.0132,
      "num_tokens": 8152660.0,
      "reward": -0.2609961926937103,
      "reward_std": 0.5313794314861298,
      "rewards/cosine_scaled_reward": -0.13049809262156487,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3343.8125610351562,
      "epoch": 0.03428571428571429,
      "grad_norm": 0.20157206058502197,
      "kl": 0.0007047653198242188,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0128,
      "num_tokens": 8319607.0,
      "reward": 0.07680468261241913,
      "reward_std": 0.3094941098242998,
      "rewards/cosine_scaled_reward": 0.038402341306209564,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2177.208396911621,
      "epoch": 0.03485714285714286,
      "grad_norm": 0.36359933018684387,
      "kl": 0.0006518363952636719,
      "learning_rate": 9.98673738502114e-07,
      "loss": 0.14,
      "num_tokens": 8430461.0,
      "reward": 0.17365121096372604,
      "reward_std": 0.5538155660033226,
      "rewards/cosine_scaled_reward": 0.08682558685541153,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2966.7291870117188,
      "epoch": 0.03542857142857143,
      "grad_norm": 0.3397689163684845,
      "kl": 0.0008535385131835938,
      "learning_rate": 9.98421786662277e-07,
      "loss": -0.0699,
      "num_tokens": 8578936.0,
      "reward": -0.7255322933197021,
      "reward_std": 0.2827052026987076,
      "rewards/cosine_scaled_reward": -0.3627661466598511,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3450.8125610351562,
      "epoch": 0.036,
      "grad_norm": 0.18789339065551758,
      "kl": 0.0008687973022460938,
      "learning_rate": 9.981479793771866e-07,
      "loss": -0.0483,
      "num_tokens": 8750479.0,
      "reward": -0.2192160151898861,
      "reward_std": 0.5799107477068901,
      "rewards/cosine_scaled_reward": -0.10960800759494305,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3443.3541870117188,
      "epoch": 0.036571428571428574,
      "grad_norm": 0.17766065895557404,
      "kl": 0.0008435249328613281,
      "learning_rate": 9.97852329991824e-07,
      "loss": -0.0151,
      "num_tokens": 8922264.0,
      "reward": -0.48261551931500435,
      "reward_std": 0.19259289279580116,
      "rewards/cosine_scaled_reward": -0.24130774475634098,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3403.0625610351562,
      "epoch": 0.037142857142857144,
      "grad_norm": 0.1934821903705597,
      "kl": 0.00086212158203125,
      "learning_rate": 9.975348529157229e-07,
      "loss": 0.0185,
      "num_tokens": 9092067.0,
      "reward": -0.46268967539072037,
      "reward_std": 0.28929166309535503,
      "rewards/cosine_scaled_reward": -0.23134482093155384,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2666.395965576172,
      "epoch": 0.037714285714285714,
      "grad_norm": 0.24617139995098114,
      "kl": 0.00070953369140625,
      "learning_rate": 9.971955636222684e-07,
      "loss": -0.0574,
      "num_tokens": 9225802.0,
      "reward": 0.30002279952168465,
      "reward_std": 0.6946442574262619,
      "rewards/cosine_scaled_reward": 0.1500114006921649,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2073.750030517578,
      "epoch": 0.038285714285714284,
      "grad_norm": 0.4215092062950134,
      "kl": 0.0011243820190429688,
      "learning_rate": 9.968344786479415e-07,
      "loss": 0.0529,
      "num_tokens": 9330898.0,
      "reward": -0.09548089653253555,
      "reward_std": 0.8580914586782455,
      "rewards/cosine_scaled_reward": -0.04774044919759035,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3528.1666870117188,
      "epoch": 0.038857142857142854,
      "grad_norm": 0.20039838552474976,
      "kl": 0.0005893707275390625,
      "learning_rate": 9.964516155915151e-07,
      "loss": -0.0095,
      "num_tokens": 9506442.0,
      "reward": -0.42071669083088636,
      "reward_std": 0.42319002375006676,
      "rewards/cosine_scaled_reward": -0.21035834541544318,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2716.7291870117188,
      "epoch": 0.03942857142857143,
      "grad_norm": 0.2517124116420746,
      "kl": 0.0007944107055664062,
      "learning_rate": 9.960469931131936e-07,
      "loss": -0.1279,
      "num_tokens": 9643061.0,
      "reward": -0.3758072182536125,
      "reward_std": 0.45701417699456215,
      "rewards/cosine_scaled_reward": -0.18790359422564507,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2648.0000228881836,
      "epoch": 0.04,
      "grad_norm": 0.29658740758895874,
      "kl": 0.0008521080017089844,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0124,
      "num_tokens": 9776189.0,
      "reward": -0.16740068793296814,
      "reward_std": 0.41449040174484253,
      "rewards/cosine_scaled_reward": -0.08370032906532288,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1873.812515258789,
      "epoch": 0.04057142857142857,
      "grad_norm": 0.3751354515552521,
      "kl": 0.0007829666137695312,
      "learning_rate": 9.951725498333448e-07,
      "loss": 0.0863,
      "num_tokens": 9871712.0,
      "reward": 0.3852356970310211,
      "reward_std": 0.5030505172908306,
      "rewards/cosine_scaled_reward": 0.19261783733963966,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2817.8334197998047,
      "epoch": 0.04114285714285714,
      "grad_norm": 0.27796244621276855,
      "kl": 0.000804901123046875,
      "learning_rate": 9.947027716509488e-07,
      "loss": 0.0581,
      "num_tokens": 10013232.0,
      "reward": -0.10409137606620789,
      "reward_std": 0.20197268202900887,
      "rewards/cosine_scaled_reward": -0.05204569548368454,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2128.0208740234375,
      "epoch": 0.04171428571428572,
      "grad_norm": 0.33463340997695923,
      "kl": 0.0006952285766601562,
      "learning_rate": 9.942113192828444e-07,
      "loss": 0.0541,
      "num_tokens": 10120693.0,
      "reward": -0.22259100899100304,
      "reward_std": 0.4416811428964138,
      "rewards/cosine_scaled_reward": -0.11129548959434032,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2830.6666717529297,
      "epoch": 0.04228571428571429,
      "grad_norm": 0.28111714124679565,
      "kl": 0.0005383491516113281,
      "learning_rate": 9.93698216681727e-07,
      "loss": 0.0078,
      "num_tokens": 10262025.0,
      "reward": -0.14538022875785828,
      "reward_std": 0.5454810187220573,
      "rewards/cosine_scaled_reward": -0.07269011810421944,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.2708435058594,
      "epoch": 0.04285714285714286,
      "grad_norm": 0.4327227771282196,
      "kl": 0.0011434555053710938,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0645,
      "num_tokens": 10341778.0,
      "reward": -0.28616778552532196,
      "reward_std": 0.6919333338737488,
      "rewards/cosine_scaled_reward": -0.1430838778614998,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3283.2083740234375,
      "epoch": 0.04342857142857143,
      "grad_norm": 0.2106700837612152,
      "kl": 0.0007944107055664062,
      "learning_rate": 9.926071618660237e-07,
      "loss": 0.0367,
      "num_tokens": 10504976.0,
      "reward": -0.337122593075037,
      "reward_std": 0.5403655767440796,
      "rewards/cosine_scaled_reward": -0.1685612890869379,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2804.854248046875,
      "epoch": 0.044,
      "grad_norm": 0.290350079536438,
      "kl": 0.0010528564453125,
      "learning_rate": 9.9202926282791e-07,
      "loss": -0.1465,
      "num_tokens": 10645033.0,
      "reward": -0.3672878537327051,
      "reward_std": 0.4394787736237049,
      "rewards/cosine_scaled_reward": -0.18364392640069127,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2761.9375610351562,
      "epoch": 0.044571428571428574,
      "grad_norm": 0.23817922174930573,
      "kl": 0.0006437301635742188,
      "learning_rate": 9.91429819907136e-07,
      "loss": 0.0617,
      "num_tokens": 10783558.0,
      "reward": 0.039801888167858124,
      "reward_std": 0.8721425756812096,
      "rewards/cosine_scaled_reward": 0.019900942221283913,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2478.0000610351562,
      "epoch": 0.045142857142857144,
      "grad_norm": 0.2438346892595291,
      "kl": 0.0007276535034179688,
      "learning_rate": 9.908088623197048e-07,
      "loss": -0.0649,
      "num_tokens": 10908622.0,
      "reward": -0.009920487180352211,
      "reward_std": 0.6310172341763973,
      "rewards/cosine_scaled_reward": -0.00496023939922452,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3292.5208740234375,
      "epoch": 0.045714285714285714,
      "grad_norm": 0.19675898551940918,
      "kl": 0.0007238388061523438,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.028,
      "num_tokens": 11073503.0,
      "reward": -0.26976824924349785,
      "reward_std": 0.44561611115932465,
      "rewards/cosine_scaled_reward": -0.13488411717116833,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2928.7500610351562,
      "epoch": 0.046285714285714284,
      "grad_norm": 0.23351161181926727,
      "kl": 0.0010251998901367188,
      "learning_rate": 9.895025252503755e-07,
      "loss": -0.0533,
      "num_tokens": 11219663.0,
      "reward": 0.14206518977880478,
      "reward_std": 0.47988787665963173,
      "rewards/cosine_scaled_reward": 0.07103258091956377,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3216.3958740234375,
      "epoch": 0.046857142857142854,
      "grad_norm": 0.22799547016620636,
      "kl": 0.0006895065307617188,
      "learning_rate": 9.888172094375033e-07,
      "loss": 0.0032,
      "num_tokens": 11380386.0,
      "reward": -0.16227489709854126,
      "reward_std": 0.33015402406454086,
      "rewards/cosine_scaled_reward": -0.08113745599985123,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 993.4583435058594,
      "epoch": 0.04742857142857143,
      "grad_norm": 0.551730751991272,
      "kl": 0.0010633468627929688,
      "learning_rate": 9.881105062929221e-07,
      "loss": 0.1338,
      "num_tokens": 11432752.0,
      "reward": 0.6598471999168396,
      "reward_std": 0.16890107188373804,
      "rewards/cosine_scaled_reward": 0.3299236036837101,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.7083740234375,
      "epoch": 0.048,
      "grad_norm": 0.29992803931236267,
      "kl": 0.0008602142333984375,
      "learning_rate": 9.873824502603459e-07,
      "loss": 0.0295,
      "num_tokens": 11527454.0,
      "reward": -0.3242928695399314,
      "reward_std": 0.7351822834461927,
      "rewards/cosine_scaled_reward": -0.16214643290732056,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2737.2083740234375,
      "epoch": 0.04857142857142857,
      "grad_norm": 0.31860730051994324,
      "kl": 0.0012292861938476562,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.0168,
      "num_tokens": 11664108.0,
      "reward": -0.3406180441379547,
      "reward_std": 0.4295322969555855,
      "rewards/cosine_scaled_reward": -0.17030901461839676,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2586.541732788086,
      "epoch": 0.04914285714285714,
      "grad_norm": 0.3602101802825928,
      "kl": 0.000606536865234375,
      "learning_rate": 9.85862422507884e-07,
      "loss": -0.1276,
      "num_tokens": 11794454.0,
      "reward": 0.20340422540903091,
      "reward_std": 0.9888063967227936,
      "rewards/cosine_scaled_reward": 0.10170210711658001,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2964.75,
      "epoch": 0.04971428571428571,
      "grad_norm": 0.3094576895236969,
      "kl": 0.00102996826171875,
      "learning_rate": 9.850705248720068e-07,
      "loss": -0.0531,
      "num_tokens": 11943002.0,
      "reward": 0.01601184532046318,
      "reward_std": 0.4094504490494728,
      "rewards/cosine_scaled_reward": 0.008005908690392971,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2426.1666870117188,
      "epoch": 0.05028571428571429,
      "grad_norm": 0.2704874277114868,
      "kl": 0.001644134521484375,
      "learning_rate": 9.8425742251254e-07,
      "loss": 0.0654,
      "num_tokens": 12065278.0,
      "reward": -0.40290449309395626,
      "reward_std": 0.6259909272193909,
      "rewards/cosine_scaled_reward": -0.20145224656153005,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2142.1041870117188,
      "epoch": 0.05085714285714286,
      "grad_norm": 0.3306039273738861,
      "kl": 0.00197601318359375,
      "learning_rate": 9.83423155058946e-07,
      "loss": -0.1126,
      "num_tokens": 12174291.0,
      "reward": -0.23806674778461456,
      "reward_std": 0.6727369725704193,
      "rewards/cosine_scaled_reward": -0.11903337389230728,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2580.7709350585938,
      "epoch": 0.05142857142857143,
      "grad_norm": 0.23774664103984833,
      "kl": 0.0007190704345703125,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.077,
      "num_tokens": 12303664.0,
      "reward": 0.3880194779485464,
      "reward_std": 1.0538864731788635,
      "rewards/cosine_scaled_reward": 0.1940097352489829,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2841.312530517578,
      "epoch": 0.052,
      "grad_norm": 0.24611423909664154,
      "kl": 0.000865936279296875,
      "learning_rate": 9.816912885430258e-07,
      "loss": 0.0599,
      "num_tokens": 12445867.0,
      "reward": -0.19608542323112488,
      "reward_std": 0.42760632932186127,
      "rewards/cosine_scaled_reward": -0.09804270416498184,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2743.6041717529297,
      "epoch": 0.052571428571428575,
      "grad_norm": 0.23887218534946442,
      "kl": 0.0006628036499023438,
      "learning_rate": 9.807937738894303e-07,
      "loss": 0.0646,
      "num_tokens": 12584352.0,
      "reward": 0.15040923655033112,
      "reward_std": 0.25195283722132444,
      "rewards/cosine_scaled_reward": 0.07520462200045586,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2130.0833435058594,
      "epoch": 0.053142857142857144,
      "grad_norm": 0.42298954725265503,
      "kl": 0.0020599365234375,
      "learning_rate": 9.798752629550546e-07,
      "loss": 0.0779,
      "num_tokens": 12692224.0,
      "reward": -0.06379163265228271,
      "reward_std": 0.3432878144085407,
      "rewards/cosine_scaled_reward": -0.03189583122730255,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2588.2709350585938,
      "epoch": 0.053714285714285714,
      "grad_norm": 0.21005931496620178,
      "kl": 0.0005979537963867188,
      "learning_rate": 9.78935800506826e-07,
      "loss": 0.0561,
      "num_tokens": 12822065.0,
      "reward": -0.2485465258359909,
      "reward_std": 0.6237562894821167,
      "rewards/cosine_scaled_reward": -0.12427325546741486,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.5417175292969,
      "epoch": 0.054285714285714284,
      "grad_norm": 0.34288352727890015,
      "kl": 0.0010595321655273438,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.0743,
      "num_tokens": 12899239.0,
      "reward": 0.16759111359715462,
      "reward_std": 0.9945086091756821,
      "rewards/cosine_scaled_reward": 0.08379554376006126,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2968.5000610351562,
      "epoch": 0.054857142857142854,
      "grad_norm": 0.20499330759048462,
      "kl": 0.0007104873657226562,
      "learning_rate": 9.769942052400235e-07,
      "loss": 0.0123,
      "num_tokens": 13047499.0,
      "reward": -0.08611700683832169,
      "reward_std": 0.7486424595117569,
      "rewards/cosine_scaled_reward": -0.04305850435048342,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2057.5625228881836,
      "epoch": 0.05542857142857143,
      "grad_norm": 0.3967874348163605,
      "kl": 0.00153350830078125,
      "learning_rate": 9.759921670520634e-07,
      "loss": 0.0677,
      "num_tokens": 13152466.0,
      "reward": -0.5269366651773453,
      "reward_std": 0.3624592386186123,
      "rewards/cosine_scaled_reward": -0.26346831768751144,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1952.1041717529297,
      "epoch": 0.056,
      "grad_norm": 0.3613995909690857,
      "kl": 0.0018596649169921875,
      "learning_rate": 9.749693666068663e-07,
      "loss": 0.0947,
      "num_tokens": 13251459.0,
      "reward": 0.9773776829242706,
      "reward_std": 0.8529446795582771,
      "rewards/cosine_scaled_reward": 0.4886888340115547,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2277.0416717529297,
      "epoch": 0.05657142857142857,
      "grad_norm": 0.3505290150642395,
      "kl": 0.0008687973022460938,
      "learning_rate": 9.739258537542835e-07,
      "loss": 0.0603,
      "num_tokens": 13366469.0,
      "reward": 0.27346290089190006,
      "reward_std": 0.47108355164527893,
      "rewards/cosine_scaled_reward": 0.13673143601045012,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.0625305175781,
      "epoch": 0.05714285714285714,
      "grad_norm": 0.36085861921310425,
      "kl": 0.0011138916015625,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0027,
      "num_tokens": 13446920.0,
      "reward": 0.4378054551780224,
      "reward_std": 0.6341788824647665,
      "rewards/cosine_scaled_reward": 0.2189027275890112,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2714.8541717529297,
      "epoch": 0.05771428571428571,
      "grad_norm": 0.2826038599014282,
      "kl": 0.0008282661437988281,
      "learning_rate": 9.717768952713511e-07,
      "loss": 0.0606,
      "num_tokens": 13583809.0,
      "reward": -0.5446555614471436,
      "reward_std": 0.35200726985931396,
      "rewards/cosine_scaled_reward": -0.2723277732729912,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2409.666717529297,
      "epoch": 0.05828571428571429,
      "grad_norm": 0.30025118589401245,
      "kl": 0.0009298324584960938,
      "learning_rate": 9.706715543782064e-07,
      "loss": -0.028,
      "num_tokens": 13705845.0,
      "reward": -0.1350867822766304,
      "reward_std": 0.3358248174190521,
      "rewards/cosine_scaled_reward": -0.0675433836877346,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2799.2708587646484,
      "epoch": 0.05885714285714286,
      "grad_norm": 0.2541813552379608,
      "kl": 0.0008907318115234375,
      "learning_rate": 9.695457105469804e-07,
      "loss": 0.0025,
      "num_tokens": 13846618.0,
      "reward": -0.6046699732542038,
      "reward_std": 0.3693853598088026,
      "rewards/cosine_scaled_reward": -0.3023349717259407,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2963.1458435058594,
      "epoch": 0.05942857142857143,
      "grad_norm": 0.2182713747024536,
      "kl": 0.000762939453125,
      "learning_rate": 9.683994186497132e-07,
      "loss": 0.0303,
      "num_tokens": 13995953.0,
      "reward": -0.20170933986082673,
      "reward_std": 0.5568380877375603,
      "rewards/cosine_scaled_reward": -0.10085467004682869,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2755.1458740234375,
      "epoch": 0.06,
      "grad_norm": 0.26797640323638916,
      "kl": 0.0011749267578125,
      "learning_rate": 9.672327345550543e-07,
      "loss": -0.0094,
      "num_tokens": 14134524.0,
      "reward": -0.6127043217420578,
      "reward_std": 0.19923977181315422,
      "rewards/cosine_scaled_reward": -0.3063521459698677,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2781.0416717529297,
      "epoch": 0.060571428571428575,
      "grad_norm": 0.4417632222175598,
      "kl": 0.0021905899047851562,
      "learning_rate": 9.66045715125541e-07,
      "loss": 0.0892,
      "num_tokens": 14274434.0,
      "reward": -0.042052820324897766,
      "reward_std": 0.555817510932684,
      "rewards/cosine_scaled_reward": -0.021026406437158585,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2453.125030517578,
      "epoch": 0.061142857142857145,
      "grad_norm": 0.2828550338745117,
      "kl": 0.000919342041015625,
      "learning_rate": 9.648384182148252e-07,
      "loss": -0.0085,
      "num_tokens": 14397428.0,
      "reward": -0.37871552258729935,
      "reward_std": 0.4371586740016937,
      "rewards/cosine_scaled_reward": -0.18935775943100452,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2142.5625,
      "epoch": 0.061714285714285715,
      "grad_norm": 0.30579039454460144,
      "kl": 0.0007009506225585938,
      "learning_rate": 9.636109026648554e-07,
      "loss": 0.0216,
      "num_tokens": 14506031.0,
      "reward": -0.2899063751101494,
      "reward_std": 0.30772218108177185,
      "rewards/cosine_scaled_reward": -0.1449531875550747,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3271.8750610351562,
      "epoch": 0.062285714285714285,
      "grad_norm": 0.212229385972023,
      "kl": 0.0011157989501953125,
      "learning_rate": 9.623632283030077e-07,
      "loss": -0.035,
      "num_tokens": 14669513.0,
      "reward": -0.5292222313582897,
      "reward_std": 0.40466723032295704,
      "rewards/cosine_scaled_reward": -0.26461111195385456,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2759.3125,
      "epoch": 0.06285714285714286,
      "grad_norm": 0.25697484612464905,
      "kl": 0.0011186599731445312,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0172,
      "num_tokens": 14807984.0,
      "reward": -0.4681231379508972,
      "reward_std": 0.29983806796371937,
      "rewards/cosine_scaled_reward": -0.23406155407428741,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2785.4166717529297,
      "epoch": 0.06342857142857143,
      "grad_norm": 0.47920992970466614,
      "kl": 0.009280204772949219,
      "learning_rate": 9.598076473627796e-07,
      "loss": 0.0485,
      "num_tokens": 14949280.0,
      "reward": 0.15108218044042587,
      "reward_std": 0.20054534077644348,
      "rewards/cosine_scaled_reward": 0.07554109394550323,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2506.416748046875,
      "epoch": 0.064,
      "grad_norm": 0.26250529289245605,
      "kl": 0.0015287399291992188,
      "learning_rate": 9.58499865339809e-07,
      "loss": -0.1711,
      "num_tokens": 15075480.0,
      "reward": -0.23084469139575958,
      "reward_std": 0.5984194576740265,
      "rewards/cosine_scaled_reward": -0.11542234569787979,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2602.5208740234375,
      "epoch": 0.06457142857142857,
      "grad_norm": 0.424373596906662,
      "kl": 0.0057659149169921875,
      "learning_rate": 9.571721736097088e-07,
      "loss": 0.0718,
      "num_tokens": 15206593.0,
      "reward": -0.253767779096961,
      "reward_std": 0.4538792669773102,
      "rewards/cosine_scaled_reward": -0.12688388722017407,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3077.0000610351562,
      "epoch": 0.06514285714285714,
      "grad_norm": 0.2078198343515396,
      "kl": 0.0006704330444335938,
      "learning_rate": 9.55824636882301e-07,
      "loss": -0.0122,
      "num_tokens": 15360649.0,
      "reward": -0.15475063771009445,
      "reward_std": 0.5910014770925045,
      "rewards/cosine_scaled_reward": -0.07737531885504723,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2643.6875610351562,
      "epoch": 0.06571428571428571,
      "grad_norm": 0.28771933913230896,
      "kl": 0.0011758804321289062,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.0676,
      "num_tokens": 15493570.0,
      "reward": -0.19310491532087326,
      "reward_std": 0.5526712536811829,
      "rewards/cosine_scaled_reward": -0.09655245952308178,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3344.7291870117188,
      "epoch": 0.06628571428571428,
      "grad_norm": 0.2425944209098816,
      "kl": 0.0008716583251953125,
      "learning_rate": 9.530702921077358e-07,
      "loss": 0.0343,
      "num_tokens": 15661161.0,
      "reward": -0.7032309919595718,
      "reward_std": 0.3479248844087124,
      "rewards/cosine_scaled_reward": -0.351615484803915,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3084.4583740234375,
      "epoch": 0.06685714285714285,
      "grad_norm": 0.20971350371837616,
      "kl": 0.0007829666137695312,
      "learning_rate": 9.516636183034564e-07,
      "loss": -0.0389,
      "num_tokens": 15815755.0,
      "reward": 0.0003270097076892853,
      "reward_std": 0.43772435188293457,
      "rewards/cosine_scaled_reward": 0.00016349367797374725,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2729.312530517578,
      "epoch": 0.06742857142857143,
      "grad_norm": 0.2613621950149536,
      "kl": 0.001468658447265625,
      "learning_rate": 9.502373679810839e-07,
      "loss": 0.0463,
      "num_tokens": 15952474.0,
      "reward": -0.06591695547103882,
      "reward_std": 0.39777015522122383,
      "rewards/cosine_scaled_reward": -0.032958466559648514,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2081.7916717529297,
      "epoch": 0.068,
      "grad_norm": 0.4527161717414856,
      "kl": 0.0013675689697265625,
      "learning_rate": 9.487916106540465e-07,
      "loss": 0.0989,
      "num_tokens": 16059348.0,
      "reward": 0.35896405577659607,
      "reward_std": 0.3922106046229601,
      "rewards/cosine_scaled_reward": 0.17948202788829803,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2978.125,
      "epoch": 0.06857142857142857,
      "grad_norm": 0.2305285632610321,
      "kl": 0.0010137557983398438,
      "learning_rate": 9.473264167865171e-07,
      "loss": -0.0725,
      "num_tokens": 16208058.0,
      "reward": -0.6749820820987225,
      "reward_std": 0.33936042711138725,
      "rewards/cosine_scaled_reward": -0.33749102614820004,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2772.125,
      "epoch": 0.06914285714285714,
      "grad_norm": 0.2650047242641449,
      "kl": 0.0009822845458984375,
      "learning_rate": 9.458418577899774e-07,
      "loss": -0.0085,
      "num_tokens": 16347408.0,
      "reward": -0.032290175557136536,
      "reward_std": 0.4409598559141159,
      "rewards/cosine_scaled_reward": -0.016145076602697372,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2094.479217529297,
      "epoch": 0.06971428571428571,
      "grad_norm": 0.24298027157783508,
      "kl": 0.0012693405151367188,
      "learning_rate": 9.443380060197385e-07,
      "loss": -0.0886,
      "num_tokens": 16453067.0,
      "reward": 0.5003323024138808,
      "reward_std": 0.5531654357910156,
      "rewards/cosine_scaled_reward": 0.2501661437563598,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1995.3958587646484,
      "epoch": 0.07028571428571428,
      "grad_norm": 0.2879078686237335,
      "kl": 0.001132965087890625,
      "learning_rate": 9.428149347714143e-07,
      "loss": 0.069,
      "num_tokens": 16554006.0,
      "reward": -0.1732923611998558,
      "reward_std": 0.6025716587901115,
      "rewards/cosine_scaled_reward": -0.08664617873728275,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3321.0625,
      "epoch": 0.07085714285714285,
      "grad_norm": 0.18874195218086243,
      "kl": 0.0006341934204101562,
      "learning_rate": 9.412727182773486e-07,
      "loss": 0.0448,
      "num_tokens": 16720257.0,
      "reward": 0.0892313290387392,
      "reward_std": 0.36883802339434624,
      "rewards/cosine_scaled_reward": 0.0446156719699502,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1953.6458740234375,
      "epoch": 0.07142857142857142,
      "grad_norm": 0.34396979212760925,
      "kl": 0.0025730133056640625,
      "learning_rate": 9.397114317029974e-07,
      "loss": -0.0302,
      "num_tokens": 16819864.0,
      "reward": 0.04853908717632294,
      "reward_std": 0.6620666459202766,
      "rewards/cosine_scaled_reward": 0.024269558489322662,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1393.2708435058594,
      "epoch": 0.072,
      "grad_norm": 0.5013756155967712,
      "kl": 0.011034011840820312,
      "learning_rate": 9.381311511432658e-07,
      "loss": 0.1767,
      "num_tokens": 16891889.0,
      "reward": -0.013652913272380829,
      "reward_std": 0.3805545046925545,
      "rewards/cosine_scaled_reward": -0.006826456636190414,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2377.8958435058594,
      "epoch": 0.07257142857142856,
      "grad_norm": 0.22264918684959412,
      "kl": 0.0008573532104492188,
      "learning_rate": 9.36531953618799e-07,
      "loss": 0.0667,
      "num_tokens": 17012724.0,
      "reward": 0.2236809842288494,
      "reward_std": 0.4961891621351242,
      "rewards/cosine_scaled_reward": 0.11184047814458609,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2564.7500610351562,
      "epoch": 0.07314285714285715,
      "grad_norm": 0.306539386510849,
      "kl": 0.0009932518005371094,
      "learning_rate": 9.34913917072228e-07,
      "loss": -0.0337,
      "num_tokens": 17141448.0,
      "reward": -0.004740983247756958,
      "reward_std": 0.3173178732395172,
      "rewards/cosine_scaled_reward": -0.002370491623878479,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2082.166717529297,
      "epoch": 0.07371428571428572,
      "grad_norm": 0.3070489764213562,
      "kl": 0.0011272430419921875,
      "learning_rate": 9.332771203643714e-07,
      "loss": -0.0082,
      "num_tokens": 17247344.0,
      "reward": -0.18208786100149155,
      "reward_std": 0.5415353253483772,
      "rewards/cosine_scaled_reward": -0.09104393050074577,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.7291717529297,
      "epoch": 0.07428571428571429,
      "grad_norm": 0.3711329698562622,
      "kl": 0.0028362274169921875,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.008,
      "num_tokens": 17330167.0,
      "reward": -0.04559193179011345,
      "reward_std": 0.6725184172391891,
      "rewards/cosine_scaled_reward": -0.022795964032411575,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2907.8333587646484,
      "epoch": 0.07485714285714286,
      "grad_norm": 0.22048690915107727,
      "kl": 0.0009622573852539062,
      "learning_rate": 9.299475664759068e-07,
      "loss": 0.0646,
      "num_tokens": 17475575.0,
      "reward": -0.6151376739144325,
      "reward_std": 0.3845426104962826,
      "rewards/cosine_scaled_reward": -0.30756882205605507,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2430.7083740234375,
      "epoch": 0.07542857142857143,
      "grad_norm": 0.22665299475193024,
      "kl": 0.0009012222290039062,
      "learning_rate": 9.282549715730579e-07,
      "loss": 0.0695,
      "num_tokens": 17598141.0,
      "reward": -0.2976334486156702,
      "reward_std": 0.5366611061617732,
      "rewards/cosine_scaled_reward": -0.14881671639159322,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2548.6458435058594,
      "epoch": 0.076,
      "grad_norm": 0.3216173052787781,
      "kl": 0.0010004043579101562,
      "learning_rate": 9.265439410565328e-07,
      "loss": 0.0916,
      "num_tokens": 17726752.0,
      "reward": 0.17475611716508865,
      "reward_std": 0.47678545862436295,
      "rewards/cosine_scaled_reward": 0.08737805113196373,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3280.7500610351562,
      "epoch": 0.07657142857142857,
      "grad_norm": 0.17866647243499756,
      "kl": 0.0007352828979492188,
      "learning_rate": 9.248145583195447e-07,
      "loss": -0.003,
      "num_tokens": 17890468.0,
      "reward": 0.4249248839914799,
      "reward_std": 0.7319479957222939,
      "rewards/cosine_scaled_reward": 0.21246242709457874,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2719.062530517578,
      "epoch": 0.07714285714285714,
      "grad_norm": 0.34077975153923035,
      "kl": 0.0010766983032226562,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.0249,
      "num_tokens": 18027487.0,
      "reward": -0.6892034411430359,
      "reward_std": 0.43735230527818203,
      "rewards/cosine_scaled_reward": -0.34460172057151794,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.0833587646484,
      "epoch": 0.07771428571428571,
      "grad_norm": 0.5860525965690613,
      "kl": 0.0029449462890625,
      "learning_rate": 9.213010742252327e-07,
      "loss": -0.1361,
      "num_tokens": 18108455.0,
      "reward": -0.3233466073870659,
      "reward_std": 0.4398806467652321,
      "rewards/cosine_scaled_reward": -0.1616733018308878,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2537.8333435058594,
      "epoch": 0.07828571428571429,
      "grad_norm": 0.21415218710899353,
      "kl": 0.000827789306640625,
      "learning_rate": 9.195171441101668e-07,
      "loss": -0.0168,
      "num_tokens": 18236475.0,
      "reward": -0.059672433882951736,
      "reward_std": 0.7180322706699371,
      "rewards/cosine_scaled_reward": -0.02983621321618557,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2120.3750076293945,
      "epoch": 0.07885714285714286,
      "grad_norm": 0.29080891609191895,
      "kl": 0.0012989044189453125,
      "learning_rate": 9.177152042508077e-07,
      "loss": 0.0494,
      "num_tokens": 18343401.0,
      "reward": -0.08782655745744705,
      "reward_std": 0.12043035682290792,
      "rewards/cosine_scaled_reward": -0.04391326941549778,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2405.2291870117188,
      "epoch": 0.07942857142857143,
      "grad_norm": 0.22457921504974365,
      "kl": 0.0009593963623046875,
      "learning_rate": 9.158953424711624e-07,
      "loss": 0.0361,
      "num_tokens": 18464648.0,
      "reward": -0.41086670011281967,
      "reward_std": 0.4968971386551857,
      "rewards/cosine_scaled_reward": -0.20543334260582924,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2780.2291717529297,
      "epoch": 0.08,
      "grad_norm": 0.2105841189622879,
      "kl": 0.0010890960693359375,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0146,
      "num_tokens": 18604483.0,
      "reward": 0.13771556317806244,
      "reward_std": 0.30897051841020584,
      "rewards/cosine_scaled_reward": 0.06885778903961182,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.2708740234375,
      "epoch": 0.08057142857142857,
      "grad_norm": 0.3650009334087372,
      "kl": 0.002017974853515625,
      "learning_rate": 9.122022088101613e-07,
      "loss": 0.0459,
      "num_tokens": 18705872.0,
      "reward": -0.379656158387661,
      "reward_std": 0.5402833297848701,
      "rewards/cosine_scaled_reward": -0.1898280642926693,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2488.7708740234375,
      "epoch": 0.08114285714285714,
      "grad_norm": 0.2864922285079956,
      "kl": 0.001262664794921875,
      "learning_rate": 9.103291169269299e-07,
      "loss": -0.0564,
      "num_tokens": 18831261.0,
      "reward": 0.011970575898885727,
      "reward_std": 0.7028900012373924,
      "rewards/cosine_scaled_reward": 0.0059852879494428635,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2584.7084350585938,
      "epoch": 0.08171428571428571,
      "grad_norm": 0.3126732409000397,
      "kl": 0.0011653900146484375,
      "learning_rate": 9.084384631108882e-07,
      "loss": 0.0683,
      "num_tokens": 18961471.0,
      "reward": 0.24577251449227333,
      "reward_std": 0.6674134684726596,
      "rewards/cosine_scaled_reward": 0.12288625724613667,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2151.1041870117188,
      "epoch": 0.08228571428571428,
      "grad_norm": 0.37290921807289124,
      "kl": 0.0016574859619140625,
      "learning_rate": 9.065303395098358e-07,
      "loss": 0.0363,
      "num_tokens": 19070388.0,
      "reward": -0.1530948244035244,
      "reward_std": 0.6015914604067802,
      "rewards/cosine_scaled_reward": -0.076547397300601,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1968.2084197998047,
      "epoch": 0.08285714285714285,
      "grad_norm": 0.35119813680648804,
      "kl": 0.002681732177734375,
      "learning_rate": 9.046048391230247e-07,
      "loss": -0.0567,
      "num_tokens": 19169698.0,
      "reward": -0.20278839766979218,
      "reward_std": 0.47369804978370667,
      "rewards/cosine_scaled_reward": -0.10139419510960579,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.3125457763672,
      "epoch": 0.08342857142857144,
      "grad_norm": 0.3501497507095337,
      "kl": 0.0024051666259765625,
      "learning_rate": 9.026620557966279e-07,
      "loss": 0.009,
      "num_tokens": 19253065.0,
      "reward": 0.25620073080062866,
      "reward_std": 0.8665501922369003,
      "rewards/cosine_scaled_reward": 0.12810037285089493,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.5417175292969,
      "epoch": 0.084,
      "grad_norm": 0.29272228479385376,
      "kl": 0.0031538009643554688,
      "learning_rate": 9.007020842191634e-07,
      "loss": -0.0467,
      "num_tokens": 19356735.0,
      "reward": 0.015910595655441284,
      "reward_std": 0.7093758508563042,
      "rewards/cosine_scaled_reward": 0.007955307140946388,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.2500305175781,
      "epoch": 0.08457142857142858,
      "grad_norm": 0.36988532543182373,
      "kl": 0.002410888671875,
      "learning_rate": 8.987250199168808e-07,
      "loss": 0.0963,
      "num_tokens": 19444131.0,
      "reward": -0.25489600747823715,
      "reward_std": 0.4693850204348564,
      "rewards/cosine_scaled_reward": -0.12744800373911858,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.7917022705078,
      "epoch": 0.08514285714285715,
      "grad_norm": 0.3559054434299469,
      "kl": 0.00274658203125,
      "learning_rate": 8.967309592491052e-07,
      "loss": 0.0878,
      "num_tokens": 19519601.0,
      "reward": -0.10238776355981827,
      "reward_std": 0.636756157502532,
      "rewards/cosine_scaled_reward": -0.05119386687874794,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2705.2500228881836,
      "epoch": 0.08571428571428572,
      "grad_norm": 0.3573072850704193,
      "kl": 0.0050811767578125,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0188,
      "num_tokens": 19655657.0,
      "reward": -0.20403114520013332,
      "reward_std": 0.5940973423421383,
      "rewards/cosine_scaled_reward": -0.10201557632535696,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2180.812545776367,
      "epoch": 0.08628571428571429,
      "grad_norm": 0.2828698456287384,
      "kl": 0.0015001296997070312,
      "learning_rate": 8.926922383915315e-07,
      "loss": -0.0244,
      "num_tokens": 19766528.0,
      "reward": 0.11930293589830399,
      "reward_std": 0.6097396910190582,
      "rewards/cosine_scaled_reward": 0.05965147539973259,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2904.4166870117188,
      "epoch": 0.08685714285714285,
      "grad_norm": 0.26096129417419434,
      "kl": 0.00130462646484375,
      "learning_rate": 8.906477750432903e-07,
      "loss": 0.1732,
      "num_tokens": 19912480.0,
      "reward": -0.3498671278357506,
      "reward_std": 0.394152645021677,
      "rewards/cosine_scaled_reward": -0.1749335676431656,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2408.5208740234375,
      "epoch": 0.08742857142857142,
      "grad_norm": 0.35171744227409363,
      "kl": 0.0016956329345703125,
      "learning_rate": 8.88586709003076e-07,
      "loss": 0.1131,
      "num_tokens": 20034233.0,
      "reward": 0.07599013298749924,
      "reward_std": 0.41721872985363007,
      "rewards/cosine_scaled_reward": 0.037995072081685066,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2731.3750610351562,
      "epoch": 0.088,
      "grad_norm": 0.26374199986457825,
      "kl": 0.001861572265625,
      "learning_rate": 8.865091407243394e-07,
      "loss": -0.0558,
      "num_tokens": 20171267.0,
      "reward": -0.2839723117649555,
      "reward_std": 0.6639672666788101,
      "rewards/cosine_scaled_reward": -0.14198614470660686,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2960.8958435058594,
      "epoch": 0.08857142857142856,
      "grad_norm": 0.27562668919563293,
      "kl": 0.0019378662109375,
      "learning_rate": 8.844151714648274e-07,
      "loss": -0.0435,
      "num_tokens": 20318922.0,
      "reward": -0.46872561052441597,
      "reward_std": 0.42786915227770805,
      "rewards/cosine_scaled_reward": -0.2343627940863371,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.7917404174805,
      "epoch": 0.08914285714285715,
      "grad_norm": 0.499097615480423,
      "kl": 0.005889892578125,
      "learning_rate": 8.823049032816478e-07,
      "loss": -0.0042,
      "num_tokens": 20400236.0,
      "reward": 0.6059545688331127,
      "reward_std": 0.690888412296772,
      "rewards/cosine_scaled_reward": 0.3029772713780403,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2041.2708435058594,
      "epoch": 0.08971428571428572,
      "grad_norm": 0.4625336229801178,
      "kl": 0.005961418151855469,
      "learning_rate": 8.801784390262943e-07,
      "loss": 0.1281,
      "num_tokens": 20504961.0,
      "reward": 0.08923859149217606,
      "reward_std": 0.16652610152959824,
      "rewards/cosine_scaled_reward": 0.04461930692195892,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2572.312530517578,
      "epoch": 0.09028571428571429,
      "grad_norm": 0.25713205337524414,
      "kl": 0.0018138885498046875,
      "learning_rate": 8.780358823396352e-07,
      "loss": 0.0261,
      "num_tokens": 20635176.0,
      "reward": -0.24785784073174,
      "reward_std": 0.6590173244476318,
      "rewards/cosine_scaled_reward": -0.1239289166405797,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3366.6875,
      "epoch": 0.09085714285714286,
      "grad_norm": 0.21494215726852417,
      "kl": 0.00112152099609375,
      "learning_rate": 8.758773376468604e-07,
      "loss": -0.0031,
      "num_tokens": 20803401.0,
      "reward": -0.47726341150701046,
      "reward_std": 0.34933631122112274,
      "rewards/cosine_scaled_reward": -0.23863169085234404,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3498.0208740234375,
      "epoch": 0.09142857142857143,
      "grad_norm": 0.19158728420734406,
      "kl": 0.0010175704956054688,
      "learning_rate": 8.737029101523929e-07,
      "loss": -0.0137,
      "num_tokens": 20977630.0,
      "reward": -0.23914687521755695,
      "reward_std": 0.37878062576055527,
      "rewards/cosine_scaled_reward": -0.11957343760877848,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2862.8959350585938,
      "epoch": 0.092,
      "grad_norm": 0.2647199034690857,
      "kl": 0.002674102783203125,
      "learning_rate": 8.715127058347614e-07,
      "loss": 0.0488,
      "num_tokens": 21120425.0,
      "reward": -0.28584553534165025,
      "reward_std": 0.7033149749040604,
      "rewards/cosine_scaled_reward": -0.14292275649495423,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2344.0208740234375,
      "epoch": 0.09257142857142857,
      "grad_norm": 0.27368515729904175,
      "kl": 0.0023193359375,
      "learning_rate": 8.693068314414344e-07,
      "loss": 0.0967,
      "num_tokens": 21238050.0,
      "reward": 0.3530673161149025,
      "reward_std": 0.8069795817136765,
      "rewards/cosine_scaled_reward": 0.1765336561948061,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3091.416717529297,
      "epoch": 0.09314285714285714,
      "grad_norm": 0.2311529964208603,
      "kl": 0.0021657943725585938,
      "learning_rate": 8.670853944836176e-07,
      "loss": 0.0074,
      "num_tokens": 21392414.0,
      "reward": -0.3498072102665901,
      "reward_std": 0.36920344084501266,
      "rewards/cosine_scaled_reward": -0.17490360513329506,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3042.8126220703125,
      "epoch": 0.09371428571428571,
      "grad_norm": 0.22658418118953705,
      "kl": 0.00127410888671875,
      "learning_rate": 8.648485032310144e-07,
      "loss": 0.0962,
      "num_tokens": 21544385.0,
      "reward": 0.0001346580684185028,
      "reward_std": 0.6785394810140133,
      "rewards/cosine_scaled_reward": 6.732158362865448e-05,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3391.354248046875,
      "epoch": 0.09428571428571429,
      "grad_norm": 0.2044445425271988,
      "kl": 0.0012722015380859375,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.0174,
      "num_tokens": 21714214.0,
      "reward": -0.43216387182474136,
      "reward_std": 0.4510202333331108,
      "rewards/cosine_scaled_reward": -0.2160819210112095,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3543.7708740234375,
      "epoch": 0.09485714285714286,
      "grad_norm": 0.20412862300872803,
      "kl": 0.0009775161743164062,
      "learning_rate": 8.603287946810513e-07,
      "loss": 0.0178,
      "num_tokens": 21891155.0,
      "reward": -0.5908009447157383,
      "reward_std": 0.4042652491480112,
      "rewards/cosine_scaled_reward": -0.2954004658386111,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.666732788086,
      "epoch": 0.09542857142857143,
      "grad_norm": 0.3152174651622772,
      "kl": 0.00511932373046875,
      "learning_rate": 8.580461976679099e-07,
      "loss": -0.0191,
      "num_tokens": 21969823.0,
      "reward": 0.7545666880905628,
      "reward_std": 0.8821780234575272,
      "rewards/cosine_scaled_reward": 0.3772833216935396,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2335.8333587646484,
      "epoch": 0.096,
      "grad_norm": 0.33240070939064026,
      "kl": 0.003902435302734375,
      "learning_rate": 8.557485869176825e-07,
      "loss": 0.0588,
      "num_tokens": 22087907.0,
      "reward": -0.26911586057394743,
      "reward_std": 0.5829970799386501,
      "rewards/cosine_scaled_reward": -0.13455793377943337,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1914.791748046875,
      "epoch": 0.09657142857142857,
      "grad_norm": 0.35050421953201294,
      "kl": 0.005504608154296875,
      "learning_rate": 8.534360744126753e-07,
      "loss": 0.1012,
      "num_tokens": 22185385.0,
      "reward": -0.21728778630495071,
      "reward_std": 0.8187869340181351,
      "rewards/cosine_scaled_reward": -0.10864389315247536,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3486.1250610351562,
      "epoch": 0.09714285714285714,
      "grad_norm": 0.21712802350521088,
      "kl": 0.0020456314086914062,
      "learning_rate": 8.511087728614862e-07,
      "loss": -0.0069,
      "num_tokens": 22358287.0,
      "reward": -0.2807197757065296,
      "reward_std": 0.188842561095953,
      "rewards/cosine_scaled_reward": -0.1403598841279745,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2643.2083435058594,
      "epoch": 0.09771428571428571,
      "grad_norm": 0.25833019614219666,
      "kl": 0.0024633407592773438,
      "learning_rate": 8.487667956935087e-07,
      "loss": -0.1294,
      "num_tokens": 22491605.0,
      "reward": 0.01894190162420273,
      "reward_std": 0.37402310594916344,
      "rewards/cosine_scaled_reward": 0.009470956400036812,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2838.8958740234375,
      "epoch": 0.09828571428571428,
      "grad_norm": 0.2890698313713074,
      "kl": 0.0052337646484375,
      "learning_rate": 8.464102570534061e-07,
      "loss": 0.0445,
      "num_tokens": 22633668.0,
      "reward": -0.39469024166464806,
      "reward_std": 0.5306678973138332,
      "rewards/cosine_scaled_reward": -0.19734511990100145,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.7916870117188,
      "epoch": 0.09885714285714285,
      "grad_norm": 0.34095853567123413,
      "kl": 0.0036678314208984375,
      "learning_rate": 8.440392717955475e-07,
      "loss": 0.2055,
      "num_tokens": 22724762.0,
      "reward": -0.05847650859504938,
      "reward_std": 0.7935373112559319,
      "rewards/cosine_scaled_reward": -0.029238261049613357,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3167.6458740234375,
      "epoch": 0.09942857142857142,
      "grad_norm": 0.2256677895784378,
      "kl": 0.00197601318359375,
      "learning_rate": 8.416539554784089e-07,
      "loss": 0.0598,
      "num_tokens": 22882653.0,
      "reward": -0.525588646531105,
      "reward_std": 0.44627620652318,
      "rewards/cosine_scaled_reward": -0.2627943083643913,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.0000457763672,
      "epoch": 0.1,
      "grad_norm": 0.38881924748420715,
      "kl": 0.008832931518554688,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.0274,
      "num_tokens": 22975461.0,
      "reward": 0.41337180882692337,
      "reward_std": 0.621935173869133,
      "rewards/cosine_scaled_reward": 0.20668590441346169,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2512.2084045410156,
      "epoch": 0.10057142857142858,
      "grad_norm": 0.24792905151844025,
      "kl": 0.00196075439453125,
      "learning_rate": 8.368407953869103e-07,
      "loss": -0.0595,
      "num_tokens": 23101543.0,
      "reward": 0.18181858723983169,
      "reward_std": 0.6874982379376888,
      "rewards/cosine_scaled_reward": 0.09090929350350052,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2633.8333740234375,
      "epoch": 0.10114285714285715,
      "grad_norm": 0.3130466043949127,
      "kl": 0.001682281494140625,
      "learning_rate": 8.344131861991828e-07,
      "loss": 0.0074,
      "num_tokens": 23234291.0,
      "reward": 0.03136664628982544,
      "reward_std": 0.5966363772749901,
      "rewards/cosine_scaled_reward": 0.015683308243751526,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3336.229248046875,
      "epoch": 0.10171428571428572,
      "grad_norm": 0.17930154502391815,
      "kl": 0.0008959770202636719,
      "learning_rate": 8.319717151140072e-07,
      "loss": 0.0608,
      "num_tokens": 23400322.0,
      "reward": -0.12859635055065155,
      "reward_std": 0.7636773735284805,
      "rewards/cosine_scaled_reward": -0.06429817155003548,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2887.4583435058594,
      "epoch": 0.10228571428571429,
      "grad_norm": 0.24900512397289276,
      "kl": 0.0021228790283203125,
      "learning_rate": 8.295165011252396e-07,
      "loss": -0.0046,
      "num_tokens": 23545100.0,
      "reward": -0.3140909820795059,
      "reward_std": 0.46572665125131607,
      "rewards/cosine_scaled_reward": -0.15704548731446266,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2308.4583435058594,
      "epoch": 0.10285714285714286,
      "grad_norm": 0.29636770486831665,
      "kl": 0.0054950714111328125,
      "learning_rate": 8.270476638965461e-07,
      "loss": -0.0668,
      "num_tokens": 23662218.0,
      "reward": -0.2494659647345543,
      "reward_std": 0.7115907035768032,
      "rewards/cosine_scaled_reward": -0.124732980504632,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2766.729232788086,
      "epoch": 0.10342857142857143,
      "grad_norm": 0.23259904980659485,
      "kl": 0.005695343017578125,
      "learning_rate": 8.245653237555705e-07,
      "loss": -0.0073,
      "num_tokens": 23801273.0,
      "reward": -0.15345774590969086,
      "reward_std": 0.5345512442290783,
      "rewards/cosine_scaled_reward": -0.07672888785600662,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3253.6458740234375,
      "epoch": 0.104,
      "grad_norm": 0.23866048455238342,
      "kl": 0.0014324188232421875,
      "learning_rate": 8.220696016880687e-07,
      "loss": 0.0049,
      "num_tokens": 23964000.0,
      "reward": -0.5079273246228695,
      "reward_std": 0.6336031965911388,
      "rewards/cosine_scaled_reward": -0.2539636502042413,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2357.2291870117188,
      "epoch": 0.10457142857142857,
      "grad_norm": 0.31123295426368713,
      "kl": 0.00495147705078125,
      "learning_rate": 8.195606193320136e-07,
      "loss": 0.1906,
      "num_tokens": 24084575.0,
      "reward": -0.5395753756165504,
      "reward_std": 0.2657657843083143,
      "rewards/cosine_scaled_reward": -0.2697876766324043,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.9792175292969,
      "epoch": 0.10514285714285715,
      "grad_norm": 0.3134949803352356,
      "kl": 0.00649261474609375,
      "learning_rate": 8.170384989716657e-07,
      "loss": -0.0113,
      "num_tokens": 24179290.0,
      "reward": -0.09402483701705933,
      "reward_std": 0.6055268943309784,
      "rewards/cosine_scaled_reward": -0.047012414783239365,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2454.770835876465,
      "epoch": 0.10571428571428572,
      "grad_norm": 0.44388100504875183,
      "kl": 0.001873016357421875,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.0984,
      "num_tokens": 24301895.0,
      "reward": -0.41640862822532654,
      "reward_std": 0.4775813100859523,
      "rewards/cosine_scaled_reward": -0.20820431411266327,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2527.3750915527344,
      "epoch": 0.10628571428571429,
      "grad_norm": 0.22966258227825165,
      "kl": 0.00251007080078125,
      "learning_rate": 8.119553365707802e-07,
      "loss": 0.0335,
      "num_tokens": 24431105.0,
      "reward": 0.3885076344013214,
      "reward_std": 0.8602791130542755,
      "rewards/cosine_scaled_reward": 0.1942538060247898,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3435.3958740234375,
      "epoch": 0.10685714285714286,
      "grad_norm": 0.22783932089805603,
      "kl": 0.001224517822265625,
      "learning_rate": 8.093945422764069e-07,
      "loss": -0.0104,
      "num_tokens": 24602028.0,
      "reward": -0.35544631630182266,
      "reward_std": 0.3423473574221134,
      "rewards/cosine_scaled_reward": -0.17772315442562103,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2700.3125610351562,
      "epoch": 0.10742857142857143,
      "grad_norm": 0.2991270422935486,
      "kl": 0.003947257995605469,
      "learning_rate": 8.068211054579943e-07,
      "loss": 0.1736,
      "num_tokens": 24738075.0,
      "reward": 0.23966709151864052,
      "reward_std": 0.7574465498328209,
      "rewards/cosine_scaled_reward": 0.11983353085815907,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3063.5001220703125,
      "epoch": 0.108,
      "grad_norm": 0.21379563212394714,
      "kl": 0.0010519027709960938,
      "learning_rate": 8.04235151541222e-07,
      "loss": 0.0489,
      "num_tokens": 24892311.0,
      "reward": -0.047808293253183365,
      "reward_std": 0.9623388051986694,
      "rewards/cosine_scaled_reward": -0.023904146626591682,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3313.4583740234375,
      "epoch": 0.10857142857142857,
      "grad_norm": 0.20801162719726562,
      "kl": 0.001056671142578125,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.1006,
      "num_tokens": 25058197.0,
      "reward": -0.4445067085325718,
      "reward_std": 0.3523157760500908,
      "rewards/cosine_scaled_reward": -0.2222533505409956,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2248.000045776367,
      "epoch": 0.10914285714285714,
      "grad_norm": 0.45326536893844604,
      "kl": 0.003253936767578125,
      "learning_rate": 7.990261971595048e-07,
      "loss": 0.0083,
      "num_tokens": 25171705.0,
      "reward": -0.3526854105293751,
      "reward_std": 0.45292405039072037,
      "rewards/cosine_scaled_reward": -0.17634270247071981,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2677.645896911621,
      "epoch": 0.10971428571428571,
      "grad_norm": 0.26036742329597473,
      "kl": 0.008434295654296875,
      "learning_rate": 7.964034505716476e-07,
      "loss": 0.0529,
      "num_tokens": 25305920.0,
      "reward": 0.11686116084456444,
      "reward_std": 0.6998385563492775,
      "rewards/cosine_scaled_reward": 0.0584305664524436,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3415.604248046875,
      "epoch": 0.11028571428571429,
      "grad_norm": 0.17603135108947754,
      "kl": 0.0010662078857421875,
      "learning_rate": 7.93768694627233e-07,
      "loss": 0.0194,
      "num_tokens": 25476889.0,
      "reward": -0.3952821143902838,
      "reward_std": 0.296985674649477,
      "rewards/cosine_scaled_reward": -0.1976410405477509,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2293.4167098999023,
      "epoch": 0.11085714285714286,
      "grad_norm": 0.3645835220813751,
      "kl": 0.006656646728515625,
      "learning_rate": 7.911220577405484e-07,
      "loss": 0.0665,
      "num_tokens": 25592205.0,
      "reward": 0.070842613466084,
      "reward_std": 0.631361898034811,
      "rewards/cosine_scaled_reward": 0.03542129183188081,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2965.6458740234375,
      "epoch": 0.11142857142857143,
      "grad_norm": 0.24365754425525665,
      "kl": 0.0016307830810546875,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0612,
      "num_tokens": 25740496.0,
      "reward": -0.3104175217449665,
      "reward_std": 0.5912666730582714,
      "rewards/cosine_scaled_reward": -0.15520876459777355,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2565.3750228881836,
      "epoch": 0.112,
      "grad_norm": 0.2600879669189453,
      "kl": 0.00702667236328125,
      "learning_rate": 7.857936576865356e-07,
      "loss": -0.0485,
      "num_tokens": 25869826.0,
      "reward": -0.014622047543525696,
      "reward_std": 0.34931765496730804,
      "rewards/cosine_scaled_reward": -0.007311023771762848,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2068.354202270508,
      "epoch": 0.11257142857142857,
      "grad_norm": 0.48206502199172974,
      "kl": 0.0059070587158203125,
      "learning_rate": 7.831121542179086e-07,
      "loss": 0.1763,
      "num_tokens": 25974819.0,
      "reward": -0.04546727240085602,
      "reward_std": 0.39029328897595406,
      "rewards/cosine_scaled_reward": -0.022733643651008606,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1956.2708740234375,
      "epoch": 0.11314285714285714,
      "grad_norm": 0.46816733479499817,
      "kl": 0.0056858062744140625,
      "learning_rate": 7.804192891917571e-07,
      "loss": 0.0688,
      "num_tokens": 26075332.0,
      "reward": 0.2702416032552719,
      "reward_std": 0.3293491117656231,
      "rewards/cosine_scaled_reward": 0.13512080535292625,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3000.666748046875,
      "epoch": 0.11371428571428571,
      "grad_norm": 0.2764574885368347,
      "kl": 0.0014142990112304688,
      "learning_rate": 7.777151938545235e-07,
      "loss": -0.0134,
      "num_tokens": 26224608.0,
      "reward": 0.04078002646565437,
      "reward_std": 0.5825254544615746,
      "rewards/cosine_scaled_reward": 0.020389998331665993,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2605.0625915527344,
      "epoch": 0.11428571428571428,
      "grad_norm": 0.386100172996521,
      "kl": 0.003597259521484375,
      "learning_rate": 7.75e-07,
      "loss": 0.0811,
      "num_tokens": 26356503.0,
      "reward": -0.17602870613336563,
      "reward_std": 0.5460017845034599,
      "rewards/cosine_scaled_reward": -0.08801434934139252,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2886.8958740234375,
      "epoch": 0.11485714285714285,
      "grad_norm": 0.25738444924354553,
      "kl": 0.0018377304077148438,
      "learning_rate": 7.72273839962904e-07,
      "loss": -0.0744,
      "num_tokens": 26501878.0,
      "reward": -0.44855231791734695,
      "reward_std": 0.4905274584889412,
      "rewards/cosine_scaled_reward": -0.22427614964544773,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2285.625045776367,
      "epoch": 0.11542857142857142,
      "grad_norm": 0.32238876819610596,
      "kl": 0.0074291229248046875,
      "learning_rate": 7.695368466124296e-07,
      "loss": 0.0146,
      "num_tokens": 26616928.0,
      "reward": 0.08226745203137398,
      "reward_std": 0.9037965089082718,
      "rewards/cosine_scaled_reward": 0.041133725084364414,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3112.8333740234375,
      "epoch": 0.116,
      "grad_norm": 0.1928633451461792,
      "kl": 0.00113677978515625,
      "learning_rate": 7.667891533457718e-07,
      "loss": -0.026,
      "num_tokens": 26772284.0,
      "reward": 0.10418719984591007,
      "reward_std": 0.6046584714204073,
      "rewards/cosine_scaled_reward": 0.052093599922955036,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2544.8958740234375,
      "epoch": 0.11657142857142858,
      "grad_norm": 0.28456971049308777,
      "kl": 0.0077342987060546875,
      "learning_rate": 7.640308940816239e-07,
      "loss": 0.0073,
      "num_tokens": 26900391.0,
      "reward": -0.42413394153118134,
      "reward_std": 0.3923846688121557,
      "rewards/cosine_scaled_reward": -0.21206695958971977,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2439.0208587646484,
      "epoch": 0.11714285714285715,
      "grad_norm": 0.2515351176261902,
      "kl": 0.003063201904296875,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0386,
      "num_tokens": 27022972.0,
      "reward": -0.41302137821912766,
      "reward_std": 0.5369090847671032,
      "rewards/cosine_scaled_reward": -0.20651067793369293,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2899.2500915527344,
      "epoch": 0.11771428571428572,
      "grad_norm": 0.24134834110736847,
      "kl": 0.0021295547485351562,
      "learning_rate": 7.584832158039378e-07,
      "loss": 0.0364,
      "num_tokens": 27167764.0,
      "reward": -0.4214736092835665,
      "reward_std": 0.4597649797797203,
      "rewards/cosine_scaled_reward": -0.21073680510744452,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2665.9583587646484,
      "epoch": 0.11828571428571429,
      "grad_norm": 0.27409258484840393,
      "kl": 0.005873680114746094,
      "learning_rate": 7.556940671764124e-07,
      "loss": 0.0453,
      "num_tokens": 27301262.0,
      "reward": -0.13723902963101864,
      "reward_std": 0.5910945013165474,
      "rewards/cosine_scaled_reward": -0.06861951481550932,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2065.4583740234375,
      "epoch": 0.11885714285714286,
      "grad_norm": 0.3946564793586731,
      "kl": 0.018975257873535156,
      "learning_rate": 7.528948933102438e-07,
      "loss": 0.0449,
      "num_tokens": 27406500.0,
      "reward": 0.5526407100260258,
      "reward_std": 0.3575515812262893,
      "rewards/cosine_scaled_reward": 0.2763203550130129,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2564.06254196167,
      "epoch": 0.11942857142857143,
      "grad_norm": 0.36452898383140564,
      "kl": 0.015535354614257812,
      "learning_rate": 7.500858306332172e-07,
      "loss": 0.0593,
      "num_tokens": 27536463.0,
      "reward": 0.1716044805943966,
      "reward_std": 0.2996017700061202,
      "rewards/cosine_scaled_reward": 0.08580224774777889,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2995.9376220703125,
      "epoch": 0.12,
      "grad_norm": 0.22107850015163422,
      "kl": 0.001888275146484375,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.0598,
      "num_tokens": 27686388.0,
      "reward": -0.5479435250163078,
      "reward_std": 0.3200632855296135,
      "rewards/cosine_scaled_reward": -0.2739717550575733,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2689.4791717529297,
      "epoch": 0.12057142857142857,
      "grad_norm": 0.2601456940174103,
      "kl": 0.005894660949707031,
      "learning_rate": 7.444385869608921e-07,
      "loss": 0.0078,
      "num_tokens": 27821411.0,
      "reward": -0.06815922260284424,
      "reward_std": 0.5627651736140251,
      "rewards/cosine_scaled_reward": -0.03407961130142212,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3288.8959350585938,
      "epoch": 0.12114285714285715,
      "grad_norm": 0.25300368666648865,
      "kl": 0.0014171600341796875,
      "learning_rate": 7.416006812042827e-07,
      "loss": 0.0609,
      "num_tokens": 27984318.0,
      "reward": 0.17400704324245453,
      "reward_std": 0.720283254981041,
      "rewards/cosine_scaled_reward": 0.08700351975858212,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2185.479217529297,
      "epoch": 0.12171428571428572,
      "grad_norm": 0.28523921966552734,
      "kl": 0.0068721771240234375,
      "learning_rate": 7.387534371007797e-07,
      "loss": 0.004,
      "num_tokens": 28094729.0,
      "reward": 0.2066820189356804,
      "reward_std": 0.7146667540073395,
      "rewards/cosine_scaled_reward": 0.10334100387990475,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2242.687545776367,
      "epoch": 0.12228571428571429,
      "grad_norm": 0.4000585079193115,
      "kl": 0.008604049682617188,
      "learning_rate": 7.358969934210438e-07,
      "loss": 0.1593,
      "num_tokens": 28208882.0,
      "reward": -0.39366581034846604,
      "reward_std": 0.4671928398311138,
      "rewards/cosine_scaled_reward": -0.19683289778186008,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2578.375030517578,
      "epoch": 0.12285714285714286,
      "grad_norm": 0.26247313618659973,
      "kl": 0.005038261413574219,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.1202,
      "num_tokens": 28338476.0,
      "reward": -0.4542321562767029,
      "reward_std": 0.4457564279437065,
      "rewards/cosine_scaled_reward": -0.2271160762757063,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 836.9583511352539,
      "epoch": 0.12342857142857143,
      "grad_norm": 0.6995976567268372,
      "kl": 0.0274505615234375,
      "learning_rate": 7.301570646506027e-07,
      "loss": 0.3925,
      "num_tokens": 28384326.0,
      "reward": 1.4526022970676422,
      "reward_std": 0.5801738128066063,
      "rewards/cosine_scaled_reward": 0.7263011261820793,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3564.6458740234375,
      "epoch": 0.124,
      "grad_norm": 0.18853364884853363,
      "kl": 0.0008840560913085938,
      "learning_rate": 7.27273859315928e-07,
      "loss": 0.0079,
      "num_tokens": 28562401.0,
      "reward": -0.1634646449238062,
      "reward_std": 0.4835543856024742,
      "rewards/cosine_scaled_reward": -0.0817323224619031,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2188.062545776367,
      "epoch": 0.12457142857142857,
      "grad_norm": 0.3206729292869568,
      "kl": 0.007763862609863281,
      "learning_rate": 7.243820139034464e-07,
      "loss": 0.0321,
      "num_tokens": 28673212.0,
      "reward": -0.21583660691976547,
      "reward_std": 0.4847491458058357,
      "rewards/cosine_scaled_reward": -0.10791829600930214,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1354.5625762939453,
      "epoch": 0.12514285714285714,
      "grad_norm": 0.7568894624710083,
      "kl": 0.0127105712890625,
      "learning_rate": 7.214816693576234e-07,
      "loss": 0.2123,
      "num_tokens": 28743739.0,
      "reward": 0.5299456119537354,
      "reward_std": 0.6226745247840881,
      "rewards/cosine_scaled_reward": 0.2649728059768677,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2359.729217529297,
      "epoch": 0.12571428571428572,
      "grad_norm": 0.2767828404903412,
      "kl": 0.0012798309326171875,
      "learning_rate": 7.185729670371604e-07,
      "loss": -0.0269,
      "num_tokens": 28863102.0,
      "reward": 0.4944605454802513,
      "reward_std": 0.5324547663331032,
      "rewards/cosine_scaled_reward": 0.24723027274012566,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.9791717529297,
      "epoch": 0.12628571428571428,
      "grad_norm": 0.5411548614501953,
      "kl": 0.02813243865966797,
      "learning_rate": 7.156560487081051e-07,
      "loss": 0.1265,
      "num_tokens": 28939709.0,
      "reward": 0.3169099148362875,
      "reward_std": 0.6918673776090145,
      "rewards/cosine_scaled_reward": 0.15845494996756315,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1893.2708740234375,
      "epoch": 0.12685714285714286,
      "grad_norm": 0.4862631857395172,
      "kl": 0.016809463500976562,
      "learning_rate": 7.127310565369415e-07,
      "loss": 0.1021,
      "num_tokens": 29035914.0,
      "reward": -0.06954247504472733,
      "reward_std": 0.5530165806412697,
      "rewards/cosine_scaled_reward": -0.03477124497294426,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2748.5625,
      "epoch": 0.12742857142857142,
      "grad_norm": 0.30448853969573975,
      "kl": 0.005116462707519531,
      "learning_rate": 7.097981330836616e-07,
      "loss": 0.0692,
      "num_tokens": 29173857.0,
      "reward": -0.2174304649233818,
      "reward_std": 0.4413216896355152,
      "rewards/cosine_scaled_reward": -0.1087152324616909,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3276.4166870117188,
      "epoch": 0.128,
      "grad_norm": 0.18361811339855194,
      "kl": 0.0010099411010742188,
      "learning_rate": 7.068574212948169e-07,
      "loss": 0.0613,
      "num_tokens": 29336993.0,
      "reward": 0.06714647263288498,
      "reward_std": 0.6574617028236389,
      "rewards/cosine_scaled_reward": 0.033573225140571594,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2702.625030517578,
      "epoch": 0.12857142857142856,
      "grad_norm": 0.2830963730812073,
      "kl": 0.002536773681640625,
      "learning_rate": 7.039090644965509e-07,
      "loss": -0.0118,
      "num_tokens": 29473259.0,
      "reward": -0.10639284551143646,
      "reward_std": 0.5328052043914795,
      "rewards/cosine_scaled_reward": -0.05319641903042793,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1861.312515258789,
      "epoch": 0.12914285714285714,
      "grad_norm": 0.35156944394111633,
      "kl": 0.015664100646972656,
      "learning_rate": 7.009532063876148e-07,
      "loss": 0.0346,
      "num_tokens": 29568650.0,
      "reward": 0.3703688979148865,
      "reward_std": 0.33065274357795715,
      "rewards/cosine_scaled_reward": 0.18518445640802383,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.8125305175781,
      "epoch": 0.12971428571428573,
      "grad_norm": 0.3623930513858795,
      "kl": 0.016778945922851562,
      "learning_rate": 6.979899910323624e-07,
      "loss": 0.1119,
      "num_tokens": 29662589.0,
      "reward": 0.3070980906486511,
      "reward_std": 0.5127528607845306,
      "rewards/cosine_scaled_reward": 0.15354903042316437,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1082.6041717529297,
      "epoch": 0.13028571428571428,
      "grad_norm": 0.5364865064620972,
      "kl": 0.018810272216796875,
      "learning_rate": 6.950195628537299e-07,
      "loss": 0.0448,
      "num_tokens": 29719618.0,
      "reward": 0.6576689593493938,
      "reward_std": 0.6490252837538719,
      "rewards/cosine_scaled_reward": 0.3288344731554389,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2231.6458740234375,
      "epoch": 0.13085714285714287,
      "grad_norm": 0.30130934715270996,
      "kl": 0.009735107421875,
      "learning_rate": 6.920420666261961e-07,
      "loss": -0.0144,
      "num_tokens": 29833121.0,
      "reward": -0.20264260238036513,
      "reward_std": 0.4636539947241545,
      "rewards/cosine_scaled_reward": -0.10132130864076316,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2730.1041870117188,
      "epoch": 0.13142857142857142,
      "grad_norm": 0.36225757002830505,
      "kl": 0.0050067901611328125,
      "learning_rate": 6.890576474687263e-07,
      "loss": -0.1013,
      "num_tokens": 29971474.0,
      "reward": 0.15790478512644768,
      "reward_std": 0.900160625576973,
      "rewards/cosine_scaled_reward": 0.07895238231867552,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2743.7083740234375,
      "epoch": 0.132,
      "grad_norm": 0.21701067686080933,
      "kl": 0.0045013427734375,
      "learning_rate": 6.860664508377001e-07,
      "loss": 0.0773,
      "num_tokens": 30108548.0,
      "reward": -0.1372587690129876,
      "reward_std": 0.5728438459336758,
      "rewards/cosine_scaled_reward": -0.06862937705591321,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2708.0833740234375,
      "epoch": 0.13257142857142856,
      "grad_norm": 0.48722752928733826,
      "kl": 0.011949539184570312,
      "learning_rate": 6.83068622519821e-07,
      "loss": -0.0551,
      "num_tokens": 30244260.0,
      "reward": 0.018355626612901688,
      "reward_std": 0.46119677275419235,
      "rewards/cosine_scaled_reward": 0.009177813306450844,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3555.2708740234375,
      "epoch": 0.13314285714285715,
      "grad_norm": 0.1710912138223648,
      "kl": 0.0009517669677734375,
      "learning_rate": 6.800643086250121e-07,
      "loss": 0.0049,
      "num_tokens": 30421249.0,
      "reward": -0.30245864391326904,
      "reward_std": 0.2900640666484833,
      "rewards/cosine_scaled_reward": -0.15122931078076363,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3123.9375610351562,
      "epoch": 0.1337142857142857,
      "grad_norm": 0.19327852129936218,
      "kl": 0.00162506103515625,
      "learning_rate": 6.770536555792944e-07,
      "loss": 0.0671,
      "num_tokens": 30578182.0,
      "reward": -0.25677131395787,
      "reward_std": 0.6208681277930737,
      "rewards/cosine_scaled_reward": -0.128385656978935,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1931.9792175292969,
      "epoch": 0.13428571428571429,
      "grad_norm": 0.46432214975357056,
      "kl": 0.024862289428710938,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0701,
      "num_tokens": 30677037.0,
      "reward": 0.25881527364254,
      "reward_std": 0.3069061152637005,
      "rewards/cosine_scaled_reward": 0.12940763682127,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.5208740234375,
      "epoch": 0.13485714285714287,
      "grad_norm": 0.551213264465332,
      "kl": 0.0158843994140625,
      "learning_rate": 6.710139192768694e-07,
      "loss": 0.2681,
      "num_tokens": 30732298.0,
      "reward": 0.17941563576459885,
      "reward_std": 0.547722615301609,
      "rewards/cosine_scaled_reward": 0.08970782160758972,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.9791870117188,
      "epoch": 0.13542857142857143,
      "grad_norm": 0.5294864773750305,
      "kl": 0.012974739074707031,
      "learning_rate": 6.679851303883891e-07,
      "loss": 0.35,
      "num_tokens": 30821337.0,
      "reward": -0.2729988917708397,
      "reward_std": 0.39208219945430756,
      "rewards/cosine_scaled_reward": -0.13649944216012955,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2661.2708740234375,
      "epoch": 0.136,
      "grad_norm": 0.26540040969848633,
      "kl": 0.008977890014648438,
      "learning_rate": 6.649505910711058e-07,
      "loss": 0.1287,
      "num_tokens": 30955702.0,
      "reward": -0.24963749200105667,
      "reward_std": 0.5366819277405739,
      "rewards/cosine_scaled_reward": -0.12481874227523804,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2663.8958740234375,
      "epoch": 0.13657142857142857,
      "grad_norm": 0.5480060577392578,
      "kl": 0.012514114379882812,
      "learning_rate": 6.619104492241847e-07,
      "loss": 0.1785,
      "num_tokens": 31089365.0,
      "reward": -0.02775234915316105,
      "reward_std": 0.7676347196102142,
      "rewards/cosine_scaled_reward": -0.013876182027161121,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2176.229248046875,
      "epoch": 0.13714285714285715,
      "grad_norm": 0.28263622522354126,
      "kl": 0.003826141357421875,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0034,
      "num_tokens": 31199812.0,
      "reward": -0.4636247009038925,
      "reward_std": 0.3770550861954689,
      "rewards/cosine_scaled_reward": -0.23181235045194626,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1963.645839691162,
      "epoch": 0.1377142857142857,
      "grad_norm": 0.5095245838165283,
      "kl": 0.02035999298095703,
      "learning_rate": 6.558139508961654e-07,
      "loss": 0.0311,
      "num_tokens": 31299575.0,
      "reward": 0.524737037718296,
      "reward_std": 0.8398763090372086,
      "rewards/cosine_scaled_reward": 0.2623685207217932,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.0,
      "epoch": 0.1382857142857143,
      "grad_norm": 0.21239568293094635,
      "kl": 0.0011444091796875,
      "learning_rate": 6.527578915497951e-07,
      "loss": 0.0,
      "num_tokens": 31477739.0,
      "reward": -0.3269985783845186,
      "reward_std": 0.2089610155671835,
      "rewards/cosine_scaled_reward": -0.1634992891922593,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2510.937530517578,
      "epoch": 0.13885714285714285,
      "grad_norm": 0.32022956013679504,
      "kl": 0.00478363037109375,
      "learning_rate": 6.496968239287603e-07,
      "loss": 0.0972,
      "num_tokens": 31603616.0,
      "reward": -0.44686760660260916,
      "reward_std": 0.3707558251917362,
      "rewards/cosine_scaled_reward": -0.22343379561789334,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2716.604248046875,
      "epoch": 0.13942857142857143,
      "grad_norm": 0.3231089115142822,
      "kl": 0.0021686553955078125,
      "learning_rate": 6.466308972251785e-07,
      "loss": -0.0145,
      "num_tokens": 31740217.0,
      "reward": -0.5142710842192173,
      "reward_std": 0.6101813912391663,
      "rewards/cosine_scaled_reward": -0.25713553559035063,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2832.625030517578,
      "epoch": 0.14,
      "grad_norm": 0.26809558272361755,
      "kl": 0.0017642974853515625,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0903,
      "num_tokens": 31881463.0,
      "reward": -0.1661408469080925,
      "reward_std": 0.7787005566060543,
      "rewards/cosine_scaled_reward": -0.08307042345404625,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2692.8541717529297,
      "epoch": 0.14057142857142857,
      "grad_norm": 0.2669066786766052,
      "kl": 0.009328842163085938,
      "learning_rate": 6.404850645156841e-07,
      "loss": 0.0709,
      "num_tokens": 32016624.0,
      "reward": -0.19150156527757645,
      "reward_std": 0.509680725634098,
      "rewards/cosine_scaled_reward": -0.09575077798217535,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2888.9375,
      "epoch": 0.14114285714285715,
      "grad_norm": 0.217579185962677,
      "kl": 0.0030298233032226562,
      "learning_rate": 6.374054580489873e-07,
      "loss": -0.1061,
      "num_tokens": 32162217.0,
      "reward": 0.31709786131978035,
      "reward_std": 0.7779325805604458,
      "rewards/cosine_scaled_reward": 0.15854893065989017,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1323.1041870117188,
      "epoch": 0.1417142857142857,
      "grad_norm": 0.6014187335968018,
      "kl": 0.03565692901611328,
      "learning_rate": 6.343215915635761e-07,
      "loss": 0.0756,
      "num_tokens": 32231006.0,
      "reward": 0.6246988326311111,
      "reward_std": 0.35837205685675144,
      "rewards/cosine_scaled_reward": 0.3123493976891041,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2613.6458740234375,
      "epoch": 0.1422857142857143,
      "grad_norm": 0.29411664605140686,
      "kl": 0.0029964447021484375,
      "learning_rate": 6.31233615362752e-07,
      "loss": 0.2094,
      "num_tokens": 32362737.0,
      "reward": -0.4900950863957405,
      "reward_std": 0.5070604905486107,
      "rewards/cosine_scaled_reward": -0.24504754319787025,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2731.5208740234375,
      "epoch": 0.14285714285714285,
      "grad_norm": 0.35469648241996765,
      "kl": 0.017116546630859375,
      "learning_rate": 6.281416799501187e-07,
      "loss": -0.0102,
      "num_tokens": 32499598.0,
      "reward": -0.21433956921100616,
      "reward_std": 0.520557913929224,
      "rewards/cosine_scaled_reward": -0.10716977342963219,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1934.125015258789,
      "epoch": 0.14342857142857143,
      "grad_norm": 0.3160950839519501,
      "kl": 0.009817123413085938,
      "learning_rate": 6.25045936022246e-07,
      "loss": 0.127,
      "num_tokens": 32598400.0,
      "reward": -0.07275501638650894,
      "reward_std": 0.4598498921841383,
      "rewards/cosine_scaled_reward": -0.03637750819325447,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2064.5834045410156,
      "epoch": 0.144,
      "grad_norm": 0.3414526581764221,
      "kl": 0.00769805908203125,
      "learning_rate": 6.219465344613258e-07,
      "loss": 0.0573,
      "num_tokens": 32703140.0,
      "reward": -0.3573624864220619,
      "reward_std": 0.4409428536891937,
      "rewards/cosine_scaled_reward": -0.17868124693632126,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2815.083335876465,
      "epoch": 0.14457142857142857,
      "grad_norm": 0.3612518012523651,
      "kl": 0.007879257202148438,
      "learning_rate": 6.188436263278172e-07,
      "loss": -0.0483,
      "num_tokens": 32843784.0,
      "reward": -0.22203051671385765,
      "reward_std": 0.31091106310486794,
      "rewards/cosine_scaled_reward": -0.11101526208221912,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2616.479248046875,
      "epoch": 0.14514285714285713,
      "grad_norm": 0.3855963945388794,
      "kl": 0.011937141418457031,
      "learning_rate": 6.157373628530852e-07,
      "loss": 0.1246,
      "num_tokens": 32975375.0,
      "reward": 0.18154123798012733,
      "reward_std": 0.5934804044663906,
      "rewards/cosine_scaled_reward": 0.09077062457799911,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2412.333335876465,
      "epoch": 0.1457142857142857,
      "grad_norm": 0.34006571769714355,
      "kl": 0.01013946533203125,
      "learning_rate": 6.126278954320294e-07,
      "loss": -0.0014,
      "num_tokens": 33097635.0,
      "reward": 0.23491641879081726,
      "reward_std": 0.5346547961235046,
      "rewards/cosine_scaled_reward": 0.11745821312069893,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3411.3541870117188,
      "epoch": 0.1462857142857143,
      "grad_norm": 0.19116432964801788,
      "kl": 0.0012416839599609375,
      "learning_rate": 6.095153756157051e-07,
      "loss": 0.0547,
      "num_tokens": 33267932.0,
      "reward": -0.4405994936823845,
      "reward_std": 0.43207642808556557,
      "rewards/cosine_scaled_reward": -0.22029974684119225,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2541.4583435058594,
      "epoch": 0.14685714285714285,
      "grad_norm": 0.3120291233062744,
      "kl": 0.0090484619140625,
      "learning_rate": 6.06399955103937e-07,
      "loss": 0.0088,
      "num_tokens": 33395922.0,
      "reward": -0.40020735282450914,
      "reward_std": 0.6490463241934776,
      "rewards/cosine_scaled_reward": -0.20010367268696427,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.3958587646484,
      "epoch": 0.14742857142857144,
      "grad_norm": 0.6874107122421265,
      "kl": 0.04048919677734375,
      "learning_rate": 6.032817857379256e-07,
      "loss": 0.1263,
      "num_tokens": 33488953.0,
      "reward": 0.3148918077349663,
      "reward_std": 0.6025057537481189,
      "rewards/cosine_scaled_reward": 0.15744590386748314,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2497.2916870117188,
      "epoch": 0.148,
      "grad_norm": 0.3090062141418457,
      "kl": 0.0051898956298828125,
      "learning_rate": 6.001610194928464e-07,
      "loss": -0.0475,
      "num_tokens": 33614655.0,
      "reward": -0.1998734101653099,
      "reward_std": 0.5551594458520412,
      "rewards/cosine_scaled_reward": -0.09993669763207436,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3108.2291870117188,
      "epoch": 0.14857142857142858,
      "grad_norm": 0.20045077800750732,
      "kl": 0.00176239013671875,
      "learning_rate": 5.97037808470444e-07,
      "loss": -0.095,
      "num_tokens": 33770126.0,
      "reward": -0.24349116533994675,
      "reward_std": 0.4448055624961853,
      "rewards/cosine_scaled_reward": -0.12174558266997337,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3407.041748046875,
      "epoch": 0.14914285714285713,
      "grad_norm": 0.20637670159339905,
      "kl": 0.0013484954833984375,
      "learning_rate": 5.939123048916173e-07,
      "loss": 0.0355,
      "num_tokens": 33939832.0,
      "reward": -0.44314368814229965,
      "reward_std": 0.4703112803399563,
      "rewards/cosine_scaled_reward": -0.22157184407114983,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2691.125045776367,
      "epoch": 0.14971428571428572,
      "grad_norm": 0.2881263196468353,
      "kl": 0.00722503662109375,
      "learning_rate": 5.907846610890011e-07,
      "loss": 0.0162,
      "num_tokens": 34074154.0,
      "reward": 0.07448863238096237,
      "reward_std": 0.6951295547187328,
      "rewards/cosine_scaled_reward": 0.03724431432783604,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2771.6041717529297,
      "epoch": 0.15028571428571427,
      "grad_norm": 0.35452720522880554,
      "kl": 0.0109100341796875,
      "learning_rate": 5.87655029499542e-07,
      "loss": -0.012,
      "num_tokens": 34213515.0,
      "reward": -0.10311572067439556,
      "reward_std": 0.3219672627747059,
      "rewards/cosine_scaled_reward": -0.05155786033719778,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3548.666748046875,
      "epoch": 0.15085714285714286,
      "grad_norm": 0.20419248938560486,
      "kl": 0.0011997222900390625,
      "learning_rate": 5.845235626570683e-07,
      "loss": 0.0187,
      "num_tokens": 34389839.0,
      "reward": -0.3663049042224884,
      "reward_std": 0.6324282512068748,
      "rewards/cosine_scaled_reward": -0.1831524483859539,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3260.1875,
      "epoch": 0.15142857142857144,
      "grad_norm": 0.23959365487098694,
      "kl": 0.0013904571533203125,
      "learning_rate": 5.813904131848564e-07,
      "loss": -0.0252,
      "num_tokens": 34552136.0,
      "reward": -0.2764420807361603,
      "reward_std": 0.49818455800414085,
      "rewards/cosine_scaled_reward": -0.13822103291749954,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2710.437530517578,
      "epoch": 0.152,
      "grad_norm": 0.32057079672813416,
      "kl": 0.0076694488525390625,
      "learning_rate": 5.78255733788191e-07,
      "loss": -0.0256,
      "num_tokens": 34688285.0,
      "reward": -0.3459478300064802,
      "reward_std": 0.25663014128804207,
      "rewards/cosine_scaled_reward": -0.17297391314059496,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.208396911621,
      "epoch": 0.15257142857142858,
      "grad_norm": 0.4645328223705292,
      "kl": 0.012868881225585938,
      "learning_rate": 5.751196772469237e-07,
      "loss": 0.113,
      "num_tokens": 34767999.0,
      "reward": -0.11789099872112274,
      "reward_std": 0.5525611527264118,
      "rewards/cosine_scaled_reward": -0.05894550122320652,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2182.375,
      "epoch": 0.15314285714285714,
      "grad_norm": 0.3756212294101715,
      "kl": 0.010714530944824219,
      "learning_rate": 5.71982396408026e-07,
      "loss": 0.1363,
      "num_tokens": 34878669.0,
      "reward": -0.077213354408741,
      "reward_std": 0.57977394759655,
      "rewards/cosine_scaled_reward": -0.0386066734790802,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3128.6251220703125,
      "epoch": 0.15371428571428572,
      "grad_norm": 0.1624217927455902,
      "kl": 0.0017490386962890625,
      "learning_rate": 5.688440441781398e-07,
      "loss": -0.0542,
      "num_tokens": 35034243.0,
      "reward": -0.375546395778656,
      "reward_std": 0.2080207783728838,
      "rewards/cosine_scaled_reward": -0.18777319695800543,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2074.270839691162,
      "epoch": 0.15428571428571428,
      "grad_norm": 0.48845088481903076,
      "kl": 0.02855682373046875,
      "learning_rate": 5.657047735161255e-07,
      "loss": -0.1012,
      "num_tokens": 35139796.0,
      "reward": 0.26241855323314667,
      "reward_std": 0.4325704537332058,
      "rewards/cosine_scaled_reward": 0.13120926916599274,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2039.5416870117188,
      "epoch": 0.15485714285714286,
      "grad_norm": 0.27519840002059937,
      "kl": 0.004604339599609375,
      "learning_rate": 5.625647374256061e-07,
      "loss": -0.1227,
      "num_tokens": 35244654.0,
      "reward": -0.13348775170743465,
      "reward_std": 0.5455809384584427,
      "rewards/cosine_scaled_reward": -0.06674387538805604,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2399.0625915527344,
      "epoch": 0.15542857142857142,
      "grad_norm": 0.3173102140426636,
      "kl": 0.0048809051513671875,
      "learning_rate": 5.594240889475106e-07,
      "loss": 0.1177,
      "num_tokens": 35365593.0,
      "reward": -0.08320139348506927,
      "reward_std": 0.7119211666285992,
      "rewards/cosine_scaled_reward": -0.04160069301724434,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3164.75,
      "epoch": 0.156,
      "grad_norm": 0.22468267381191254,
      "kl": 0.001804351806640625,
      "learning_rate": 5.562829811526154e-07,
      "loss": 0.0294,
      "num_tokens": 35523117.0,
      "reward": -0.3206188827753067,
      "reward_std": 0.5574362277984619,
      "rewards/cosine_scaled_reward": -0.16030943393707275,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2586.1666870117188,
      "epoch": 0.15657142857142858,
      "grad_norm": 0.3104928731918335,
      "kl": 0.0055084228515625,
      "learning_rate": 5.531415671340826e-07,
      "loss": 0.0949,
      "num_tokens": 35653757.0,
      "reward": -0.13973749428987503,
      "reward_std": 0.3990135118365288,
      "rewards/cosine_scaled_reward": -0.06986876204609871,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2072.541717529297,
      "epoch": 0.15714285714285714,
      "grad_norm": 0.41220641136169434,
      "kl": 0.013563156127929688,
      "learning_rate": 5.5e-07,
      "loss": 0.0363,
      "num_tokens": 35759299.0,
      "reward": -0.4124254733324051,
      "reward_std": 0.5271046534180641,
      "rewards/cosine_scaled_reward": -0.20621273759752512,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1188.1250457763672,
      "epoch": 0.15771428571428572,
      "grad_norm": 0.4833068251609802,
      "kl": 0.0210723876953125,
      "learning_rate": 5.468584328659172e-07,
      "loss": -0.0615,
      "num_tokens": 35822281.0,
      "reward": 0.5711349472403526,
      "reward_std": 0.7510395795106888,
      "rewards/cosine_scaled_reward": 0.28556746058166027,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2676.9375915527344,
      "epoch": 0.15828571428571428,
      "grad_norm": 0.28584277629852295,
      "kl": 0.0041713714599609375,
      "learning_rate": 5.437170188473847e-07,
      "loss": 0.0217,
      "num_tokens": 35956750.0,
      "reward": 0.05125083029270172,
      "reward_std": 0.6981733292341232,
      "rewards/cosine_scaled_reward": 0.025625411421060562,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2614.9375,
      "epoch": 0.15885714285714286,
      "grad_norm": 0.29526305198669434,
      "kl": 0.002593994140625,
      "learning_rate": 5.405759110524894e-07,
      "loss": 0.0156,
      "num_tokens": 36088627.0,
      "reward": -0.3907754272222519,
      "reward_std": 0.5090431272983551,
      "rewards/cosine_scaled_reward": -0.19538771361112595,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2388.791748046875,
      "epoch": 0.15942857142857142,
      "grad_norm": 0.32363101840019226,
      "kl": 0.0035810470581054688,
      "learning_rate": 5.37435262574394e-07,
      "loss": 0.1454,
      "num_tokens": 36209013.0,
      "reward": 0.351345656439662,
      "reward_std": 0.7279459312558174,
      "rewards/cosine_scaled_reward": 0.1756728133186698,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.5,
      "epoch": 0.16,
      "grad_norm": 0.4550730884075165,
      "kl": 0.0157623291015625,
      "learning_rate": 5.342952264838747e-07,
      "loss": -0.0517,
      "num_tokens": 36295233.0,
      "reward": 0.032998040318489075,
      "reward_std": 0.4106169492006302,
      "rewards/cosine_scaled_reward": 0.016499027609825134,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2025.9166793823242,
      "epoch": 0.16057142857142856,
      "grad_norm": 0.4892213046550751,
      "kl": 0.01910400390625,
      "learning_rate": 5.311559558218603e-07,
      "loss": 0.2053,
      "num_tokens": 36398009.0,
      "reward": 0.2391605954617262,
      "reward_std": 0.40038658678531647,
      "rewards/cosine_scaled_reward": 0.1195802828297019,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2755.166717529297,
      "epoch": 0.16114285714285714,
      "grad_norm": 0.3091196119785309,
      "kl": 0.00545501708984375,
      "learning_rate": 5.28017603591974e-07,
      "loss": -0.0154,
      "num_tokens": 36536653.0,
      "reward": -0.05713912099599838,
      "reward_std": 0.6415699534118176,
      "rewards/cosine_scaled_reward": -0.028569556772708893,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2594.9583587646484,
      "epoch": 0.16171428571428573,
      "grad_norm": 0.3001720607280731,
      "kl": 0.009159088134765625,
      "learning_rate": 5.248803227530763e-07,
      "loss": 0.1765,
      "num_tokens": 36666971.0,
      "reward": -0.5184054747223854,
      "reward_std": 0.3386564552783966,
      "rewards/cosine_scaled_reward": -0.2592027336359024,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2470.2291717529297,
      "epoch": 0.16228571428571428,
      "grad_norm": 0.43797507882118225,
      "kl": 0.0067348480224609375,
      "learning_rate": 5.21744266211809e-07,
      "loss": 0.0402,
      "num_tokens": 36791950.0,
      "reward": 0.5365435220301151,
      "reward_std": 0.8577289432287216,
      "rewards/cosine_scaled_reward": 0.26827176474034786,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3468.2709350585938,
      "epoch": 0.16285714285714287,
      "grad_norm": 0.20558501780033112,
      "kl": 0.0013113021850585938,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.059,
      "num_tokens": 36964079.0,
      "reward": -0.32812320441007614,
      "reward_std": 0.5332767590880394,
      "rewards/cosine_scaled_reward": -0.16406160034239292,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2288.3125610351562,
      "epoch": 0.16342857142857142,
      "grad_norm": 0.24818028509616852,
      "kl": 0.002147674560546875,
      "learning_rate": 5.154764373429315e-07,
      "loss": -0.0275,
      "num_tokens": 37080458.0,
      "reward": 0.6154246423393488,
      "reward_std": 0.8250385522842407,
      "rewards/cosine_scaled_reward": 0.30771232303231955,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2368.520896911621,
      "epoch": 0.164,
      "grad_norm": 0.2976325750350952,
      "kl": 0.007648468017578125,
      "learning_rate": 5.123449705004581e-07,
      "loss": 0.0946,
      "num_tokens": 37200447.0,
      "reward": 0.4257568195462227,
      "reward_std": 0.9012775421142578,
      "rewards/cosine_scaled_reward": 0.21287840977311134,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2584.395896911621,
      "epoch": 0.16457142857142856,
      "grad_norm": 0.26449525356292725,
      "kl": 0.0072422027587890625,
      "learning_rate": 5.09215338910999e-07,
      "loss": 0.0002,
      "num_tokens": 37330510.0,
      "reward": -0.43290044367313385,
      "reward_std": 0.4573391415178776,
      "rewards/cosine_scaled_reward": -0.21645020693540573,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2852.7083740234375,
      "epoch": 0.16514285714285715,
      "grad_norm": 0.23969526588916779,
      "kl": 0.0036468505859375,
      "learning_rate": 5.060876951083828e-07,
      "loss": 0.1021,
      "num_tokens": 37473044.0,
      "reward": -0.14536024630069733,
      "reward_std": 0.7508940920233727,
      "rewards/cosine_scaled_reward": -0.07268010824918747,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2355.1250915527344,
      "epoch": 0.1657142857142857,
      "grad_norm": 0.2979152202606201,
      "kl": 0.0029964447021484375,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.1276,
      "num_tokens": 37591598.0,
      "reward": -0.27352139353752136,
      "reward_std": 0.5863078981637955,
      "rewards/cosine_scaled_reward": -0.13676068745553493,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3103.0833740234375,
      "epoch": 0.1662857142857143,
      "grad_norm": 0.24183134734630585,
      "kl": 0.0017833709716796875,
      "learning_rate": 4.998389805071536e-07,
      "loss": 0.0017,
      "num_tokens": 37746126.0,
      "reward": -0.4938964769244194,
      "reward_std": 0.20535914227366447,
      "rewards/cosine_scaled_reward": -0.24694822914898396,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2742.666748046875,
      "epoch": 0.16685714285714287,
      "grad_norm": 0.3063388764858246,
      "kl": 0.00244903564453125,
      "learning_rate": 4.967182142620745e-07,
      "loss": 0.0973,
      "num_tokens": 37883798.0,
      "reward": 0.00018896162509918213,
      "reward_std": 0.8789166212081909,
      "rewards/cosine_scaled_reward": 9.44770872592926e-05,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.7708587646484,
      "epoch": 0.16742857142857143,
      "grad_norm": 0.48416784405708313,
      "kl": 0.020442962646484375,
      "learning_rate": 4.93600044896063e-07,
      "loss": -0.0533,
      "num_tokens": 37961991.0,
      "reward": 0.5087592005729675,
      "reward_std": 0.6853836588561535,
      "rewards/cosine_scaled_reward": 0.2543795704841614,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3275.375,
      "epoch": 0.168,
      "grad_norm": 0.2511613368988037,
      "kl": 0.0014057159423828125,
      "learning_rate": 4.904846243842949e-07,
      "loss": 0.0496,
      "num_tokens": 38125461.0,
      "reward": -0.48528067022562027,
      "reward_std": 0.555925726890564,
      "rewards/cosine_scaled_reward": -0.24264032766222954,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3450.3334350585938,
      "epoch": 0.16857142857142857,
      "grad_norm": 0.2048538625240326,
      "kl": 0.001056671142578125,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0367,
      "num_tokens": 38297365.0,
      "reward": -0.2943028609151952,
      "reward_std": 0.4288143813610077,
      "rewards/cosine_scaled_reward": -0.1471514304575976,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3004.9166870117188,
      "epoch": 0.16914285714285715,
      "grad_norm": 0.2455970197916031,
      "kl": 0.001949310302734375,
      "learning_rate": 4.842626371469149e-07,
      "loss": 0.0845,
      "num_tokens": 38447913.0,
      "reward": -0.4560352563858032,
      "reward_std": 0.5781249962747097,
      "rewards/cosine_scaled_reward": -0.22801762074232101,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2529.437545776367,
      "epoch": 0.1697142857142857,
      "grad_norm": 0.29431456327438354,
      "kl": 0.008335113525390625,
      "learning_rate": 4.811563736721829e-07,
      "loss": 0.0079,
      "num_tokens": 38575026.0,
      "reward": 0.16207153722643852,
      "reward_std": 0.46869122236967087,
      "rewards/cosine_scaled_reward": 0.08103577420115471,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.666732788086,
      "epoch": 0.1702857142857143,
      "grad_norm": 0.5596351623535156,
      "kl": 0.015949249267578125,
      "learning_rate": 4.780534655386743e-07,
      "loss": -0.1605,
      "num_tokens": 38670062.0,
      "reward": -0.20284654013812542,
      "reward_std": 0.5799939371645451,
      "rewards/cosine_scaled_reward": -0.10142326634377241,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2124.8333740234375,
      "epoch": 0.17085714285714285,
      "grad_norm": 0.49901139736175537,
      "kl": 0.0066089630126953125,
      "learning_rate": 4.749540639777539e-07,
      "loss": 0.2655,
      "num_tokens": 38778390.0,
      "reward": -0.23837678879499435,
      "reward_std": 0.5732098147273064,
      "rewards/cosine_scaled_reward": -0.11918839067220688,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2721.2709045410156,
      "epoch": 0.17142857142857143,
      "grad_norm": 0.27538150548934937,
      "kl": 0.001575469970703125,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0042,
      "num_tokens": 38915167.0,
      "reward": -0.44970532320439816,
      "reward_std": 0.338998232036829,
      "rewards/cosine_scaled_reward": -0.22485265973955393,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2737.937545776367,
      "epoch": 0.172,
      "grad_norm": 0.36887478828430176,
      "kl": 0.017746925354003906,
      "learning_rate": 4.68766384637248e-07,
      "loss": 0.0646,
      "num_tokens": 39053920.0,
      "reward": -0.25950850173830986,
      "reward_std": 0.16469359770417213,
      "rewards/cosine_scaled_reward": -0.12975424528121948,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2430.8958740234375,
      "epoch": 0.17257142857142857,
      "grad_norm": 0.29716700315475464,
      "kl": 0.012701034545898438,
      "learning_rate": 4.656784084364238e-07,
      "loss": 0.0492,
      "num_tokens": 39176351.0,
      "reward": 0.08118153735995293,
      "reward_std": 0.26172393187880516,
      "rewards/cosine_scaled_reward": 0.04059076961129904,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2375.4166870117188,
      "epoch": 0.17314285714285715,
      "grad_norm": 0.29471853375434875,
      "kl": 0.00960540771484375,
      "learning_rate": 4.6259454195101267e-07,
      "loss": 0.0308,
      "num_tokens": 39296539.0,
      "reward": -0.1111888438463211,
      "reward_std": 0.5966824367642403,
      "rewards/cosine_scaled_reward": -0.05559442937374115,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3561.3958740234375,
      "epoch": 0.1737142857142857,
      "grad_norm": 0.20573833584785461,
      "kl": 0.0010528564453125,
      "learning_rate": 4.59514935484316e-07,
      "loss": 0.0075,
      "num_tokens": 39473918.0,
      "reward": -0.32946310192346573,
      "reward_std": 0.5383296981453896,
      "rewards/cosine_scaled_reward": -0.16473154537379742,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2468.7084045410156,
      "epoch": 0.1742857142857143,
      "grad_norm": 0.2480940818786621,
      "kl": 0.0030117034912109375,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.0284,
      "num_tokens": 39598176.0,
      "reward": 0.03428598493337631,
      "reward_std": 0.5898709297180176,
      "rewards/cosine_scaled_reward": 0.01714298501610756,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1914.5209045410156,
      "epoch": 0.17485714285714285,
      "grad_norm": 0.4037153124809265,
      "kl": 0.009237289428710938,
      "learning_rate": 4.5336910277482155e-07,
      "loss": 0.1171,
      "num_tokens": 39695629.0,
      "reward": 0.6262711547315121,
      "reward_std": 0.7676540613174438,
      "rewards/cosine_scaled_reward": 0.31313556246459484,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3471.2084350585938,
      "epoch": 0.17542857142857143,
      "grad_norm": 0.18108604848384857,
      "kl": 0.0008935928344726562,
      "learning_rate": 4.503031760712397e-07,
      "loss": 0.0372,
      "num_tokens": 39868547.0,
      "reward": -0.5682974830269814,
      "reward_std": 0.362131267786026,
      "rewards/cosine_scaled_reward": -0.28414873220026493,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1946.3333435058594,
      "epoch": 0.176,
      "grad_norm": 0.427402138710022,
      "kl": 0.012170791625976562,
      "learning_rate": 4.4724210845020494e-07,
      "loss": 0.0688,
      "num_tokens": 39967107.0,
      "reward": 0.6227611526846886,
      "reward_std": 0.776022270321846,
      "rewards/cosine_scaled_reward": 0.3113805763423443,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3468.5833740234375,
      "epoch": 0.17657142857142857,
      "grad_norm": 0.21143701672554016,
      "kl": 0.0014905929565429688,
      "learning_rate": 4.441860491038345e-07,
      "loss": 0.0234,
      "num_tokens": 40140067.0,
      "reward": -0.3016166687011719,
      "reward_std": 0.5722797363996506,
      "rewards/cosine_scaled_reward": -0.15080832783132792,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2890.5834350585938,
      "epoch": 0.17714285714285713,
      "grad_norm": 0.22804640233516693,
      "kl": 0.0019989013671875,
      "learning_rate": 4.4113514698014953e-07,
      "loss": -0.0995,
      "num_tokens": 40285223.0,
      "reward": 0.00922529399394989,
      "reward_std": 0.8645202741026878,
      "rewards/cosine_scaled_reward": 0.004612648859620094,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2540.4792098999023,
      "epoch": 0.1777142857142857,
      "grad_norm": 0.4666746258735657,
      "kl": 0.009735107421875,
      "learning_rate": 4.3808955077581546e-07,
      "loss": 0.0154,
      "num_tokens": 40413298.0,
      "reward": -0.29096972569823265,
      "reward_std": 0.3306393250823021,
      "rewards/cosine_scaled_reward": -0.14548486191779375,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2994.437530517578,
      "epoch": 0.1782857142857143,
      "grad_norm": 0.35905346274375916,
      "kl": 0.0028743743896484375,
      "learning_rate": 4.350494089288943e-07,
      "loss": -0.0335,
      "num_tokens": 40562971.0,
      "reward": -0.363456416875124,
      "reward_std": 0.3647861182689667,
      "rewards/cosine_scaled_reward": -0.18172819539904594,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2380.0833587646484,
      "epoch": 0.17885714285714285,
      "grad_norm": 0.4951138198375702,
      "kl": 0.0026788711547851562,
      "learning_rate": 4.3201486961161093e-07,
      "loss": -0.0585,
      "num_tokens": 40683791.0,
      "reward": -0.10693264147266746,
      "reward_std": 0.5804431773722172,
      "rewards/cosine_scaled_reward": -0.05346631887368858,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2596.604248046875,
      "epoch": 0.17942857142857144,
      "grad_norm": 0.3246864974498749,
      "kl": 0.0043659210205078125,
      "learning_rate": 4.2898608072313045e-07,
      "loss": 0.2706,
      "num_tokens": 40814932.0,
      "reward": 0.4242757335305214,
      "reward_std": 1.1075529158115387,
      "rewards/cosine_scaled_reward": 0.21213785372674465,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2499.791778564453,
      "epoch": 0.18,
      "grad_norm": 0.27629679441452026,
      "kl": 0.0028667449951171875,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.0475,
      "num_tokens": 40940766.0,
      "reward": -0.3353143408894539,
      "reward_std": 0.637674517929554,
      "rewards/cosine_scaled_reward": -0.16765715926885605,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3229.0416870117188,
      "epoch": 0.18057142857142858,
      "grad_norm": 0.23208899796009064,
      "kl": 0.001750946044921875,
      "learning_rate": 4.2294634442070553e-07,
      "loss": -0.0575,
      "num_tokens": 41101688.0,
      "reward": -0.3000589460134506,
      "reward_std": 0.4578623231500387,
      "rewards/cosine_scaled_reward": -0.15002946369349957,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2577.125030517578,
      "epoch": 0.18114285714285713,
      "grad_norm": 0.232590913772583,
      "kl": 0.0039806365966796875,
      "learning_rate": 4.1993569137498776e-07,
      "loss": -0.0006,
      "num_tokens": 41231234.0,
      "reward": 0.20818227902054787,
      "reward_std": 0.40563616156578064,
      "rewards/cosine_scaled_reward": 0.10409114044159651,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2359.3750610351562,
      "epoch": 0.18171428571428572,
      "grad_norm": 0.26052311062812805,
      "kl": 0.003261566162109375,
      "learning_rate": 4.1693137748017915e-07,
      "loss": 0.0403,
      "num_tokens": 41349608.0,
      "reward": 0.13113708421587944,
      "reward_std": 0.5818060413002968,
      "rewards/cosine_scaled_reward": 0.06556854210793972,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2511.3541870117188,
      "epoch": 0.18228571428571427,
      "grad_norm": 0.31813371181488037,
      "kl": 0.0041961669921875,
      "learning_rate": 4.1393354916230005e-07,
      "loss": 0.0482,
      "num_tokens": 41476213.0,
      "reward": 0.3437543660402298,
      "reward_std": 0.48744695633649826,
      "rewards/cosine_scaled_reward": 0.1718771643936634,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2694.854232788086,
      "epoch": 0.18285714285714286,
      "grad_norm": 0.23167455196380615,
      "kl": 0.00571441650390625,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0731,
      "num_tokens": 41612898.0,
      "reward": 0.04213899374008179,
      "reward_std": 0.5815633870661259,
      "rewards/cosine_scaled_reward": 0.021069496870040894,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.0,
      "epoch": 0.18342857142857144,
      "grad_norm": 0.19942978024482727,
      "kl": 0.00109100341796875,
      "learning_rate": 4.079579333738039e-07,
      "loss": 0.0,
      "num_tokens": 41791110.0,
      "reward": -0.4013543911278248,
      "reward_std": 0.21282335743308067,
      "rewards/cosine_scaled_reward": -0.2006771918386221,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2271.437545776367,
      "epoch": 0.184,
      "grad_norm": 0.33824867010116577,
      "kl": 0.013680458068847656,
      "learning_rate": 4.0498043714627006e-07,
      "loss": 0.0733,
      "num_tokens": 41905971.0,
      "reward": -0.16819965280592442,
      "reward_std": 0.7291267365217209,
      "rewards/cosine_scaled_reward": -0.08409981848672032,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2732.0416717529297,
      "epoch": 0.18457142857142858,
      "grad_norm": 0.3594345450401306,
      "kl": 0.011373519897460938,
      "learning_rate": 4.020100089676376e-07,
      "loss": 0.0273,
      "num_tokens": 42042989.0,
      "reward": -0.20197953283786774,
      "reward_std": 0.6335406377911568,
      "rewards/cosine_scaled_reward": -0.10098976641893387,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1872.7292022705078,
      "epoch": 0.18514285714285714,
      "grad_norm": 0.28887686133384705,
      "kl": 0.0067596435546875,
      "learning_rate": 3.9904679361238526e-07,
      "loss": -0.096,
      "num_tokens": 42138424.0,
      "reward": -0.19923657178878784,
      "reward_std": 0.6053799614310265,
      "rewards/cosine_scaled_reward": -0.09961828961968422,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2770.062545776367,
      "epoch": 0.18571428571428572,
      "grad_norm": 0.3336716890335083,
      "kl": 0.003292083740234375,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0489,
      "num_tokens": 42277711.0,
      "reward": -0.5184944495558739,
      "reward_std": 0.2977650426328182,
      "rewards/cosine_scaled_reward": -0.25924722105264664,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1340.2916717529297,
      "epoch": 0.18628571428571428,
      "grad_norm": 0.6301169991493225,
      "kl": 0.020992279052734375,
      "learning_rate": 3.931425787051832e-07,
      "loss": 0.0975,
      "num_tokens": 42347565.0,
      "reward": -0.12958931922912598,
      "reward_std": 0.37651485204696655,
      "rewards/cosine_scaled_reward": -0.06479465775191784,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2393.8333587646484,
      "epoch": 0.18685714285714286,
      "grad_norm": 0.34227386116981506,
      "kl": 0.010608673095703125,
      "learning_rate": 3.902018669163384e-07,
      "loss": 0.2647,
      "num_tokens": 42467905.0,
      "reward": -0.35234155505895615,
      "reward_std": 0.40670277923345566,
      "rewards/cosine_scaled_reward": -0.17617077007889748,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2259.1250228881836,
      "epoch": 0.18742857142857142,
      "grad_norm": 0.4104394018650055,
      "kl": 0.013408660888671875,
      "learning_rate": 3.872689434630585e-07,
      "loss": 0.0312,
      "num_tokens": 42581863.0,
      "reward": -0.2085840255022049,
      "reward_std": 0.539259634912014,
      "rewards/cosine_scaled_reward": -0.10429202765226364,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3043.4375610351562,
      "epoch": 0.188,
      "grad_norm": 0.2447911649942398,
      "kl": 0.0016994476318359375,
      "learning_rate": 3.843439512918949e-07,
      "loss": 0.0559,
      "num_tokens": 42734560.0,
      "reward": 0.12149301916360855,
      "reward_std": 1.0036925375461578,
      "rewards/cosine_scaled_reward": 0.0607465049251914,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1997.8541717529297,
      "epoch": 0.18857142857142858,
      "grad_norm": 0.4679599106311798,
      "kl": 0.019840240478515625,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0583,
      "num_tokens": 42836877.0,
      "reward": 0.17294897139072418,
      "reward_std": 0.8219190053641796,
      "rewards/cosine_scaled_reward": 0.0864744782447815,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1952.9167289733887,
      "epoch": 0.18914285714285714,
      "grad_norm": 0.45652568340301514,
      "kl": 0.020021438598632812,
      "learning_rate": 3.785183306423767e-07,
      "loss": 0.0989,
      "num_tokens": 42936137.0,
      "reward": 0.13750150427222252,
      "reward_std": 0.17961042560636997,
      "rewards/cosine_scaled_reward": 0.06875075213611126,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2125.4166717529297,
      "epoch": 0.18971428571428572,
      "grad_norm": 0.3350309431552887,
      "kl": 0.014866828918457031,
      "learning_rate": 3.7561798609655373e-07,
      "loss": -0.0255,
      "num_tokens": 43043629.0,
      "reward": -0.17669325042515993,
      "reward_std": 0.4804052673280239,
      "rewards/cosine_scaled_reward": -0.08834662148728967,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2099.1250228881836,
      "epoch": 0.19028571428571428,
      "grad_norm": 0.5369983315467834,
      "kl": 0.010837554931640625,
      "learning_rate": 3.72726140684072e-07,
      "loss": 0.2866,
      "num_tokens": 43150087.0,
      "reward": -0.18136774376034737,
      "reward_std": 0.4798622354865074,
      "rewards/cosine_scaled_reward": -0.09068387281149626,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3412.8125610351562,
      "epoch": 0.19085714285714286,
      "grad_norm": 0.1947650909423828,
      "kl": 0.0010356903076171875,
      "learning_rate": 3.6984293534939737e-07,
      "loss": -0.0183,
      "num_tokens": 43320226.0,
      "reward": -0.28891574777662754,
      "reward_std": 0.44802433252334595,
      "rewards/cosine_scaled_reward": -0.14445787388831377,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.6666870117188,
      "epoch": 0.19142857142857142,
      "grad_norm": 0.4157560467720032,
      "kl": 0.0060596466064453125,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.1137,
      "num_tokens": 43414110.0,
      "reward": -0.3695136718451977,
      "reward_std": 0.4990896135568619,
      "rewards/cosine_scaled_reward": -0.18475682754069567,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2307.500045776367,
      "epoch": 0.192,
      "grad_norm": 0.29402440786361694,
      "kl": 0.0069637298583984375,
      "learning_rate": 3.641030065789562e-07,
      "loss": 0.0892,
      "num_tokens": 43530186.0,
      "reward": -0.007013067603111267,
      "reward_std": 0.6947371922433376,
      "rewards/cosine_scaled_reward": -0.0035065338015556335,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2586.0208740234375,
      "epoch": 0.19257142857142856,
      "grad_norm": 0.48218098282814026,
      "kl": 0.01905059814453125,
      "learning_rate": 3.612465628992203e-07,
      "loss": 0.0188,
      "num_tokens": 43660219.0,
      "reward": 0.34220655262470245,
      "reward_std": 0.47233958914875984,
      "rewards/cosine_scaled_reward": 0.17110328003764153,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2399.0209350585938,
      "epoch": 0.19314285714285714,
      "grad_norm": 0.3615526258945465,
      "kl": 0.007982254028320312,
      "learning_rate": 3.5839931879571725e-07,
      "loss": 0.1552,
      "num_tokens": 43782692.0,
      "reward": 0.5692258452763781,
      "reward_std": 1.0823566317558289,
      "rewards/cosine_scaled_reward": 0.28461292263818905,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3281.166748046875,
      "epoch": 0.19371428571428573,
      "grad_norm": 0.20542646944522858,
      "kl": 0.0014972686767578125,
      "learning_rate": 3.555614130391079e-07,
      "loss": 0.0219,
      "num_tokens": 43947064.0,
      "reward": -0.37549396604299545,
      "reward_std": 0.42432290129363537,
      "rewards/cosine_scaled_reward": -0.18774697184562683,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1948.2083358764648,
      "epoch": 0.19428571428571428,
      "grad_norm": 0.5580980777740479,
      "kl": 0.015346527099609375,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.1066,
      "num_tokens": 44046482.0,
      "reward": -0.09587634727358818,
      "reward_std": 0.37748729810118675,
      "rewards/cosine_scaled_reward": -0.04793817549943924,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.2917022705078,
      "epoch": 0.19485714285714287,
      "grad_norm": 0.4597311019897461,
      "kl": 0.021306991577148438,
      "learning_rate": 3.4991416936678276e-07,
      "loss": 0.2272,
      "num_tokens": 44118388.0,
      "reward": 0.12345625646412373,
      "reward_std": 0.5400365628302097,
      "rewards/cosine_scaled_reward": 0.06172813195735216,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2098.729232788086,
      "epoch": 0.19542857142857142,
      "grad_norm": 0.5311233401298523,
      "kl": 0.016597747802734375,
      "learning_rate": 3.471051066897562e-07,
      "loss": 0.1392,
      "num_tokens": 44224671.0,
      "reward": 0.09169729612767696,
      "reward_std": 0.7804772108793259,
      "rewards/cosine_scaled_reward": 0.04584864107891917,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.8333587646484,
      "epoch": 0.196,
      "grad_norm": 0.37737464904785156,
      "kl": 0.00925445556640625,
      "learning_rate": 3.4430593282358777e-07,
      "loss": 0.1748,
      "num_tokens": 44317867.0,
      "reward": -0.17036119103431702,
      "reward_std": 0.8656125217676163,
      "rewards/cosine_scaled_reward": -0.08518059551715851,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2364.9167098999023,
      "epoch": 0.19657142857142856,
      "grad_norm": 0.420017808675766,
      "kl": 0.022002220153808594,
      "learning_rate": 3.4151678419606233e-07,
      "loss": 0.0984,
      "num_tokens": 44436783.0,
      "reward": -0.07778553664684296,
      "reward_std": 0.6604952029883862,
      "rewards/cosine_scaled_reward": -0.03889276832342148,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1328.0,
      "epoch": 0.19714285714285715,
      "grad_norm": 0.6512780785560608,
      "kl": 0.020231246948242188,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0694,
      "num_tokens": 44506143.0,
      "reward": 0.46496348083019257,
      "reward_std": 0.42599966563284397,
      "rewards/cosine_scaled_reward": 0.23248173296451569,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2967.9791870117188,
      "epoch": 0.1977142857142857,
      "grad_norm": 0.27706509828567505,
      "kl": 0.00400543212890625,
      "learning_rate": 3.359691059183761e-07,
      "loss": 0.2109,
      "num_tokens": 44655182.0,
      "reward": -0.2751462832093239,
      "reward_std": 0.48510361462831497,
      "rewards/cosine_scaled_reward": -0.13757313415408134,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2848.125,
      "epoch": 0.1982857142857143,
      "grad_norm": 0.22808820009231567,
      "kl": 0.00789642333984375,
      "learning_rate": 3.3321084665422803e-07,
      "loss": -0.0171,
      "num_tokens": 44798228.0,
      "reward": -0.2193898782134056,
      "reward_std": 0.39861927926540375,
      "rewards/cosine_scaled_reward": -0.10969493724405766,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2773.1666870117188,
      "epoch": 0.19885714285714284,
      "grad_norm": 0.2631858289241791,
      "kl": 0.0020580291748046875,
      "learning_rate": 3.3046315338757026e-07,
      "loss": -0.0332,
      "num_tokens": 44937184.0,
      "reward": -0.23417676240205765,
      "reward_std": 0.4104016348719597,
      "rewards/cosine_scaled_reward": -0.11708838120102882,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3129.2084350585938,
      "epoch": 0.19942857142857143,
      "grad_norm": 0.2336844503879547,
      "kl": 0.001857757568359375,
      "learning_rate": 3.2772616003709616e-07,
      "loss": 0.0764,
      "num_tokens": 45093458.0,
      "reward": -0.11799243092536926,
      "reward_std": 0.512191891670227,
      "rewards/cosine_scaled_reward": -0.05899622291326523,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0625457763672,
      "epoch": 0.2,
      "grad_norm": 0.3072298467159271,
      "kl": 0.0224151611328125,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.1081,
      "num_tokens": 45175661.0,
      "reward": 0.5722237005829811,
      "reward_std": 0.33796822652220726,
      "rewards/cosine_scaled_reward": 0.2861118447035551,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2634.166717529297,
      "epoch": 0.20057142857142857,
      "grad_norm": 0.2797272205352783,
      "kl": 0.003326416015625,
      "learning_rate": 3.222848061454764e-07,
      "loss": 0.2635,
      "num_tokens": 45307765.0,
      "reward": -0.06509780511260033,
      "reward_std": 0.7609989158809185,
      "rewards/cosine_scaled_reward": -0.03254890255630016,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1977.9375,
      "epoch": 0.20114285714285715,
      "grad_norm": 0.4983066916465759,
      "kl": 0.020462989807128906,
      "learning_rate": 3.195807108082429e-07,
      "loss": 0.0084,
      "num_tokens": 45408382.0,
      "reward": 0.057065196335315704,
      "reward_std": 0.47916316613554955,
      "rewards/cosine_scaled_reward": 0.028532586991786957,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2337.979217529297,
      "epoch": 0.2017142857142857,
      "grad_norm": 0.30659788846969604,
      "kl": 0.005828857421875,
      "learning_rate": 3.168878457820915e-07,
      "loss": 0.1821,
      "num_tokens": 45527085.0,
      "reward": -0.3150169067084789,
      "reward_std": 0.7280392572283745,
      "rewards/cosine_scaled_reward": -0.15750844962894917,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2067.145835876465,
      "epoch": 0.2022857142857143,
      "grad_norm": 0.3004125952720642,
      "kl": 0.017545700073242188,
      "learning_rate": 3.142063423134644e-07,
      "loss": 0.0771,
      "num_tokens": 45631768.0,
      "reward": -0.013154599815607071,
      "reward_std": 0.4110375605523586,
      "rewards/cosine_scaled_reward": -0.006577307358384132,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2189.7917098999023,
      "epoch": 0.20285714285714285,
      "grad_norm": 0.4463716447353363,
      "kl": 0.0159759521484375,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0418,
      "num_tokens": 45742506.0,
      "reward": 0.08110996335744858,
      "reward_std": 0.33503045327961445,
      "rewards/cosine_scaled_reward": 0.040554989129304886,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2110.8542098999023,
      "epoch": 0.20342857142857143,
      "grad_norm": 0.37933966517448425,
      "kl": 0.02321624755859375,
      "learning_rate": 3.0887794225945143e-07,
      "loss": -0.0443,
      "num_tokens": 45849299.0,
      "reward": 0.5204504579305649,
      "reward_std": 0.5081758014857769,
      "rewards/cosine_scaled_reward": 0.26022522151470184,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3009.8958740234375,
      "epoch": 0.204,
      "grad_norm": 0.22186511754989624,
      "kl": 0.0023040771484375,
      "learning_rate": 3.062313053727671e-07,
      "loss": 0.0384,
      "num_tokens": 46000422.0,
      "reward": -0.5638711154460907,
      "reward_std": 0.42032088339328766,
      "rewards/cosine_scaled_reward": -0.28193555772304535,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2124.437530517578,
      "epoch": 0.20457142857142857,
      "grad_norm": 0.48417916893959045,
      "kl": 0.012571334838867188,
      "learning_rate": 3.0359654942835247e-07,
      "loss": 0.2297,
      "num_tokens": 46108431.0,
      "reward": 0.3528481721878052,
      "reward_std": 0.594680666923523,
      "rewards/cosine_scaled_reward": 0.17642410099506378,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2094.8542098999023,
      "epoch": 0.20514285714285715,
      "grad_norm": 0.4315508306026459,
      "kl": 0.013898849487304688,
      "learning_rate": 3.0097380284049523e-07,
      "loss": 0.053,
      "num_tokens": 46214660.0,
      "reward": -0.27906909096054733,
      "reward_std": 0.774463415145874,
      "rewards/cosine_scaled_reward": -0.1395345416967757,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2567.1250610351562,
      "epoch": 0.2057142857142857,
      "grad_norm": 0.285454660654068,
      "kl": 0.009855270385742188,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.0508,
      "num_tokens": 46344038.0,
      "reward": -0.10896847397089005,
      "reward_std": 0.5441469997167587,
      "rewards/cosine_scaled_reward": -0.05448423605412245,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2514.5,
      "epoch": 0.2062857142857143,
      "grad_norm": 0.26314839720726013,
      "kl": 0.0074367523193359375,
      "learning_rate": 2.9576484845877793e-07,
      "loss": 0.043,
      "num_tokens": 46471574.0,
      "reward": 0.004594132304191589,
      "reward_std": 0.6141269728541374,
      "rewards/cosine_scaled_reward": 0.002297069877386093,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2410.2708740234375,
      "epoch": 0.20685714285714285,
      "grad_norm": 0.2571261525154114,
      "kl": 0.0035762786865234375,
      "learning_rate": 2.931788945420058e-07,
      "loss": 0.0212,
      "num_tokens": 46592595.0,
      "reward": 0.23570144176483154,
      "reward_std": 0.9038522392511368,
      "rewards/cosine_scaled_reward": 0.11785072460770607,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2346.1250610351562,
      "epoch": 0.20742857142857143,
      "grad_norm": 0.4718479812145233,
      "kl": 0.0143280029296875,
      "learning_rate": 2.9060545772359305e-07,
      "loss": 0.0335,
      "num_tokens": 46710693.0,
      "reward": -0.08351481426507235,
      "reward_std": 0.6729232966899872,
      "rewards/cosine_scaled_reward": -0.041757403407245874,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3191.6458740234375,
      "epoch": 0.208,
      "grad_norm": 0.25456610321998596,
      "kl": 0.0029468536376953125,
      "learning_rate": 2.8804466342921987e-07,
      "loss": -0.0697,
      "num_tokens": 46869184.0,
      "reward": -0.24707527458667755,
      "reward_std": 0.4364708364009857,
      "rewards/cosine_scaled_reward": -0.12353762984275818,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2315.5209045410156,
      "epoch": 0.20857142857142857,
      "grad_norm": 0.33200034499168396,
      "kl": 0.00350189208984375,
      "learning_rate": 2.854966364683872e-07,
      "loss": -0.0,
      "num_tokens": 46985621.0,
      "reward": -0.40703647769987583,
      "reward_std": 0.5758531466126442,
      "rewards/cosine_scaled_reward": -0.20351823465898633,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2923.937530517578,
      "epoch": 0.20914285714285713,
      "grad_norm": 0.23750042915344238,
      "kl": 0.003543853759765625,
      "learning_rate": 2.829615010283344e-07,
      "loss": -0.013,
      "num_tokens": 47132150.0,
      "reward": -0.4153160899877548,
      "reward_std": 0.33962608128786087,
      "rewards/cosine_scaled_reward": -0.2076580412685871,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2619.208396911621,
      "epoch": 0.20971428571428571,
      "grad_norm": 0.4440971612930298,
      "kl": 0.019357681274414062,
      "learning_rate": 2.8043938066798645e-07,
      "loss": 0.0553,
      "num_tokens": 47263032.0,
      "reward": 0.14148210734128952,
      "reward_std": 0.9161304086446762,
      "rewards/cosine_scaled_reward": 0.0707410629838705,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2354.354248046875,
      "epoch": 0.2102857142857143,
      "grad_norm": 0.31312811374664307,
      "kl": 0.00652313232421875,
      "learning_rate": 2.7793039831193133e-07,
      "loss": 0.0197,
      "num_tokens": 47382209.0,
      "reward": 0.4818605841137469,
      "reward_std": 1.0587524473667145,
      "rewards/cosine_scaled_reward": 0.24093026970513165,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2897.875,
      "epoch": 0.21085714285714285,
      "grad_norm": 0.22899067401885986,
      "kl": 0.0037288665771484375,
      "learning_rate": 2.7543467624442956e-07,
      "loss": -0.0322,
      "num_tokens": 47527295.0,
      "reward": -0.12967145442962646,
      "reward_std": 0.5554239340126514,
      "rewards/cosine_scaled_reward": -0.06483572721481323,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2788.3958740234375,
      "epoch": 0.21142857142857144,
      "grad_norm": 0.2895820140838623,
      "kl": 0.003108978271484375,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0391,
      "num_tokens": 47666670.0,
      "reward": -0.20442558825016022,
      "reward_std": 0.4632416293025017,
      "rewards/cosine_scaled_reward": -0.10221279412508011,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2021.9166793823242,
      "epoch": 0.212,
      "grad_norm": 0.46538272500038147,
      "kl": 0.022820472717285156,
      "learning_rate": 2.7048349887476037e-07,
      "loss": 0.0362,
      "num_tokens": 47769830.0,
      "reward": 0.32230387814342976,
      "reward_std": 0.5243347375653684,
      "rewards/cosine_scaled_reward": 0.16115193953737617,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2881.1666717529297,
      "epoch": 0.21257142857142858,
      "grad_norm": 0.25469550490379333,
      "kl": 0.002590179443359375,
      "learning_rate": 2.6802828488599294e-07,
      "loss": -0.0075,
      "num_tokens": 47913934.0,
      "reward": -0.4967922382056713,
      "reward_std": 0.17979411222040653,
      "rewards/cosine_scaled_reward": -0.24839610792696476,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2935.0000610351562,
      "epoch": 0.21314285714285713,
      "grad_norm": 0.26110002398490906,
      "kl": 0.00124359130859375,
      "learning_rate": 2.655868138008171e-07,
      "loss": -0.0101,
      "num_tokens": 48061966.0,
      "reward": 0.28749898076057434,
      "reward_std": 0.6488082036376,
      "rewards/cosine_scaled_reward": 0.14374948665499687,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3053.4375610351562,
      "epoch": 0.21371428571428572,
      "grad_norm": 0.22372771799564362,
      "kl": 0.002582550048828125,
      "learning_rate": 2.631592046130896e-07,
      "loss": 0.1397,
      "num_tokens": 48215599.0,
      "reward": -0.5365136712789536,
      "reward_std": 0.4280487932264805,
      "rewards/cosine_scaled_reward": -0.2682568356394768,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1970.3333358764648,
      "epoch": 0.21428571428571427,
      "grad_norm": 0.4923068583011627,
      "kl": 0.022411346435546875,
      "learning_rate": 2.6074557564105724e-07,
      "loss": -0.1989,
      "num_tokens": 48315947.0,
      "reward": 0.056493550539016724,
      "reward_std": 0.5003200061619282,
      "rewards/cosine_scaled_reward": 0.028246776200830936,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2141.437515258789,
      "epoch": 0.21485714285714286,
      "grad_norm": 0.2769676744937897,
      "kl": 0.0077648162841796875,
      "learning_rate": 2.583460445215911e-07,
      "loss": -0.1635,
      "num_tokens": 48423980.0,
      "reward": 0.3381408303976059,
      "reward_std": 0.4275904409587383,
      "rewards/cosine_scaled_reward": 0.16907040774822235,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2033.104248046875,
      "epoch": 0.21542857142857144,
      "grad_norm": 0.3097473680973053,
      "kl": 0.0059566497802734375,
      "learning_rate": 2.5596072820445254e-07,
      "loss": 0.2215,
      "num_tokens": 48527305.0,
      "reward": -0.06256039813160896,
      "reward_std": 0.6639396920800209,
      "rewards/cosine_scaled_reward": -0.03128020092844963,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2935.1875,
      "epoch": 0.216,
      "grad_norm": 0.26714199781417847,
      "kl": 0.0024471282958984375,
      "learning_rate": 2.5358974294659373e-07,
      "loss": -0.0197,
      "num_tokens": 48674110.0,
      "reward": -0.44828396290540695,
      "reward_std": 0.32541000843048096,
      "rewards/cosine_scaled_reward": -0.22414197400212288,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2753.9792098999023,
      "epoch": 0.21657142857142858,
      "grad_norm": 0.46519750356674194,
      "kl": 0.012783050537109375,
      "learning_rate": 2.512332043064913e-07,
      "loss": -0.0604,
      "num_tokens": 48811893.0,
      "reward": -0.3965173475444317,
      "reward_std": 0.294017824344337,
      "rewards/cosine_scaled_reward": -0.19825865840539336,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2460.7708587646484,
      "epoch": 0.21714285714285714,
      "grad_norm": 0.38828662037849426,
      "kl": 0.015163421630859375,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.0109,
      "num_tokens": 48935722.0,
      "reward": 0.1380084827542305,
      "reward_std": 0.838990144431591,
      "rewards/cosine_scaled_reward": 0.06900423765182495,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3124.3333740234375,
      "epoch": 0.21771428571428572,
      "grad_norm": 0.2345580905675888,
      "kl": 0.0048999786376953125,
      "learning_rate": 2.465639255873246e-07,
      "loss": 0.0489,
      "num_tokens": 49091774.0,
      "reward": -0.5142972506582737,
      "reward_std": 0.43132054805755615,
      "rewards/cosine_scaled_reward": -0.2571486262604594,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2855.9375610351562,
      "epoch": 0.21828571428571428,
      "grad_norm": 0.2583065629005432,
      "kl": 0.0037899017333984375,
      "learning_rate": 2.4425141308231765e-07,
      "loss": -0.0191,
      "num_tokens": 49234511.0,
      "reward": -0.32289815321564674,
      "reward_std": 0.5398626290261745,
      "rewards/cosine_scaled_reward": -0.16144907008856535,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2359.1041870117188,
      "epoch": 0.21885714285714286,
      "grad_norm": 0.2668103873729706,
      "kl": 0.003749847412109375,
      "learning_rate": 2.4195380233209006e-07,
      "loss": -0.0061,
      "num_tokens": 49353196.0,
      "reward": 0.04249673895537853,
      "reward_std": 0.5773097351193428,
      "rewards/cosine_scaled_reward": 0.02124837739393115,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3301.041748046875,
      "epoch": 0.21942857142857142,
      "grad_norm": 0.21543462574481964,
      "kl": 0.0013751983642578125,
      "learning_rate": 2.3967120531894857e-07,
      "loss": -0.0099,
      "num_tokens": 49517442.0,
      "reward": -0.13908865815028548,
      "reward_std": 0.3829270862042904,
      "rewards/cosine_scaled_reward": -0.06954432907514274,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1987.6667098999023,
      "epoch": 0.22,
      "grad_norm": 0.48938560485839844,
      "kl": 0.023218154907226562,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0919,
      "num_tokens": 49618466.0,
      "reward": 0.08302738517522812,
      "reward_std": 0.382623303681612,
      "rewards/cosine_scaled_reward": 0.041513677686452866,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2939.6458740234375,
      "epoch": 0.22057142857142858,
      "grad_norm": 0.28820157051086426,
      "kl": 0.0025806427001953125,
      "learning_rate": 2.3515149676898552e-07,
      "loss": -0.034,
      "num_tokens": 49765545.0,
      "reward": -0.2633480429649353,
      "reward_std": 0.5663026869297028,
      "rewards/cosine_scaled_reward": -0.13167402520775795,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1900.6250228881836,
      "epoch": 0.22114285714285714,
      "grad_norm": 0.6309099197387695,
      "kl": 0.03489875793457031,
      "learning_rate": 2.3291460551638237e-07,
      "loss": 0.0862,
      "num_tokens": 49862631.0,
      "reward": 0.8468033410608768,
      "reward_std": 0.44973103795200586,
      "rewards/cosine_scaled_reward": 0.4234016705304384,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1040.0833587646484,
      "epoch": 0.22171428571428572,
      "grad_norm": 0.49885639548301697,
      "kl": 0.030452728271484375,
      "learning_rate": 2.306931685585657e-07,
      "loss": -0.0405,
      "num_tokens": 49917811.0,
      "reward": 0.8224863847717643,
      "reward_std": 0.9109100252389908,
      "rewards/cosine_scaled_reward": 0.41124319238588214,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1138.8541717529297,
      "epoch": 0.22228571428571428,
      "grad_norm": 0.5563756227493286,
      "kl": 0.0333251953125,
      "learning_rate": 2.2848729416523859e-07,
      "loss": -0.022,
      "num_tokens": 49977864.0,
      "reward": 0.6305245533585548,
      "reward_std": 0.6203858032822609,
      "rewards/cosine_scaled_reward": 0.3152622692286968,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2377.1666870117188,
      "epoch": 0.22285714285714286,
      "grad_norm": 0.2759566605091095,
      "kl": 0.0078125,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.1241,
      "num_tokens": 50098244.0,
      "reward": -0.42858413606882095,
      "reward_std": 0.5400056019425392,
      "rewards/cosine_scaled_reward": -0.21429206058382988,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2341.0833587646484,
      "epoch": 0.22342857142857142,
      "grad_norm": 0.4552629888057709,
      "kl": 0.01029205322265625,
      "learning_rate": 2.2412266235313973e-07,
      "loss": -0.0161,
      "num_tokens": 50215944.0,
      "reward": -0.3929029032588005,
      "reward_std": 0.24793048202991486,
      "rewards/cosine_scaled_reward": -0.19645144790410995,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3172.9583435058594,
      "epoch": 0.224,
      "grad_norm": 0.22934892773628235,
      "kl": 0.0015659332275390625,
      "learning_rate": 2.2196411766036487e-07,
      "loss": 0.0007,
      "num_tokens": 50374774.0,
      "reward": -0.2935205027461052,
      "reward_std": 0.3377592619508505,
      "rewards/cosine_scaled_reward": -0.146760243922472,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2505.979217529297,
      "epoch": 0.22457142857142856,
      "grad_norm": 0.2346705198287964,
      "kl": 0.004924774169921875,
      "learning_rate": 2.1982156097370557e-07,
      "loss": 0.0914,
      "num_tokens": 50501181.0,
      "reward": -0.4504806846380234,
      "reward_std": 0.5376848392188549,
      "rewards/cosine_scaled_reward": -0.2252403423190117,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.2708358764648,
      "epoch": 0.22514285714285714,
      "grad_norm": 0.4237222373485565,
      "kl": 0.02034759521484375,
      "learning_rate": 2.1769509671835223e-07,
      "loss": -0.1076,
      "num_tokens": 50593294.0,
      "reward": 0.2521429820917547,
      "reward_std": 0.720090851187706,
      "rewards/cosine_scaled_reward": 0.12607147614471614,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3169.479248046875,
      "epoch": 0.2257142857142857,
      "grad_norm": 0.20394039154052734,
      "kl": 0.00328826904296875,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.1086,
      "num_tokens": 50751405.0,
      "reward": -0.14242761582136154,
      "reward_std": 0.6643783301115036,
      "rewards/cosine_scaled_reward": -0.07121380046010017,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1952.1875915527344,
      "epoch": 0.22628571428571428,
      "grad_norm": 0.313944548368454,
      "kl": 0.006565093994140625,
      "learning_rate": 2.134908592756607e-07,
      "loss": 0.185,
      "num_tokens": 50851554.0,
      "reward": -0.08985854685306549,
      "reward_std": 0.3944113999605179,
      "rewards/cosine_scaled_reward": -0.04492926225066185,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2726.75,
      "epoch": 0.22685714285714287,
      "grad_norm": 0.31909239292144775,
      "kl": 0.003192901611328125,
      "learning_rate": 2.1141329099692406e-07,
      "loss": 0.0178,
      "num_tokens": 50988486.0,
      "reward": 0.4016297087073326,
      "reward_std": 0.4423007359728217,
      "rewards/cosine_scaled_reward": 0.20081482455134392,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2519.9584045410156,
      "epoch": 0.22742857142857142,
      "grad_norm": 0.3040643334388733,
      "kl": 0.0029087066650390625,
      "learning_rate": 2.0935222495670968e-07,
      "loss": 0.0979,
      "num_tokens": 51115648.0,
      "reward": 0.3028845489025116,
      "reward_std": 0.7478218302130699,
      "rewards/cosine_scaled_reward": 0.15144225861877203,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2820.8125,
      "epoch": 0.228,
      "grad_norm": 0.38658609986305237,
      "kl": 0.008646011352539062,
      "learning_rate": 2.0730776160846853e-07,
      "loss": 0.0512,
      "num_tokens": 51257047.0,
      "reward": -0.2828827500343323,
      "reward_std": 0.5114560127258301,
      "rewards/cosine_scaled_reward": -0.14144137874245644,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 805.1458549499512,
      "epoch": 0.22857142857142856,
      "grad_norm": 0.6325322985649109,
      "kl": 0.02809906005859375,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.0474,
      "num_tokens": 51301778.0,
      "reward": 1.05329729616642,
      "reward_std": 0.6592238266021013,
      "rewards/cosine_scaled_reward": 0.52664864808321,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2285.5209045410156,
      "epoch": 0.22914285714285715,
      "grad_norm": 0.2752256691455841,
      "kl": 0.0121612548828125,
      "learning_rate": 2.032690407508949e-07,
      "loss": 0.1225,
      "num_tokens": 51417255.0,
      "reward": -0.1001143604516983,
      "reward_std": 0.5159653499722481,
      "rewards/cosine_scaled_reward": -0.05005717650055885,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1967.4375457763672,
      "epoch": 0.2297142857142857,
      "grad_norm": 0.5139122009277344,
      "kl": 0.01324462890625,
      "learning_rate": 2.0127498008311922e-07,
      "loss": 0.0994,
      "num_tokens": 51518244.0,
      "reward": -0.23355354368686676,
      "reward_std": 0.4070703499019146,
      "rewards/cosine_scaled_reward": -0.11677676998078823,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1201.2916793823242,
      "epoch": 0.2302857142857143,
      "grad_norm": 0.6655187010765076,
      "kl": 0.034503936767578125,
      "learning_rate": 1.9929791578083655e-07,
      "loss": -0.0482,
      "num_tokens": 51581546.0,
      "reward": -0.011656701564788818,
      "reward_std": 0.6958346888422966,
      "rewards/cosine_scaled_reward": -0.005828343331813812,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2547.125030517578,
      "epoch": 0.23085714285714284,
      "grad_norm": 0.32773658633232117,
      "kl": 0.00426483154296875,
      "learning_rate": 1.9733794420337213e-07,
      "loss": 0.1827,
      "num_tokens": 51710132.0,
      "reward": -0.30341653153300285,
      "reward_std": 0.6679345816373825,
      "rewards/cosine_scaled_reward": -0.15170826390385628,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2246.2917098999023,
      "epoch": 0.23142857142857143,
      "grad_norm": 0.4948166608810425,
      "kl": 0.020887374877929688,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.2173,
      "num_tokens": 51823558.0,
      "reward": 0.32773495465517044,
      "reward_std": 0.43168997671455145,
      "rewards/cosine_scaled_reward": 0.16386747732758522,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2673.000030517578,
      "epoch": 0.232,
      "grad_norm": 0.4358680546283722,
      "kl": 0.006679534912109375,
      "learning_rate": 1.934696604901642e-07,
      "loss": 0.116,
      "num_tokens": 51958126.0,
      "reward": -0.44332827627658844,
      "reward_std": 0.464891217648983,
      "rewards/cosine_scaled_reward": -0.22166412882506847,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.5000228881836,
      "epoch": 0.23257142857142857,
      "grad_norm": 0.5162947773933411,
      "kl": 0.015869140625,
      "learning_rate": 1.915615368891117e-07,
      "loss": 0.1665,
      "num_tokens": 52043758.0,
      "reward": 0.42337319999933243,
      "reward_std": 0.5919698104262352,
      "rewards/cosine_scaled_reward": 0.21168660186231136,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.083366394043,
      "epoch": 0.23314285714285715,
      "grad_norm": 0.585673451423645,
      "kl": 0.019334793090820312,
      "learning_rate": 1.8967088307307e-07,
      "loss": 0.1973,
      "num_tokens": 52117886.0,
      "reward": 0.12191886268556118,
      "reward_std": 0.744186770170927,
      "rewards/cosine_scaled_reward": 0.06095943506807089,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2180.645866394043,
      "epoch": 0.2337142857142857,
      "grad_norm": 0.3679039776325226,
      "kl": 0.011453628540039062,
      "learning_rate": 1.8779779118983867e-07,
      "loss": 0.0241,
      "num_tokens": 52228245.0,
      "reward": 0.07945476472377777,
      "reward_std": 0.7460962496697903,
      "rewards/cosine_scaled_reward": 0.039727382361888885,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3364.2291870117188,
      "epoch": 0.2342857142857143,
      "grad_norm": 0.1959182769060135,
      "kl": 0.0014972686767578125,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.0497,
      "num_tokens": 52396376.0,
      "reward": -0.09703963249921799,
      "reward_std": 0.6702800244092941,
      "rewards/cosine_scaled_reward": -0.048519810661673546,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2733.8333740234375,
      "epoch": 0.23485714285714285,
      "grad_norm": 0.4071482717990875,
      "kl": 0.0033740997314453125,
      "learning_rate": 1.8410465752883758e-07,
      "loss": 0.0787,
      "num_tokens": 52533924.0,
      "reward": -0.032442666590213776,
      "reward_std": 0.6601308509707451,
      "rewards/cosine_scaled_reward": -0.01622132584452629,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3497.3958740234375,
      "epoch": 0.23542857142857143,
      "grad_norm": 0.20792892575263977,
      "kl": 0.0020923614501953125,
      "learning_rate": 1.822847957491922e-07,
      "loss": 0.0532,
      "num_tokens": 52708939.0,
      "reward": -0.6876078844070435,
      "reward_std": 0.24106604978442192,
      "rewards/cosine_scaled_reward": -0.34380391985177994,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2646.354232788086,
      "epoch": 0.236,
      "grad_norm": 0.25247225165367126,
      "kl": 0.010334014892578125,
      "learning_rate": 1.804828558898332e-07,
      "loss": 0.0482,
      "num_tokens": 52842144.0,
      "reward": 0.3487217575311661,
      "reward_std": 0.8396928831934929,
      "rewards/cosine_scaled_reward": 0.17436087876558304,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2190.041732788086,
      "epoch": 0.23657142857142857,
      "grad_norm": 0.37403154373168945,
      "kl": 0.017669677734375,
      "learning_rate": 1.7869892577476722e-07,
      "loss": 0.0377,
      "num_tokens": 52953050.0,
      "reward": 0.02339554950594902,
      "reward_std": 0.5981272980570793,
      "rewards/cosine_scaled_reward": 0.011697770096361637,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2704.625045776367,
      "epoch": 0.23714285714285716,
      "grad_norm": 0.5488267540931702,
      "kl": 0.01036834716796875,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.0983,
      "num_tokens": 53088536.0,
      "reward": -0.4232526607811451,
      "reward_std": 0.331368088722229,
      "rewards/cosine_scaled_reward": -0.21162631921470165,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3500.1458740234375,
      "epoch": 0.2377142857142857,
      "grad_norm": 0.20574693381786346,
      "kl": 0.0016450881958007812,
      "learning_rate": 1.7518544168045524e-07,
      "loss": -0.0222,
      "num_tokens": 53262651.0,
      "reward": -0.3312137499451637,
      "reward_std": 0.22738385573029518,
      "rewards/cosine_scaled_reward": -0.16560687310993671,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.87504196167,
      "epoch": 0.2382857142857143,
      "grad_norm": 0.785120964050293,
      "kl": 0.019487380981445312,
      "learning_rate": 1.7345605894346726e-07,
      "loss": 0.1738,
      "num_tokens": 53353089.0,
      "reward": 0.30004075169563293,
      "reward_std": 0.3757967611309141,
      "rewards/cosine_scaled_reward": 0.15002036094665527,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2226.2916870117188,
      "epoch": 0.23885714285714285,
      "grad_norm": 0.5053194165229797,
      "kl": 0.01377105712890625,
      "learning_rate": 1.7174502842694212e-07,
      "loss": 0.1283,
      "num_tokens": 53465675.0,
      "reward": -0.448532085865736,
      "reward_std": 0.38258010521531105,
      "rewards/cosine_scaled_reward": -0.22426604200154543,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2314.875015258789,
      "epoch": 0.23942857142857144,
      "grad_norm": 0.3890438973903656,
      "kl": 0.005889892578125,
      "learning_rate": 1.7005243352409333e-07,
      "loss": 0.018,
      "num_tokens": 53582981.0,
      "reward": -0.3278130046091974,
      "reward_std": 0.30072507075965405,
      "rewards/cosine_scaled_reward": -0.1639064911287278,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.1250228881836,
      "epoch": 0.24,
      "grad_norm": 0.5274333357810974,
      "kl": 0.02419281005859375,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.0839,
      "num_tokens": 53669915.0,
      "reward": 0.08054505288600922,
      "reward_std": 0.3769830437377095,
      "rewards/cosine_scaled_reward": 0.040272533893585205,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3035.104217529297,
      "epoch": 0.24057142857142857,
      "grad_norm": 0.25986582040786743,
      "kl": 0.002590179443359375,
      "learning_rate": 1.6672287963562852e-07,
      "loss": 0.0036,
      "num_tokens": 53821504.0,
      "reward": -0.3971610963344574,
      "reward_std": 0.42149341851472855,
      "rewards/cosine_scaled_reward": -0.1985805444419384,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3569.7916870117188,
      "epoch": 0.24114285714285713,
      "grad_norm": 0.18861441314220428,
      "kl": 0.0012664794921875,
      "learning_rate": 1.6508608292777203e-07,
      "loss": 0.0098,
      "num_tokens": 53999874.0,
      "reward": -0.4841906800866127,
      "reward_std": 0.3270874619483948,
      "rewards/cosine_scaled_reward": -0.24209534004330635,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2044.2083587646484,
      "epoch": 0.24171428571428571,
      "grad_norm": 0.3931296765804291,
      "kl": 0.015514373779296875,
      "learning_rate": 1.6346804638120098e-07,
      "loss": 0.0304,
      "num_tokens": 54103636.0,
      "reward": 0.31378229707479477,
      "reward_std": 0.7438326478004456,
      "rewards/cosine_scaled_reward": 0.15689115412533283,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3155.916748046875,
      "epoch": 0.2422857142857143,
      "grad_norm": 0.23850037157535553,
      "kl": 0.0032806396484375,
      "learning_rate": 1.6186884885673413e-07,
      "loss": 0.0152,
      "num_tokens": 54260976.0,
      "reward": -0.1346125192940235,
      "reward_std": 0.6563430987298489,
      "rewards/cosine_scaled_reward": -0.06730626057833433,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3406.041748046875,
      "epoch": 0.24285714285714285,
      "grad_norm": 0.18366895616054535,
      "kl": 0.001506805419921875,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0637,
      "num_tokens": 54430154.0,
      "reward": -0.7047783136367798,
      "reward_std": 0.22173772007226944,
      "rewards/cosine_scaled_reward": -0.3523891530930996,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2864.2083435058594,
      "epoch": 0.24342857142857144,
      "grad_norm": 0.23228085041046143,
      "kl": 0.0048847198486328125,
      "learning_rate": 1.5872728172265146e-07,
      "loss": -0.0251,
      "num_tokens": 54574260.0,
      "reward": -0.05515944957733154,
      "reward_std": 0.19446351379156113,
      "rewards/cosine_scaled_reward": -0.027579709887504578,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3079.7709350585938,
      "epoch": 0.244,
      "grad_norm": 0.21749088168144226,
      "kl": 0.0016574859619140625,
      "learning_rate": 1.5718506522858572e-07,
      "loss": -0.035,
      "num_tokens": 54728161.0,
      "reward": -0.31441882718354464,
      "reward_std": 0.4984753504395485,
      "rewards/cosine_scaled_reward": -0.15720941359177232,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2344.812530517578,
      "epoch": 0.24457142857142858,
      "grad_norm": 0.26461076736450195,
      "kl": 0.00640869140625,
      "learning_rate": 1.5566199398026147e-07,
      "loss": -0.0287,
      "num_tokens": 54847000.0,
      "reward": 0.19997850060462952,
      "reward_std": 0.6211978904902935,
      "rewards/cosine_scaled_reward": 0.09998924285173416,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2461.812530517578,
      "epoch": 0.24514285714285713,
      "grad_norm": 0.3109036982059479,
      "kl": 0.00499725341796875,
      "learning_rate": 1.5415814221002265e-07,
      "loss": 0.13,
      "num_tokens": 54971407.0,
      "reward": -0.4674905724823475,
      "reward_std": 0.33745603263378143,
      "rewards/cosine_scaled_reward": -0.2337452843785286,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3317.6459350585938,
      "epoch": 0.24571428571428572,
      "grad_norm": 0.21356281638145447,
      "kl": 0.001575469970703125,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.0489,
      "num_tokens": 55136462.0,
      "reward": 0.11995480954647064,
      "reward_std": 0.8040067255496979,
      "rewards/cosine_scaled_reward": 0.05997740477323532,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.0,
      "epoch": 0.24628571428571427,
      "grad_norm": 0.17928774654865265,
      "kl": 0.0009527206420898438,
      "learning_rate": 1.5120838934595337e-07,
      "loss": 0.0,
      "num_tokens": 55315274.0,
      "reward": -0.5093374960124493,
      "reward_std": 0.20916565880179405,
      "rewards/cosine_scaled_reward": -0.25466873310506344,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2002.8542098999023,
      "epoch": 0.24685714285714286,
      "grad_norm": 0.45603108406066895,
      "kl": 0.013976097106933594,
      "learning_rate": 1.4976263201891613e-07,
      "loss": -0.0071,
      "num_tokens": 55416811.0,
      "reward": 0.010444401763379574,
      "reward_std": 0.6464076600968838,
      "rewards/cosine_scaled_reward": 0.005222200881689787,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.4166870117188,
      "epoch": 0.24742857142857144,
      "grad_norm": 0.3496110737323761,
      "kl": 0.015367507934570312,
      "learning_rate": 1.483363816965435e-07,
      "loss": -0.0874,
      "num_tokens": 55502091.0,
      "reward": 0.2004665769636631,
      "reward_std": 0.4371606968343258,
      "rewards/cosine_scaled_reward": 0.10023329593241215,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3065.2500610351562,
      "epoch": 0.248,
      "grad_norm": 0.2298591136932373,
      "kl": 0.0030975341796875,
      "learning_rate": 1.469297078922642e-07,
      "loss": -0.0334,
      "num_tokens": 55655283.0,
      "reward": 0.04710858315229416,
      "reward_std": 0.42717816680669785,
      "rewards/cosine_scaled_reward": 0.02355429343879223,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2757.7291717529297,
      "epoch": 0.24857142857142858,
      "grad_norm": 0.4120491147041321,
      "kl": 0.012348175048828125,
      "learning_rate": 1.4554267916537495e-07,
      "loss": -0.0036,
      "num_tokens": 55794170.0,
      "reward": 0.13828672468662262,
      "reward_std": 0.4802464433014393,
      "rewards/cosine_scaled_reward": 0.06914335861802101,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2647.8125228881836,
      "epoch": 0.24914285714285714,
      "grad_norm": 0.40533170104026794,
      "kl": 0.017702102661132812,
      "learning_rate": 1.4417536311769885e-07,
      "loss": 0.1134,
      "num_tokens": 55927697.0,
      "reward": 0.27192703634500504,
      "reward_std": 0.8968725055456161,
      "rewards/cosine_scaled_reward": 0.13596352562308311,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1862.0208778381348,
      "epoch": 0.24971428571428572,
      "grad_norm": 0.9058906435966492,
      "kl": 0.04045867919921875,
      "learning_rate": 1.4282782639029128e-07,
      "loss": 0.0546,
      "num_tokens": 56022282.0,
      "reward": 0.15178360044956207,
      "reward_std": 0.6174656040966511,
      "rewards/cosine_scaled_reward": 0.07589179277420044,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2569.291717529297,
      "epoch": 0.2502857142857143,
      "grad_norm": 0.25029805302619934,
      "kl": 0.007534027099609375,
      "learning_rate": 1.4150013466019114e-07,
      "loss": 0.0397,
      "num_tokens": 56151440.0,
      "reward": -0.22741758823394775,
      "reward_std": 0.537294939160347,
      "rewards/cosine_scaled_reward": -0.11370879039168358,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1111.958381652832,
      "epoch": 0.25085714285714283,
      "grad_norm": 0.45296990871429443,
      "kl": 0.02793121337890625,
      "learning_rate": 1.4019235263722034e-07,
      "loss": 0.0986,
      "num_tokens": 56210862.0,
      "reward": -0.31671690940856934,
      "reward_std": 0.350432638078928,
      "rewards/cosine_scaled_reward": -0.15835845470428467,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1919.1458587646484,
      "epoch": 0.25142857142857145,
      "grad_norm": 0.4494737386703491,
      "kl": 0.011789321899414062,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.0479,
      "num_tokens": 56309089.0,
      "reward": 0.27276327461004257,
      "reward_std": 0.5138226337730885,
      "rewards/cosine_scaled_reward": 0.1363816224038601,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3518.625,
      "epoch": 0.252,
      "grad_norm": 0.19750487804412842,
      "kl": 0.0019445419311523438,
      "learning_rate": 1.3763677169699217e-07,
      "loss": 0.0253,
      "num_tokens": 56485063.0,
      "reward": -0.358419805765152,
      "reward_std": 0.32525915279984474,
      "rewards/cosine_scaled_reward": -0.1792098954319954,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2622.8541870117188,
      "epoch": 0.25257142857142856,
      "grad_norm": 0.331227570772171,
      "kl": 0.011693954467773438,
      "learning_rate": 1.3638909733514452e-07,
      "loss": 0.0307,
      "num_tokens": 56617584.0,
      "reward": 0.23914362490177155,
      "reward_std": 0.5278170146048069,
      "rewards/cosine_scaled_reward": 0.11957181245088577,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3383.729248046875,
      "epoch": 0.25314285714285717,
      "grad_norm": 0.1942642778158188,
      "kl": 0.00176239013671875,
      "learning_rate": 1.351615817851748e-07,
      "loss": 0.049,
      "num_tokens": 56786303.0,
      "reward": -0.6218864843249321,
      "reward_std": 0.2640235126018524,
      "rewards/cosine_scaled_reward": -0.31094324216246605,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3413.5833740234375,
      "epoch": 0.2537142857142857,
      "grad_norm": 0.1867051124572754,
      "kl": 0.0013294219970703125,
      "learning_rate": 1.3395428487445914e-07,
      "loss": 0.0401,
      "num_tokens": 56956251.0,
      "reward": -0.6013847589492798,
      "reward_std": 0.32602211087942123,
      "rewards/cosine_scaled_reward": -0.3006923794746399,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2585.5833740234375,
      "epoch": 0.2542857142857143,
      "grad_norm": 0.3707902133464813,
      "kl": 0.01132965087890625,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0547,
      "num_tokens": 57086335.0,
      "reward": 0.8475865125656128,
      "reward_std": 0.9629704058170319,
      "rewards/cosine_scaled_reward": 0.4237932413816452,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1880.5209045410156,
      "epoch": 0.25485714285714284,
      "grad_norm": 0.5581598877906799,
      "kl": 0.020392417907714844,
      "learning_rate": 1.316005813502869e-07,
      "loss": 0.19,
      "num_tokens": 57182540.0,
      "reward": -0.18871257454156876,
      "reward_std": 0.3908994784578681,
      "rewards/cosine_scaled_reward": -0.09435627982020378,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2767.1041717529297,
      "epoch": 0.25542857142857145,
      "grad_norm": 0.4027818739414215,
      "kl": 0.0086669921875,
      "learning_rate": 1.3045428945301953e-07,
      "loss": 0.051,
      "num_tokens": 57320965.0,
      "reward": -0.5574091803282499,
      "reward_std": 0.29374449513852596,
      "rewards/cosine_scaled_reward": -0.27870458643883467,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2482.3125610351562,
      "epoch": 0.256,
      "grad_norm": 0.1986325979232788,
      "kl": 0.0056400299072265625,
      "learning_rate": 1.2932844562179352e-07,
      "loss": 0.0354,
      "num_tokens": 57446116.0,
      "reward": 0.4690280854701996,
      "reward_std": 0.40538226813077927,
      "rewards/cosine_scaled_reward": 0.2345140352845192,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2598.9375610351562,
      "epoch": 0.25657142857142856,
      "grad_norm": 0.2474825382232666,
      "kl": 0.003582000732421875,
      "learning_rate": 1.2822310472864885e-07,
      "loss": 0.1014,
      "num_tokens": 57578473.0,
      "reward": 0.4089887887239456,
      "reward_std": 0.8240047469735146,
      "rewards/cosine_scaled_reward": 0.20449439622461796,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1180.7708740234375,
      "epoch": 0.2571428571428571,
      "grad_norm": 0.5087565183639526,
      "kl": 0.02840423583984375,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.2658,
      "num_tokens": 57640442.0,
      "reward": 0.25451022386550903,
      "reward_std": 0.42495069094002247,
      "rewards/cosine_scaled_reward": 0.1272551193833351,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2582.8541870117188,
      "epoch": 0.25771428571428573,
      "grad_norm": 0.47060224413871765,
      "kl": 0.0055389404296875,
      "learning_rate": 1.260741462457165e-07,
      "loss": 0.0476,
      "num_tokens": 57771091.0,
      "reward": -0.6611975803971291,
      "reward_std": 0.29068057239055634,
      "rewards/cosine_scaled_reward": -0.3305987734347582,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.93754196167,
      "epoch": 0.2582857142857143,
      "grad_norm": 0.6933012008666992,
      "kl": 0.04656410217285156,
      "learning_rate": 1.2503063339313356e-07,
      "loss": 0.1749,
      "num_tokens": 57864196.0,
      "reward": -0.06097408011555672,
      "reward_std": 0.3460182901471853,
      "rewards/cosine_scaled_reward": -0.03048703959211707,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2453.6875,
      "epoch": 0.25885714285714284,
      "grad_norm": 0.5309619307518005,
      "kl": 0.019407272338867188,
      "learning_rate": 1.2400783294793668e-07,
      "loss": 0.1398,
      "num_tokens": 57987841.0,
      "reward": -0.14627531357109547,
      "reward_std": 0.40048458334058523,
      "rewards/cosine_scaled_reward": -0.07313766423612833,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2865.8958740234375,
      "epoch": 0.25942857142857145,
      "grad_norm": 0.2787575423717499,
      "kl": 0.0030364990234375,
      "learning_rate": 1.2300579475997657e-07,
      "loss": 0.0809,
      "num_tokens": 58131152.0,
      "reward": -0.03645841544494033,
      "reward_std": 0.538176491856575,
      "rewards/cosine_scaled_reward": -0.01822919282130897,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2737.3125610351562,
      "epoch": 0.26,
      "grad_norm": 0.2434437870979309,
      "kl": 0.0023136138916015625,
      "learning_rate": 1.220245676671809e-07,
      "loss": -0.09,
      "num_tokens": 58269443.0,
      "reward": -0.0827246904373169,
      "reward_std": 0.6058220788836479,
      "rewards/cosine_scaled_reward": -0.041362347081303596,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3515.9166870117188,
      "epoch": 0.26057142857142856,
      "grad_norm": 0.22416526079177856,
      "kl": 0.0019941329956054688,
      "learning_rate": 1.2106419949317388e-07,
      "loss": 0.0015,
      "num_tokens": 58444267.0,
      "reward": -0.3426622897386551,
      "reward_std": 0.2587408199906349,
      "rewards/cosine_scaled_reward": -0.17133113741874695,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2765.5416717529297,
      "epoch": 0.2611428571428571,
      "grad_norm": 0.3764567971229553,
      "kl": 0.018613815307617188,
      "learning_rate": 1.2012473704494537e-07,
      "loss": -0.0034,
      "num_tokens": 58582509.0,
      "reward": -0.07094722986221313,
      "reward_std": 0.6748832985758781,
      "rewards/cosine_scaled_reward": -0.035473618656396866,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.041732788086,
      "epoch": 0.26171428571428573,
      "grad_norm": 0.49085304141044617,
      "kl": 0.01363372802734375,
      "learning_rate": 1.1920622611056974e-07,
      "loss": 0.4777,
      "num_tokens": 58666343.0,
      "reward": -0.2368110716342926,
      "reward_std": 0.48651882261037827,
      "rewards/cosine_scaled_reward": -0.118405532091856,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2421.395896911621,
      "epoch": 0.2622857142857143,
      "grad_norm": 0.40670979022979736,
      "kl": 0.016485214233398438,
      "learning_rate": 1.1830871145697412e-07,
      "loss": 0.1308,
      "num_tokens": 58789170.0,
      "reward": -0.010643558576703072,
      "reward_std": 0.8577997833490372,
      "rewards/cosine_scaled_reward": -0.005321786738932133,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2716.1875,
      "epoch": 0.26285714285714284,
      "grad_norm": 0.40470758080482483,
      "kl": 0.01605224609375,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.0628,
      "num_tokens": 58925859.0,
      "reward": 0.18111078813672066,
      "reward_std": 0.2746178447268903,
      "rewards/cosine_scaled_reward": 0.09055539406836033,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2102.7916870117188,
      "epoch": 0.2634285714285714,
      "grad_norm": 0.6008480191230774,
      "kl": 0.015697479248046875,
      "learning_rate": 1.1657684494105386e-07,
      "loss": 0.2683,
      "num_tokens": 59032085.0,
      "reward": -0.23030911991372705,
      "reward_std": 0.6799526810646057,
      "rewards/cosine_scaled_reward": -0.11515455157496035,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3317.604248046875,
      "epoch": 0.264,
      "grad_norm": 0.18618419766426086,
      "kl": 0.0010318756103515625,
      "learning_rate": 1.1574257748745986e-07,
      "loss": 0.0053,
      "num_tokens": 59198026.0,
      "reward": 0.1968272104859352,
      "reward_std": 0.4609448295086622,
      "rewards/cosine_scaled_reward": 0.09841361455619335,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3123.4168090820312,
      "epoch": 0.26457142857142857,
      "grad_norm": 0.19308748841285706,
      "kl": 0.0020313262939453125,
      "learning_rate": 1.1492947512799328e-07,
      "loss": 0.0844,
      "num_tokens": 59354562.0,
      "reward": -0.48374156653881073,
      "reward_std": 0.6329436413943768,
      "rewards/cosine_scaled_reward": -0.24187077954411507,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2666.5625,
      "epoch": 0.2651428571428571,
      "grad_norm": 0.29937809705734253,
      "kl": 0.010850906372070312,
      "learning_rate": 1.1413757749211602e-07,
      "loss": 0.0164,
      "num_tokens": 59489061.0,
      "reward": 0.12404083646833897,
      "reward_std": 0.621801532804966,
      "rewards/cosine_scaled_reward": 0.06202041823416948,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3420.354248046875,
      "epoch": 0.26571428571428574,
      "grad_norm": 0.20473025739192963,
      "kl": 0.0012063980102539062,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.0185,
      "num_tokens": 59658794.0,
      "reward": -0.1353940162807703,
      "reward_std": 0.5671922750771046,
      "rewards/cosine_scaled_reward": -0.06769700860604644,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2224.750045776367,
      "epoch": 0.2662857142857143,
      "grad_norm": 0.28893810510635376,
      "kl": 0.014387130737304688,
      "learning_rate": 1.1261754973965422e-07,
      "loss": 0.1965,
      "num_tokens": 59772038.0,
      "reward": -0.47738330624997616,
      "reward_std": 0.32058568112552166,
      "rewards/cosine_scaled_reward": -0.23869163822382689,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3208.9791870117188,
      "epoch": 0.26685714285714285,
      "grad_norm": 0.24921633303165436,
      "kl": 0.0014972686767578125,
      "learning_rate": 1.1188949370707787e-07,
      "loss": 0.0734,
      "num_tokens": 59932489.0,
      "reward": -0.10125970467925072,
      "reward_std": 0.4448701348155737,
      "rewards/cosine_scaled_reward": -0.050629859790205956,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2674.4375228881836,
      "epoch": 0.2674285714285714,
      "grad_norm": 0.4345049262046814,
      "kl": 0.012060165405273438,
      "learning_rate": 1.1118279056249653e-07,
      "loss": -0.0264,
      "num_tokens": 60067174.0,
      "reward": 0.15898653864860535,
      "reward_std": 0.5897807292640209,
      "rewards/cosine_scaled_reward": 0.07949326187372208,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2993.8958740234375,
      "epoch": 0.268,
      "grad_norm": 0.19168232381343842,
      "kl": 0.0040569305419921875,
      "learning_rate": 1.1049747474962444e-07,
      "loss": 0.1375,
      "num_tokens": 60216449.0,
      "reward": -0.49420715123414993,
      "reward_std": 0.4377583935856819,
      "rewards/cosine_scaled_reward": -0.24710355699062347,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1846.8958587646484,
      "epoch": 0.26857142857142857,
      "grad_norm": 0.6746844053268433,
      "kl": 0.0324859619140625,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.1055,
      "num_tokens": 60309816.0,
      "reward": 0.983911968767643,
      "reward_std": 0.7739077722653747,
      "rewards/cosine_scaled_reward": 0.49195596762001514,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2508.1666870117188,
      "epoch": 0.26914285714285713,
      "grad_norm": 0.2162114381790161,
      "kl": 0.004150390625,
      "learning_rate": 1.0919113768029517e-07,
      "loss": -0.049,
      "num_tokens": 60436376.0,
      "reward": -0.14057038724422455,
      "reward_std": 0.5592290014028549,
      "rewards/cosine_scaled_reward": -0.07028519362211227,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3250.7916870117188,
      "epoch": 0.26971428571428574,
      "grad_norm": 0.24621860682964325,
      "kl": 0.0020599365234375,
      "learning_rate": 1.0857018009286381e-07,
      "loss": 0.0899,
      "num_tokens": 60598222.0,
      "reward": -0.35493073239922523,
      "reward_std": 0.4200459411367774,
      "rewards/cosine_scaled_reward": -0.17746536619961262,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2814.0208587646484,
      "epoch": 0.2702857142857143,
      "grad_norm": 0.3132895529270172,
      "kl": 0.00676727294921875,
      "learning_rate": 1.0797073717209013e-07,
      "loss": 0.1305,
      "num_tokens": 60740639.0,
      "reward": -0.14024419710040092,
      "reward_std": 0.6155532822012901,
      "rewards/cosine_scaled_reward": -0.07012210600078106,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.2917022705078,
      "epoch": 0.27085714285714285,
      "grad_norm": 0.2247830480337143,
      "kl": 0.00786590576171875,
      "learning_rate": 1.0739283813397639e-07,
      "loss": 0.0938,
      "num_tokens": 60819421.0,
      "reward": 0.2218828909099102,
      "reward_std": 0.6827406734228134,
      "rewards/cosine_scaled_reward": 0.11094144824892282,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2696.4584197998047,
      "epoch": 0.2714285714285714,
      "grad_norm": 0.2896968722343445,
      "kl": 0.00408172607421875,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.1801,
      "num_tokens": 60955163.0,
      "reward": 0.13359621167182922,
      "reward_std": 0.7966333255171776,
      "rewards/cosine_scaled_reward": 0.06679810583591461,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3057.5,
      "epoch": 0.272,
      "grad_norm": 0.21167173981666565,
      "kl": 0.0032901763916015625,
      "learning_rate": 1.063017833182728e-07,
      "loss": 0.0928,
      "num_tokens": 61108295.0,
      "reward": -0.6038347482681274,
      "reward_std": 0.3766014724969864,
      "rewards/cosine_scaled_reward": -0.30191735550761223,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2832.604217529297,
      "epoch": 0.2725714285714286,
      "grad_norm": 0.28865718841552734,
      "kl": 0.007213592529296875,
      "learning_rate": 1.0578868071715544e-07,
      "loss": -0.0879,
      "num_tokens": 61251244.0,
      "reward": -0.4621117692440748,
      "reward_std": 0.3360144942998886,
      "rewards/cosine_scaled_reward": -0.23105588043108582,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3392.6458740234375,
      "epoch": 0.27314285714285713,
      "grad_norm": 0.18357343971729279,
      "kl": 0.001720428466796875,
      "learning_rate": 1.0529722834905125e-07,
      "loss": -0.032,
      "num_tokens": 61420799.0,
      "reward": -0.3251003101468086,
      "reward_std": 0.37241343408823013,
      "rewards/cosine_scaled_reward": -0.16255014389753342,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2110.25004196167,
      "epoch": 0.2737142857142857,
      "grad_norm": 0.452695369720459,
      "kl": 0.0191802978515625,
      "learning_rate": 1.0482745016665526e-07,
      "loss": 0.1311,
      "num_tokens": 61527719.0,
      "reward": 0.4394497722387314,
      "reward_std": 0.30791839864104986,
      "rewards/cosine_scaled_reward": 0.2197248861193657,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3178.5208740234375,
      "epoch": 0.2742857142857143,
      "grad_norm": 0.2389601320028305,
      "kl": 0.002948760986328125,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0376,
      "num_tokens": 61687032.0,
      "reward": 0.2565951645374298,
      "reward_std": 0.6847127676010132,
      "rewards/cosine_scaled_reward": 0.1282975897192955,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2711.5625228881836,
      "epoch": 0.27485714285714286,
      "grad_norm": 0.6793060302734375,
      "kl": 0.02097320556640625,
      "learning_rate": 1.0395300688680625e-07,
      "loss": -0.0007,
      "num_tokens": 61823763.0,
      "reward": -0.17994186095893383,
      "reward_std": 0.5131379179656506,
      "rewards/cosine_scaled_reward": -0.08997092954814434,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2249.0833435058594,
      "epoch": 0.2754285714285714,
      "grad_norm": 0.33344191312789917,
      "kl": 0.0082855224609375,
      "learning_rate": 1.0354838440848501e-07,
      "loss": 0.1291,
      "num_tokens": 61938103.0,
      "reward": 0.008612923324108124,
      "reward_std": 0.5992633532732725,
      "rewards/cosine_scaled_reward": 0.004306461662054062,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2914.4791870117188,
      "epoch": 0.276,
      "grad_norm": 0.26747846603393555,
      "kl": 0.0048503875732421875,
      "learning_rate": 1.0316552135205837e-07,
      "loss": 0.061,
      "num_tokens": 62083878.0,
      "reward": -0.18974144756793976,
      "reward_std": 0.6735084727406502,
      "rewards/cosine_scaled_reward": -0.09487072005867958,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.7292289733887,
      "epoch": 0.2765714285714286,
      "grad_norm": 0.5921136736869812,
      "kl": 0.02447509765625,
      "learning_rate": 1.0280443637773163e-07,
      "loss": 0.0267,
      "num_tokens": 62176649.0,
      "reward": -0.4410746842622757,
      "reward_std": 0.301199066452682,
      "rewards/cosine_scaled_reward": -0.22053734213113785,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.4791870117188,
      "epoch": 0.27714285714285714,
      "grad_norm": 0.36696869134902954,
      "kl": 0.01856231689453125,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.2368,
      "num_tokens": 62263576.0,
      "reward": 0.08185825496912003,
      "reward_std": 0.23910308256745338,
      "rewards/cosine_scaled_reward": 0.04092914238572121,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.6250534057617,
      "epoch": 0.2777142857142857,
      "grad_norm": 0.44206833839416504,
      "kl": 0.017059326171875,
      "learning_rate": 1.0214767000817596e-07,
      "loss": 0.2164,
      "num_tokens": 62354842.0,
      "reward": -0.1533157378435135,
      "reward_std": 0.4944304316304624,
      "rewards/cosine_scaled_reward": -0.07665786519646645,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1240.7708358764648,
      "epoch": 0.2782857142857143,
      "grad_norm": 0.6005072593688965,
      "kl": 0.02964019775390625,
      "learning_rate": 1.0185202062281336e-07,
      "loss": 0.2567,
      "num_tokens": 62420027.0,
      "reward": 0.3417629040777683,
      "reward_std": 0.5913028866052628,
      "rewards/cosine_scaled_reward": 0.17088143806904554,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3259.6041870117188,
      "epoch": 0.27885714285714286,
      "grad_norm": 0.2043127417564392,
      "kl": 0.0017719268798828125,
      "learning_rate": 1.0157821333772304e-07,
      "loss": 0.0477,
      "num_tokens": 62583244.0,
      "reward": 0.000857822597026825,
      "reward_std": 0.6294433549046516,
      "rewards/cosine_scaled_reward": 0.0004289112985134125,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2507.875045776367,
      "epoch": 0.2794285714285714,
      "grad_norm": 0.41613826155662537,
      "kl": 0.0082244873046875,
      "learning_rate": 1.013262614978859e-07,
      "loss": 0.171,
      "num_tokens": 62708254.0,
      "reward": -0.3011748939752579,
      "reward_std": 0.6403844729065895,
      "rewards/cosine_scaled_reward": -0.15058743953704834,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2246.9166870117188,
      "epoch": 0.28,
      "grad_norm": 0.3775407671928406,
      "kl": 0.00557708740234375,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.2481,
      "num_tokens": 62821314.0,
      "reward": 0.13066441006958485,
      "reward_std": 0.39571982994675636,
      "rewards/cosine_scaled_reward": 0.06533220689743757,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2470.5417098999023,
      "epoch": 0.2805714285714286,
      "grad_norm": 0.34662094712257385,
      "kl": 0.010288238525390625,
      "learning_rate": 1.0088797220727779e-07,
      "loss": 0.1097,
      "num_tokens": 62945936.0,
      "reward": -0.09428609162569046,
      "reward_std": 0.7784368544816971,
      "rewards/cosine_scaled_reward": -0.04714304953813553,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3139.291748046875,
      "epoch": 0.28114285714285714,
      "grad_norm": 0.23511607944965363,
      "kl": 0.0025539398193359375,
      "learning_rate": 1.0070165611810855e-07,
      "loss": 0.0161,
      "num_tokens": 63102874.0,
      "reward": -0.3465424180030823,
      "reward_std": 0.5582562312483788,
      "rewards/cosine_scaled_reward": -0.17327120155096054,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2491.479232788086,
      "epoch": 0.2817142857142857,
      "grad_norm": 0.3599470555782318,
      "kl": 0.013715744018554688,
      "learning_rate": 1.005372381963547e-07,
      "loss": 0.0382,
      "num_tokens": 63228357.0,
      "reward": 0.5691114738583565,
      "reward_std": 1.0396928787231445,
      "rewards/cosine_scaled_reward": 0.2845557164400816,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2203.6667098999023,
      "epoch": 0.2822857142857143,
      "grad_norm": 0.40486791729927063,
      "kl": 0.01262664794921875,
      "learning_rate": 1.0039472645551372e-07,
      "loss": -0.0022,
      "num_tokens": 63339497.0,
      "reward": 0.44267672300338745,
      "reward_std": 0.6116999462246895,
      "rewards/cosine_scaled_reward": 0.22133836150169373,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3396.854248046875,
      "epoch": 0.28285714285714286,
      "grad_norm": 0.18696698546409607,
      "kl": 0.001537322998046875,
      "learning_rate": 1.002741278414069e-07,
      "loss": -0.024,
      "num_tokens": 63508690.0,
      "reward": -0.15206171572208405,
      "reward_std": 0.6857472285628319,
      "rewards/cosine_scaled_reward": -0.07603085786104202,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1400.7708587646484,
      "epoch": 0.2834285714285714,
      "grad_norm": 0.508706271648407,
      "kl": 0.018621444702148438,
      "learning_rate": 1.0017544823184055e-07,
      "loss": 0.2565,
      "num_tokens": 63581303.0,
      "reward": -0.016182963736355305,
      "reward_std": 0.6838839948177338,
      "rewards/cosine_scaled_reward": -0.008091474417597055,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3447.9583740234375,
      "epoch": 0.284,
      "grad_norm": 0.20476852357387543,
      "kl": 0.0011720657348632812,
      "learning_rate": 1.0009869243631952e-07,
      "loss": 0.0013,
      "num_tokens": 63753261.0,
      "reward": -0.3728942945599556,
      "reward_std": 0.4060557000339031,
      "rewards/cosine_scaled_reward": -0.18644713005051017,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2509.0833587646484,
      "epoch": 0.2845714285714286,
      "grad_norm": 0.3915875554084778,
      "kl": 0.0099945068359375,
      "learning_rate": 1.000438641958131e-07,
      "loss": 0.1323,
      "num_tokens": 63881005.0,
      "reward": -0.10937303304672241,
      "reward_std": 0.6931461840867996,
      "rewards/cosine_scaled_reward": -0.054686516523361206,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2609.8959197998047,
      "epoch": 0.28514285714285714,
      "grad_norm": 0.31072404980659485,
      "kl": 0.005950927734375,
      "learning_rate": 1.0001096618257236e-07,
      "loss": 0.0247,
      "num_tokens": 64012160.0,
      "reward": 0.3292629097122699,
      "reward_std": 0.9715245068073273,
      "rewards/cosine_scaled_reward": 0.16463144560111687,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1381.4583435058594,
      "epoch": 0.2857142857142857,
      "grad_norm": 0.41521623730659485,
      "kl": 0.023681640625,
      "learning_rate": 1e-07,
      "loss": 0.032,
      "num_tokens": 64084134.0,
      "reward": -0.11208531260490417,
      "reward_std": 0.5163028538227081,
      "rewards/cosine_scaled_reward": -0.05604265257716179,
      "step": 500
    },
    {
      "epoch": 0.2857142857142857,
      "step": 500,
      "total_flos": 0.0,
      "train_loss": 0.04327104251924902,
      "train_runtime": 47844.8241,
      "train_samples_per_second": 0.502,
      "train_steps_per_second": 0.01
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}