{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.27938535222510474,
  "eval_steps": 500,
  "global_step": 200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 6335.173723493303,
      "epoch": 0.0013969267611255239,
      "grad_norm": 0.1530565852782136,
      "learning_rate": 0.0,
      "loss": 0.0119,
      "num_tokens": 1330201.0,
      "reward": 0.3924093656241894,
      "reward_std": 0.32559600685323986,
      "rewards/reward_func": 0.3924093528517655,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "epoch": 0.0027938535222510478,
      "grad_norm": 0.15303408357738274,
      "learning_rate": 1.3888888888888887e-08,
      "loss": 0.0119,
      "step": 2
    },
    {
      "clip_ratio": 0.010408175710056509,
      "epoch": 0.004190780283376571,
      "grad_norm": 0.1764633199425111,
      "learning_rate": 2.7777777777777774e-08,
      "loss": 0.0125,
      "step": 3
    },
    {
      "clip_ratio": 0.010680466823812042,
      "epoch": 0.0055877070445020955,
      "grad_norm": 0.2275277428050235,
      "learning_rate": 4.166666666666666e-08,
      "loss": 0.0126,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5737.668526785715,
      "epoch": 0.006984633805627619,
      "grad_norm": 0.19107039038527468,
      "learning_rate": 5.555555555555555e-08,
      "loss": 0.0293,
      "num_tokens": 2545372.0,
      "reward": 0.4294457712343761,
      "reward_std": 0.37008823986564365,
      "rewards/reward_func": 0.4294457531401089,
      "step": 5
    },
    {
      "clip_ratio": 0.010828850524766105,
      "epoch": 0.008381560566753142,
      "grad_norm": 0.2240354781456917,
      "learning_rate": 6.944444444444444e-08,
      "loss": 0.0301,
      "step": 6
    },
    {
      "clip_ratio": 0.010841753533376115,
      "epoch": 0.009778487327878668,
      "grad_norm": 0.2767916742737982,
      "learning_rate": 8.333333333333333e-08,
      "loss": 0.0301,
      "step": 7
    },
    {
      "clip_ratio": 0.010758176312914916,
      "epoch": 0.011175414089004191,
      "grad_norm": 0.2668705057271519,
      "learning_rate": 9.722222222222222e-08,
      "loss": 0.0301,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6511.857439313616,
      "epoch": 0.012572340850129714,
      "grad_norm": 0.19266419886931196,
      "learning_rate": 1.111111111111111e-07,
      "loss": 0.0051,
      "num_tokens": 3911895.0,
      "reward": 0.32038674876093864,
      "reward_std": 0.3298808889729636,
      "rewards/reward_func": 0.32038673226322445,
      "step": 9
    },
    {
      "clip_ratio": 0.012134518141725234,
      "epoch": 0.013969267611255238,
      "grad_norm": 0.21591989333017872,
      "learning_rate": 1.25e-07,
      "loss": 0.0058,
      "step": 10
    },
    {
      "clip_ratio": 0.012228245115173715,
      "epoch": 0.015366194372380763,
      "grad_norm": 0.20676433229073893,
      "learning_rate": 1.3888888888888888e-07,
      "loss": 0.0058,
      "step": 11
    },
    {
      "clip_ratio": 0.012062976469418831,
      "epoch": 0.016763121133506285,
      "grad_norm": 0.2100849809088676,
      "learning_rate": 1.527777777777778e-07,
      "loss": 0.0058,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5776.398141043527,
      "epoch": 0.01816004789463181,
      "grad_norm": 0.21639349074679098,
      "learning_rate": 1.6666666666666665e-07,
      "loss": 0.0233,
      "num_tokens": 5132268.0,
      "reward": 0.4814984883580889,
      "reward_std": 0.3754643425345421,
      "rewards/reward_func": 0.48149845749139786,
      "step": 13
    },
    {
      "clip_ratio": 0.010207080947501319,
      "epoch": 0.019556974655757335,
      "grad_norm": 0.22611722201173162,
      "learning_rate": 1.8055555555555554e-07,
      "loss": 0.0241,
      "step": 14
    },
    {
      "clip_ratio": 0.010003214618856353,
      "epoch": 0.02095390141688286,
      "grad_norm": 0.21387408972131952,
      "learning_rate": 1.9444444444444445e-07,
      "loss": 0.024,
      "step": 15
    },
    {
      "clip_ratio": 0.010145484004169703,
      "epoch": 0.022350828178008382,
      "grad_norm": 0.2965931721160837,
      "learning_rate": 2.0833333333333333e-07,
      "loss": 0.024,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6396.923740931919,
      "epoch": 0.023747754939133905,
      "grad_norm": 0.17812701051388663,
      "learning_rate": 2.222222222222222e-07,
      "loss": 0.0143,
      "num_tokens": 6474962.0,
      "reward": 0.4179248466555561,
      "reward_std": 0.3605238295027188,
      "rewards/reward_func": 0.41792482949261156,
      "step": 17
    },
    {
      "clip_ratio": 0.010267588176897593,
      "epoch": 0.02514468170025943,
      "grad_norm": 0.18587492539762027,
      "learning_rate": 2.361111111111111e-07,
      "loss": 0.015,
      "step": 18
    },
    {
      "clip_ratio": 0.010148471055020179,
      "epoch": 0.026541608461384952,
      "grad_norm": 0.1810944495339226,
      "learning_rate": 2.5e-07,
      "loss": 0.0149,
      "step": 19
    },
    {
      "clip_ratio": 0.01020466532957341,
      "epoch": 0.027938535222510476,
      "grad_norm": 0.1924193708866086,
      "learning_rate": 2.638888888888889e-07,
      "loss": 0.015,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5085.755362374442,
      "epoch": 0.029335461983636,
      "grad_norm": 0.3165497451368565,
      "learning_rate": 2.7777777777777776e-07,
      "loss": 0.0104,
      "num_tokens": 7560523.0,
      "reward": 0.464439152606896,
      "reward_std": 0.3689843277846064,
      "rewards/reward_func": 0.46443913131952286,
      "step": 21
    },
    {
      "clip_ratio": 0.010546388570219278,
      "epoch": 0.030732388744761526,
      "grad_norm": 0.3395774959095009,
      "learning_rate": 2.916666666666667e-07,
      "loss": 0.0113,
      "step": 22
    },
    {
      "clip_ratio": 0.010336563357019,
      "epoch": 0.032129315505887046,
      "grad_norm": 0.31582946500885495,
      "learning_rate": 3.055555555555556e-07,
      "loss": 0.0112,
      "step": 23
    },
    {
      "clip_ratio": 0.010099471979109305,
      "epoch": 0.03352624226701257,
      "grad_norm": 0.36247326697578075,
      "learning_rate": 3.194444444444444e-07,
      "loss": 0.0112,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6461.342215401785,
      "epoch": 0.03492316902813809,
      "grad_norm": 0.15567466235487185,
      "learning_rate": 3.333333333333333e-07,
      "loss": 0.0193,
      "num_tokens": 8914764.0,
      "reward": 0.38339252876383917,
      "reward_std": 0.3311073939715113,
      "rewards/reward_func": 0.3833925181201526,
      "step": 25
    },
    {
      "clip_ratio": 0.009515652732391442,
      "epoch": 0.03632009578926362,
      "grad_norm": 0.25805173133838866,
      "learning_rate": 3.472222222222222e-07,
      "loss": 0.0199,
      "step": 26
    },
    {
      "clip_ratio": 0.009263422406677688,
      "epoch": 0.03771702255038915,
      "grad_norm": 0.4090162080972072,
      "learning_rate": 3.6111111111111107e-07,
      "loss": 0.0199,
      "step": 27
    },
    {
      "clip_ratio": 0.009094293162758862,
      "epoch": 0.03911394931151467,
      "grad_norm": 0.14691760483745145,
      "learning_rate": 3.75e-07,
      "loss": 0.0198,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5968.250279017857,
      "epoch": 0.040510876072640194,
      "grad_norm": 0.16406943154337036,
      "learning_rate": 3.888888888888889e-07,
      "loss": 0.0115,
      "num_tokens": 10173741.0,
      "reward": 0.3594382884246962,
      "reward_std": 0.32283720054796766,
      "rewards/reward_func": 0.35943826926606043,
      "step": 29
    },
    {
      "clip_ratio": 0.01090411888435483,
      "epoch": 0.04190780283376572,
      "grad_norm": 0.21927196182565817,
      "learning_rate": 4.027777777777778e-07,
      "loss": 0.0124,
      "step": 30
    },
    {
      "clip_ratio": 0.010733458446338773,
      "epoch": 0.04330472959489124,
      "grad_norm": 0.3447213393816689,
      "learning_rate": 4.1666666666666667e-07,
      "loss": 0.0121,
      "step": 31
    },
    {
      "clip_ratio": 0.010630057725523199,
      "epoch": 0.044701656356016764,
      "grad_norm": 0.3620568701207513,
      "learning_rate": 4.3055555555555555e-07,
      "loss": 0.0121,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5812.648210797991,
      "epoch": 0.04609858311714229,
      "grad_norm": 0.16987419224516487,
      "learning_rate": 4.444444444444444e-07,
      "loss": 0.0179,
      "num_tokens": 11403427.0,
      "reward": 0.3713971259338515,
      "reward_std": 0.3195132836699486,
      "rewards/reward_func": 0.3713971110326903,
      "step": 33
    },
    {
      "clip_ratio": 0.010423385953929807,
      "epoch": 0.04749550987826781,
      "grad_norm": 0.16329217057193554,
      "learning_rate": 4.5833333333333327e-07,
      "loss": 0.0185,
      "step": 34
    },
    {
      "clip_ratio": 0.010347368278806763,
      "epoch": 0.048892436639393334,
      "grad_norm": 0.15422959333602954,
      "learning_rate": 4.722222222222222e-07,
      "loss": 0.0185,
      "step": 35
    },
    {
      "clip_ratio": 0.010140369646251202,
      "epoch": 0.05028936340051886,
      "grad_norm": 0.14689712083262277,
      "learning_rate": 4.861111111111111e-07,
      "loss": 0.0184,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5546.617606026785,
      "epoch": 0.05168629016164438,
      "grad_norm": 0.23439115342799705,
      "learning_rate": 5e-07,
      "loss": 0.0161,
      "num_tokens": 12578819.0,
      "reward": 0.4297656629766737,
      "reward_std": 0.357943703021322,
      "rewards/reward_func": 0.42976564009274754,
      "step": 37
    },
    {
      "clip_ratio": 0.010506943932601384,
      "epoch": 0.053083216922769905,
      "grad_norm": 0.28362735010406454,
      "learning_rate": 5.138888888888889e-07,
      "loss": 0.0168,
      "step": 38
    },
    {
      "clip_ratio": 0.010338681018246072,
      "epoch": 0.05448014368389543,
      "grad_norm": 0.22707404637725537,
      "learning_rate": 5.277777777777777e-07,
      "loss": 0.0167,
      "step": 39
    },
    {
      "clip_ratio": 0.010024449788033962,
      "epoch": 0.05587707044502095,
      "grad_norm": 0.17811231341381648,
      "learning_rate": 5.416666666666666e-07,
      "loss": 0.0165,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6501.444231305803,
      "epoch": 0.057273997206146475,
      "grad_norm": 0.19673275542337104,
      "learning_rate": 5.555555555555555e-07,
      "loss": 0.0203,
      "num_tokens": 13942392.0,
      "reward": 0.39034509126629147,
      "reward_std": 0.3664030634931156,
      "rewards/reward_func": 0.3903450625283377,
      "step": 41
    },
    {
      "clip_ratio": 0.010402361529746227,
      "epoch": 0.058670923967272,
      "grad_norm": 0.46687244963414065,
      "learning_rate": 5.694444444444444e-07,
      "loss": 0.0211,
      "step": 42
    },
    {
      "clip_ratio": 0.010496154893189669,
      "epoch": 0.06006785072839753,
      "grad_norm": 0.19428194405187207,
      "learning_rate": 5.833333333333334e-07,
      "loss": 0.0209,
      "step": 43
    },
    {
      "clip_ratio": 0.010158670766811286,
      "epoch": 0.06146477748952305,
      "grad_norm": 0.15306555166745645,
      "learning_rate": 5.972222222222222e-07,
      "loss": 0.0208,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6369.643048967634,
      "epoch": 0.06286170425064858,
      "grad_norm": 0.1871015916141642,
      "learning_rate": 6.111111111111112e-07,
      "loss": 0.0161,
      "num_tokens": 15279078.0,
      "reward": 0.36025070399045944,
      "reward_std": 0.35143060450042996,
      "rewards/reward_func": 0.360250677381243,
      "step": 45
    },
    {
      "clip_ratio": 0.01079715442444597,
      "epoch": 0.06425863101177409,
      "grad_norm": 0.6294866160536174,
      "learning_rate": 6.249999999999999e-07,
      "loss": 0.0169,
      "step": 46
    },
    {
      "clip_ratio": 0.010736021545848675,
      "epoch": 0.06565555777289962,
      "grad_norm": 2.137657977104326,
      "learning_rate": 6.388888888888888e-07,
      "loss": 0.0171,
      "step": 47
    },
    {
      "clip_ratio": 0.010864090407267213,
      "epoch": 0.06705248453402514,
      "grad_norm": 0.18428434234054034,
      "learning_rate": 6.527777777777777e-07,
      "loss": 0.0166,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6655.908412388393,
      "epoch": 0.06844941129515067,
      "grad_norm": 0.16953542603782387,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.0148,
      "num_tokens": 16672949.0,
      "reward": 0.35046685327376637,
      "reward_std": 0.30536604832325664,
      "rewards/reward_func": 0.3504668373082365,
      "step": 49
    },
    {
      "clip_ratio": 0.010905002310339893,
      "epoch": 0.06984633805627619,
      "grad_norm": 1.9607105946099048,
      "learning_rate": 6.805555555555556e-07,
      "loss": 0.0153,
      "step": 50
    },
    {
      "clip_ratio": 0.011137549964977162,
      "epoch": 0.07124326481740172,
      "grad_norm": 0.16619610980435887,
      "learning_rate": 6.944444444444444e-07,
      "loss": 0.0153,
      "step": 51
    },
    {
      "clip_ratio": 0.011474365035870246,
      "epoch": 0.07264019157852725,
      "grad_norm": 0.13731779782413311,
      "learning_rate": 7.083333333333334e-07,
      "loss": 0.0151,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6259.326799665178,
      "epoch": 0.07403711833965276,
      "grad_norm": 0.18005417578145988,
      "learning_rate": 7.222222222222221e-07,
      "loss": 0.0124,
      "num_tokens": 17987081.0,
      "reward": 0.3450017161667347,
      "reward_std": 0.3385175902928625,
      "rewards/reward_func": 0.34500169487936155,
      "step": 53
    },
    {
      "clip_ratio": 0.012061000069869416,
      "epoch": 0.0754340451007783,
      "grad_norm": 0.3653574862365132,
      "learning_rate": 7.361111111111111e-07,
      "loss": 0.0132,
      "step": 54
    },
    {
      "clip_ratio": 0.01210923127031752,
      "epoch": 0.07683097186190381,
      "grad_norm": 0.1572526876806962,
      "learning_rate": 7.5e-07,
      "loss": 0.013,
      "step": 55
    },
    {
      "clip_ratio": 0.012545851857534476,
      "epoch": 0.07822789862302934,
      "grad_norm": 0.1348151811096135,
      "learning_rate": 7.638888888888888e-07,
      "loss": 0.0128,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 6945.081944056919,
      "epoch": 0.07962482538415486,
      "grad_norm": 0.1355243529384343,
      "learning_rate": 7.777777777777778e-07,
      "loss": 0.0203,
      "num_tokens": 19437527.0,
      "reward": 0.2853341962077788,
      "reward_std": 0.28026825402464184,
      "rewards/reward_func": 0.28533417971006464,
      "step": 57
    },
    {
      "clip_ratio": 0.010294904433456915,
      "epoch": 0.08102175214528039,
      "grad_norm": 1.4215854683138216,
      "learning_rate": 7.916666666666666e-07,
      "loss": 0.0211,
      "step": 58
    },
    {
      "clip_ratio": 0.010296880932790893,
      "epoch": 0.0824186789064059,
      "grad_norm": 0.1328390444888187,
      "learning_rate": 8.055555555555556e-07,
      "loss": 0.0208,
      "step": 59
    },
    {
      "clip_ratio": 0.010402131692639418,
      "epoch": 0.08381560566753143,
      "grad_norm": 0.10630427559017747,
      "learning_rate": 8.194444444444443e-07,
      "loss": 0.0207,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5832.984967912947,
      "epoch": 0.08521253242865695,
      "grad_norm": 0.2568979639198368,
      "learning_rate": 8.333333333333333e-07,
      "loss": 0.0135,
      "num_tokens": 20669861.0,
      "reward": 0.37696379157049315,
      "reward_std": 0.3524509519338608,
      "rewards/reward_func": 0.37696376868656706,
      "step": 61
    },
    {
      "clip_ratio": 0.011475847900978156,
      "epoch": 0.08660945918978248,
      "grad_norm": 0.26568198717380825,
      "learning_rate": 8.472222222222222e-07,
      "loss": 0.0142,
      "step": 62
    },
    {
      "clip_ratio": 0.011013164584125792,
      "epoch": 0.088006385950908,
      "grad_norm": 0.5160311893436309,
      "learning_rate": 8.611111111111111e-07,
      "loss": 0.014,
      "step": 63
    },
    {
      "clip_ratio": 0.010899153238694583,
      "epoch": 0.08940331271203353,
      "grad_norm": 0.16319985648001487,
      "learning_rate": 8.75e-07,
      "loss": 0.0138,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5130.444091796875,
      "epoch": 0.09080023947315904,
      "grad_norm": 0.28767641683692746,
      "learning_rate": 8.888888888888888e-07,
      "loss": 0.0025,
      "num_tokens": 21764127.0,
      "reward": 0.46981193338121685,
      "reward_std": 0.3410501836666039,
      "rewards/reward_func": 0.4698119152869497,
      "step": 65
    },
    {
      "clip_ratio": 0.011167634411581926,
      "epoch": 0.09219716623428457,
      "grad_norm": 1786685.0973493713,
      "learning_rate": 9.027777777777778e-07,
      "loss": 31.3025,
      "step": 66
    },
    {
      "clip_ratio": 0.011178254083331143,
      "epoch": 0.09359409299541009,
      "grad_norm": 0.2699319084211939,
      "learning_rate": 9.166666666666665e-07,
      "loss": 0.0029,
      "step": 67
    },
    {
      "clip_ratio": 0.010996093108717884,
      "epoch": 0.09499101975653562,
      "grad_norm": 0.1796173018225043,
      "learning_rate": 9.305555555555555e-07,
      "loss": 0.0026,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5374.097150530134,
      "epoch": 0.09638794651766115,
      "grad_norm": 0.29248413368918635,
      "learning_rate": 9.444444444444444e-07,
      "loss": 0.0204,
      "num_tokens": 22905462.0,
      "reward": 0.42480308030332836,
      "reward_std": 0.3633535067949976,
      "rewards/reward_func": 0.4248030515653746,
      "step": 69
    },
    {
      "clip_ratio": 0.010870669968426228,
      "epoch": 0.09778487327878667,
      "grad_norm": 0.2916766016433164,
      "learning_rate": 9.583333333333334e-07,
      "loss": 0.0211,
      "step": 70
    },
    {
      "clip_ratio": 0.010642989072948694,
      "epoch": 0.0991818000399122,
      "grad_norm": 0.2574669164587784,
      "learning_rate": 9.722222222222222e-07,
      "loss": 0.0209,
      "step": 71
    },
    {
      "clip_ratio": 0.010603911941871047,
      "epoch": 0.10057872680103772,
      "grad_norm": 0.3923293286285201,
      "learning_rate": 9.861111111111112e-07,
      "loss": 0.0206,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4780.178763253348,
      "epoch": 0.10197565356216325,
      "grad_norm": 0.3357433852104651,
      "learning_rate": 1e-06,
      "loss": 0.0111,
      "num_tokens": 23928872.0,
      "reward": 0.49998574065310614,
      "reward_std": 0.3773101898176329,
      "rewards/reward_func": 0.4999857119151524,
      "step": 73
    },
    {
      "clip_ratio": 0.012297127628698945,
      "epoch": 0.10337258032328876,
      "grad_norm": 0.24331462448033314,
      "learning_rate": 1e-06,
      "loss": 0.0117,
      "step": 74
    },
    {
      "clip_ratio": 0.012708199177203434,
      "epoch": 0.10476950708441429,
      "grad_norm": 0.6107245136629853,
      "learning_rate": 1e-06,
      "loss": 0.0114,
      "step": 75
    },
    {
      "clip_ratio": 0.013628917180800013,
      "epoch": 0.10616643384553981,
      "grad_norm": 0.1571665107730879,
      "learning_rate": 1e-06,
      "loss": 0.0112,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5568.459463936942,
      "epoch": 0.10756336060666534,
      "grad_norm": 0.3303048447870775,
      "learning_rate": 1e-06,
      "loss": 0.0085,
      "num_tokens": 25108534.0,
      "reward": 0.3940093676958765,
      "reward_std": 0.32062976009079386,
      "rewards/reward_func": 0.39400935066597803,
      "step": 77
    },
    {
      "clip_ratio": 0.012926211846726281,
      "epoch": 0.10896028736779086,
      "grad_norm": 0.4586726422820888,
      "learning_rate": 1e-06,
      "loss": 0.0092,
      "step": 78
    },
    {
      "clip_ratio": 0.013429554013003196,
      "epoch": 0.11035721412891639,
      "grad_norm": 0.46850871504202196,
      "learning_rate": 1e-06,
      "loss": 0.0089,
      "step": 79
    },
    {
      "clip_ratio": 0.01421830172850085,
      "epoch": 0.1117541408900419,
      "grad_norm": 0.2694884119514331,
      "learning_rate": 1e-06,
      "loss": 0.0087,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5048.066615513393,
      "epoch": 0.11315106765116743,
      "grad_norm": 0.4818094588291497,
      "learning_rate": 1e-06,
      "loss": 0.0003,
      "num_tokens": 26188824.0,
      "reward": 0.4587076764021601,
      "reward_std": 0.3670096014227186,
      "rewards/reward_func": 0.45870765511478695,
      "step": 81
    },
    {
      "clip_ratio": 0.014440144545265607,
      "epoch": 0.11454799441229295,
      "grad_norm": 39.00637105645605,
      "learning_rate": 1e-06,
      "loss": 0.0041,
      "step": 82
    },
    {
      "clip_ratio": 0.014427063853612967,
      "epoch": 0.11594492117341848,
      "grad_norm": 0.9150065270465777,
      "learning_rate": 1e-06,
      "loss": 0.0009,
      "step": 83
    },
    {
      "clip_ratio": 0.015046841731028897,
      "epoch": 0.117341847934544,
      "grad_norm": 0.24009263771304865,
      "learning_rate": 1e-06,
      "loss": 0.0007,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4776.74507359096,
      "epoch": 0.11873877469566953,
      "grad_norm": 0.4772391378994009,
      "learning_rate": 1e-06,
      "loss": 0.0175,
      "num_tokens": 27213245.0,
      "reward": 0.46243097526686533,
      "reward_std": 0.3514525145292282,
      "rewards/reward_func": 0.4624309518507549,
      "step": 85
    },
    {
      "clip_ratio": 0.013508185378408857,
      "epoch": 0.12013570145679506,
      "grad_norm": 166639.4843736821,
      "learning_rate": 1e-06,
      "loss": 7.3176,
      "step": 86
    },
    {
      "clip_ratio": 0.013775874627754092,
      "epoch": 0.12153262821792057,
      "grad_norm": 27.738301939542676,
      "learning_rate": 1e-06,
      "loss": 0.019,
      "step": 87
    },
    {
      "clip_ratio": 0.014050438667514495,
      "epoch": 0.1229295549790461,
      "grad_norm": 2.3910898137109857,
      "learning_rate": 1e-06,
      "loss": 0.0182,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4914.035984584263,
      "epoch": 0.12432648174017162,
      "grad_norm": 0.5313282819608155,
      "learning_rate": 1e-06,
      "loss": 0.0136,
      "num_tokens": 28265422.0,
      "reward": 0.493380460355963,
      "reward_std": 0.39978101530245375,
      "rewards/reward_func": 0.4933804316180093,
      "step": 89
    },
    {
      "clip_ratio": 0.017403810085462674,
      "epoch": 0.12572340850129715,
      "grad_norm": 2.5053609302596263,
      "learning_rate": 1e-06,
      "loss": 0.0153,
      "step": 90
    },
    {
      "clip_ratio": 0.018204916534679278,
      "epoch": 0.12712033526242267,
      "grad_norm": 1.3673142623484746,
      "learning_rate": 1e-06,
      "loss": 0.0147,
      "step": 91
    },
    {
      "clip_ratio": 0.019155576026865413,
      "epoch": 0.12851726202354818,
      "grad_norm": 0.285245341653243,
      "learning_rate": 1e-06,
      "loss": 0.0145,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4607.285940987723,
      "epoch": 0.12991418878467373,
      "grad_norm": 0.5382478014891808,
      "learning_rate": 1e-06,
      "loss": 0.0251,
      "num_tokens": 29256889.0,
      "reward": 0.4975069910287857,
      "reward_std": 0.39376414673668997,
      "rewards/reward_func": 0.4975069612264633,
      "step": 93
    },
    {
      "clip_ratio": 0.017131159414670298,
      "epoch": 0.13131111554579925,
      "grad_norm": 277.06893594236533,
      "learning_rate": 1e-06,
      "loss": 0.041,
      "step": 94
    },
    {
      "clip_ratio": 0.017470890456544503,
      "epoch": 0.13270804230692476,
      "grad_norm": 0.595127984757126,
      "learning_rate": 1e-06,
      "loss": 0.0264,
      "step": 95
    },
    {
      "clip_ratio": 0.017918821956430162,
      "epoch": 0.13410496906805028,
      "grad_norm": 0.3727932311400719,
      "learning_rate": 1e-06,
      "loss": 0.026,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 5105.030866350447,
      "epoch": 0.13550189582917582,
      "grad_norm": 0.5204808748332284,
      "learning_rate": 1e-06,
      "loss": 0.0166,
      "num_tokens": 30346587.0,
      "reward": 0.40901294563497814,
      "reward_std": 0.3415069899388722,
      "rewards/reward_func": 0.4090129222188677,
      "step": 97
    },
    {
      "clip_ratio": 0.0174449899766062,
      "epoch": 0.13689882259030134,
      "grad_norm": 1.1528181627310667,
      "learning_rate": 1e-06,
      "loss": 0.0183,
      "step": 98
    },
    {
      "clip_ratio": 0.01785454393497535,
      "epoch": 0.13829574935142686,
      "grad_norm": 0.6193098655808096,
      "learning_rate": 1e-06,
      "loss": 0.0177,
      "step": 99
    },
    {
      "clip_ratio": 0.018185677073363746,
      "epoch": 0.13969267611255237,
      "grad_norm": 0.29035255969562146,
      "learning_rate": 1e-06,
      "loss": 0.0174,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3904.9848981584823,
      "epoch": 0.14108960287367792,
      "grad_norm": 0.709202941284989,
      "learning_rate": 1e-06,
      "loss": 0.0147,
      "num_tokens": 31202372.0,
      "reward": 0.529426036136491,
      "reward_std": 0.35220117547682356,
      "rewards/reward_func": 0.5294260127203805,
      "step": 101
    },
    {
      "clip_ratio": 0.01801375742070377,
      "epoch": 0.14248652963480343,
      "grad_norm": 1.0665575599442703,
      "learning_rate": 1e-06,
      "loss": 0.0166,
      "step": 102
    },
    {
      "clip_ratio": 0.017245101110477532,
      "epoch": 0.14388345639592895,
      "grad_norm": 0.5821159923874696,
      "learning_rate": 1e-06,
      "loss": 0.0158,
      "step": 103
    },
    {
      "clip_ratio": 0.01721734370637153,
      "epoch": 0.1452803831570545,
      "grad_norm": 0.4938223790926132,
      "learning_rate": 1e-06,
      "loss": 0.0153,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4391.734933035715,
      "epoch": 0.14667730991818,
      "grad_norm": 0.6621058895011234,
      "learning_rate": 1e-06,
      "loss": 0.0067,
      "num_tokens": 32151129.0,
      "reward": 0.5155340305396489,
      "reward_std": 0.36261362050260815,
      "rewards/reward_func": 0.5155340198959623,
      "step": 105
    },
    {
      "clip_ratio": 0.021650432209883417,
      "epoch": 0.14807423667930553,
      "grad_norm": 7.221717083371915,
      "learning_rate": 1e-06,
      "loss": 0.0095,
      "step": 106
    },
    {
      "clip_ratio": 0.022131933498063257,
      "epoch": 0.14947116344043104,
      "grad_norm": 3.9217845571015726,
      "learning_rate": 1e-06,
      "loss": 0.0087,
      "step": 107
    },
    {
      "clip_ratio": 0.023090457543730736,
      "epoch": 0.1508680902015566,
      "grad_norm": 1.0915253624674428,
      "learning_rate": 1e-06,
      "loss": 0.0083,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4265.617579868862,
      "epoch": 0.1522650169626821,
      "grad_norm": 0.7674370802581663,
      "learning_rate": 1e-06,
      "loss": 0.0076,
      "num_tokens": 33074935.0,
      "reward": 0.49383693294865744,
      "reward_std": 0.3673490881919861,
      "rewards/reward_func": 0.49383691059691565,
      "step": 109
    },
    {
      "clip_ratio": 0.01965414373470204,
      "epoch": 0.15366194372380762,
      "grad_norm": 3.114603095736279,
      "learning_rate": 1e-06,
      "loss": 0.0099,
      "step": 110
    },
    {
      "clip_ratio": 0.020012260481183018,
      "epoch": 0.15505887048493314,
      "grad_norm": 4.049634875724923,
      "learning_rate": 1e-06,
      "loss": 0.0094,
      "step": 111
    },
    {
      "clip_ratio": 0.02028396732306906,
      "epoch": 0.15645579724605868,
      "grad_norm": 0.5429450253769021,
      "learning_rate": 1e-06,
      "loss": 0.0089,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3556.7655116489955,
      "epoch": 0.1578527240071842,
      "grad_norm": 0.8213126340122913,
      "learning_rate": 1e-06,
      "loss": 0.0142,
      "num_tokens": 33860148.0,
      "reward": 0.520222473357405,
      "reward_std": 0.3601721642272813,
      "rewards/reward_func": 0.5202224499412945,
      "step": 113
    },
    {
      "clip_ratio": 0.021029458713850806,
      "epoch": 0.15924965076830971,
      "grad_norm": 511.99698568152866,
      "learning_rate": 1e-06,
      "loss": 0.0391,
      "step": 114
    },
    {
      "clip_ratio": 0.020824986709547893,
      "epoch": 0.16064657752943523,
      "grad_norm": 1.6554233851238163,
      "learning_rate": 1e-06,
      "loss": 0.0163,
      "step": 115
    },
    {
      "clip_ratio": 0.021296620901141847,
      "epoch": 0.16204350429056077,
      "grad_norm": 5.389449179546646,
      "learning_rate": 1e-06,
      "loss": 0.016,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4172.214425223215,
      "epoch": 0.1634404310516863,
      "grad_norm": 0.7951777611207881,
      "learning_rate": 1e-06,
      "loss": 0.0188,
      "num_tokens": 34766754.0,
      "reward": 0.43932128697633743,
      "reward_std": 0.36100248460258755,
      "rewards/reward_func": 0.4393212656889643,
      "step": 117
    },
    {
      "clip_ratio": 0.024627858506781713,
      "epoch": 0.1648373578128118,
      "grad_norm": 5.497957714291199,
      "learning_rate": 1e-06,
      "loss": 0.0267,
      "step": 118
    },
    {
      "clip_ratio": 0.024561051279306412,
      "epoch": 0.16623428457393732,
      "grad_norm": 4.006479624309579,
      "learning_rate": 1e-06,
      "loss": 0.0259,
      "step": 119
    },
    {
      "clip_ratio": 0.02518816019541451,
      "epoch": 0.16763121133506287,
      "grad_norm": 5.696382550533168,
      "learning_rate": 1e-06,
      "loss": 0.0217,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3450.438938685826,
      "epoch": 0.16902813809618839,
      "grad_norm": 0.8756816112591876,
      "learning_rate": 1e-06,
      "loss": 0.0128,
      "num_tokens": 35530451.0,
      "reward": 0.4647199107067926,
      "reward_std": 0.35261689871549606,
      "rewards/reward_func": 0.46471990112747463,
      "step": 121
    },
    {
      "clip_ratio": 0.023490584788045714,
      "epoch": 0.1704250648573139,
      "grad_norm": 8.184182062225446,
      "learning_rate": 1e-06,
      "loss": 0.0176,
      "step": 122
    },
    {
      "clip_ratio": 0.022877583213682686,
      "epoch": 0.17182199161843945,
      "grad_norm": 2.337796998952532,
      "learning_rate": 1e-06,
      "loss": 0.0158,
      "step": 123
    },
    {
      "clip_ratio": 0.022725746002314345,
      "epoch": 0.17321891837956496,
      "grad_norm": 1.204853002031202,
      "learning_rate": 1e-06,
      "loss": 0.0149,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3755.143040248326,
      "epoch": 0.17461584514069048,
      "grad_norm": 1.1667266762961181,
      "learning_rate": 1e-06,
      "loss": 0.0009,
      "num_tokens": 36355620.0,
      "reward": 0.48860155258859905,
      "reward_std": 0.3995990295495306,
      "rewards/reward_func": 0.4886015323655946,
      "step": 125
    },
    {
      "clip_ratio": 0.024759572331926653,
      "epoch": 0.176012771901816,
      "grad_norm": 11.549731072093149,
      "learning_rate": 1e-06,
      "loss": 0.0075,
      "step": 126
    },
    {
      "clip_ratio": 0.023941728619060347,
      "epoch": 0.17740969866294154,
      "grad_norm": 1.5478778519158778,
      "learning_rate": 1e-06,
      "loss": 0.0043,
      "step": 127
    },
    {
      "clip_ratio": 0.023272603883274963,
      "epoch": 0.17880662542406706,
      "grad_norm": 1.0384027782585576,
      "learning_rate": 1e-06,
      "loss": 0.0034,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3561.3930053710938,
      "epoch": 0.18020355218519257,
      "grad_norm": 1.203850582780552,
      "learning_rate": 1e-06,
      "loss": 0.0059,
      "num_tokens": 37143444.0,
      "reward": 0.4855222978762218,
      "reward_std": 0.38453598746231626,
      "rewards/reward_func": 0.4855222872325352,
      "step": 129
    },
    {
      "clip_ratio": 0.024004271253943443,
      "epoch": 0.1816004789463181,
      "grad_norm": 11.686104385649747,
      "learning_rate": 1e-06,
      "loss": 0.012,
      "step": 130
    },
    {
      "clip_ratio": 0.02320788980328611,
      "epoch": 0.18299740570744363,
      "grad_norm": 51.97949282308589,
      "learning_rate": 1e-06,
      "loss": 0.0129,
      "step": 131
    },
    {
      "clip_ratio": 0.023830042220652103,
      "epoch": 0.18439433246856915,
      "grad_norm": 77704.4679582965,
      "learning_rate": 1e-06,
      "loss": 0.929,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3375.382795061384,
      "epoch": 0.18579125922969467,
      "grad_norm": 1.1905136253909123,
      "learning_rate": 1e-06,
      "loss": 0.0195,
      "num_tokens": 37893570.0,
      "reward": 0.4746183859450476,
      "reward_std": 0.41642780708415167,
      "rewards/reward_func": 0.47461836465767454,
      "step": 133
    },
    {
      "clip_ratio": 0.03054676218224423,
      "epoch": 0.18718818599082018,
      "grad_norm": 47.607522363933455,
      "learning_rate": 1e-06,
      "loss": 0.0306,
      "step": 134
    },
    {
      "clip_ratio": 0.03216503720198359,
      "epoch": 0.18858511275194573,
      "grad_norm": 6.197384439543361,
      "learning_rate": 1e-06,
      "loss": 0.0265,
      "step": 135
    },
    {
      "clip_ratio": 0.03178960005087512,
      "epoch": 0.18998203951307124,
      "grad_norm": 2.236284664240462,
      "learning_rate": 1e-06,
      "loss": 0.0236,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3824.760410853795,
      "epoch": 0.19137896627419676,
      "grad_norm": 1.0181749371365818,
      "learning_rate": 1e-06,
      "loss": 0.0301,
      "num_tokens": 38731414.0,
      "reward": 0.4555432179144451,
      "reward_std": 0.371168036546026,
      "rewards/reward_func": 0.45554319130522863,
      "step": 137
    },
    {
      "clip_ratio": 0.02426370605826378,
      "epoch": 0.1927758930353223,
      "grad_norm": 15.973977289606324,
      "learning_rate": 1e-06,
      "loss": 2.1765,
      "step": 138
    },
    {
      "clip_ratio": 0.023818871338984797,
      "epoch": 0.19417281979644782,
      "grad_norm": 125.42621828076582,
      "learning_rate": 1e-06,
      "loss": 0.0544,
      "step": 139
    },
    {
      "clip_ratio": 0.0235800734455032,
      "epoch": 0.19556974655757334,
      "grad_norm": 5.708397430999111,
      "learning_rate": 1e-06,
      "loss": 0.0358,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3121.5970720563614,
      "epoch": 0.19696667331869885,
      "grad_norm": 0.9463162061122932,
      "learning_rate": 1e-06,
      "loss": 0.0074,
      "num_tokens": 39432472.0,
      "reward": 0.3878828393561499,
      "reward_std": 0.32353205340249197,
      "rewards/reward_func": 0.3878828233906201,
      "step": 141
    },
    {
      "clip_ratio": 0.031209798529744148,
      "epoch": 0.1983636000798244,
      "grad_norm": 19.178703569706425,
      "learning_rate": 1e-06,
      "loss": 0.0158,
      "step": 142
    },
    {
      "clip_ratio": 0.031645855334188254,
      "epoch": 0.19976052684094991,
      "grad_norm": 73.92547009994209,
      "learning_rate": 1e-06,
      "loss": 0.0183,
      "step": 143
    },
    {
      "clip_ratio": 0.032449710848076005,
      "epoch": 0.20115745360207543,
      "grad_norm": 5.770213423174226,
      "learning_rate": 1e-06,
      "loss": 0.0164,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3645.8165196010045,
      "epoch": 0.20255438036320095,
      "grad_norm": 1.2228735898177363,
      "learning_rate": 1e-06,
      "loss": 0.0139,
      "num_tokens": 40234738.0,
      "reward": 0.4959501911486898,
      "reward_std": 0.3776449923004423,
      "rewards/reward_func": 0.4959501826337406,
      "step": 145
    },
    {
      "clip_ratio": 0.029987079756600515,
      "epoch": 0.2039513071243265,
      "grad_norm": 45.612373429574255,
      "learning_rate": 1e-06,
      "loss": 0.0274,
      "step": 146
    },
    {
      "clip_ratio": 0.029181892717523233,
      "epoch": 0.205348233885452,
      "grad_norm": 61.829029657734615,
      "learning_rate": 1e-06,
      "loss": 0.0238,
      "step": 147
    },
    {
      "clip_ratio": 0.028760566908333982,
      "epoch": 0.20674516064657752,
      "grad_norm": 6.854234290541286,
      "learning_rate": 1e-06,
      "loss": 0.0212,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3616.0256783621653,
      "epoch": 0.20814208740770304,
      "grad_norm": 1.1080044572109065,
      "learning_rate": 1e-06,
      "loss": 0.0188,
      "num_tokens": 41031965.0,
      "reward": 0.37510563326733454,
      "reward_std": 0.35230499718870434,
      "rewards/reward_func": 0.37510561889835764,
      "step": 149
    },
    {
      "clip_ratio": 0.032221541074769836,
      "epoch": 0.20953901416882859,
      "grad_norm": 22328.844087735877,
      "learning_rate": 1e-06,
      "loss": 0.7198,
      "step": 150
    },
    {
      "clip_ratio": 0.03243154074464526,
      "epoch": 0.2109359409299541,
      "grad_norm": 11006.630246836325,
      "learning_rate": 1e-06,
      "loss": 1.3521,
      "step": 151
    },
    {
      "clip_ratio": 0.032671846981559484,
      "epoch": 0.21233286769107962,
      "grad_norm": 52.44059977916511,
      "learning_rate": 1e-06,
      "loss": 0.0309,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3217.7909197126114,
      "epoch": 0.21372979445220514,
      "grad_norm": 1.3203188901357052,
      "learning_rate": 1e-06,
      "loss": 0.0184,
      "num_tokens": 41750731.0,
      "reward": 0.4333155059388706,
      "reward_std": 0.3546490435089384,
      "rewards/reward_func": 0.43331548997334074,
      "step": 153
    },
    {
      "clip_ratio": 0.032571413687297275,
      "epoch": 0.21512672121333068,
      "grad_norm": 56932.32164959619,
      "learning_rate": 1e-06,
      "loss": 1.4737,
      "step": 154
    },
    {
      "clip_ratio": 0.03264636732637882,
      "epoch": 0.2165236479744562,
      "grad_norm": 569.4118154787187,
      "learning_rate": 1e-06,
      "loss": 0.0464,
      "step": 155
    },
    {
      "clip_ratio": 0.033495722870741575,
      "epoch": 0.2179205747355817,
      "grad_norm": 14.372919673445498,
      "learning_rate": 1e-06,
      "loss": 0.0297,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2904.224591936384,
      "epoch": 0.21931750149670726,
      "grad_norm": 1.4884091255614038,
      "learning_rate": 1e-06,
      "loss": 0.0027,
      "num_tokens": 42407374.0,
      "reward": 0.46237824857234955,
      "reward_std": 0.38831384958965437,
      "rewards/reward_func": 0.4623782304780824,
      "step": 157
    },
    {
      "clip_ratio": 0.03049316949078015,
      "epoch": 0.22071442825783277,
      "grad_norm": 753.2718719394906,
      "learning_rate": 1e-06,
      "loss": 0.0615,
      "step": 158
    },
    {
      "clip_ratio": 0.028818647909377302,
      "epoch": 0.2221113550189583,
      "grad_norm": 193.7516952886048,
      "learning_rate": 1e-06,
      "loss": 0.03,
      "step": 159
    },
    {
      "clip_ratio": 0.028797397390007973,
      "epoch": 0.2235082817800838,
      "grad_norm": 86.17864803711156,
      "learning_rate": 1e-06,
      "loss": 0.0187,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2706.137860979353,
      "epoch": 0.22490520854120935,
      "grad_norm": 1.437051783056985,
      "learning_rate": 1e-06,
      "loss": 0.004,
      "num_tokens": 43026529.0,
      "reward": 0.40501056824411663,
      "reward_std": 0.3891604415008,
      "rewards/reward_func": 0.4050105522785868,
      "step": 161
    },
    {
      "clip_ratio": 0.03492548343326364,
      "epoch": 0.22630213530233487,
      "grad_norm": 1637.6323089099733,
      "learning_rate": 1e-06,
      "loss": 0.0796,
      "step": 162
    },
    {
      "clip_ratio": 0.034652630399380414,
      "epoch": 0.22769906206346038,
      "grad_norm": 294.8538016270806,
      "learning_rate": 1e-06,
      "loss": 0.0397,
      "step": 163
    },
    {
      "clip_ratio": 0.03514658712915012,
      "epoch": 0.2290959888245859,
      "grad_norm": 1729.0881090208584,
      "learning_rate": 1e-06,
      "loss": 0.1081,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3302.3215855189733,
      "epoch": 0.23049291558571144,
      "grad_norm": 1.3478961700451182,
      "learning_rate": 1e-06,
      "loss": 0.0095,
      "num_tokens": 43761689.0,
      "reward": 0.354110734803336,
      "reward_std": 0.3442910239100456,
      "rewards/reward_func": 0.35411071511251585,
      "step": 165
    },
    {
      "clip_ratio": 0.03175032351698194,
      "epoch": 0.23188984234683696,
      "grad_norm": 34.2352104129209,
      "learning_rate": 1e-06,
      "loss": 0.0236,
      "step": 166
    },
    {
      "clip_ratio": 0.031168780555682524,
      "epoch": 0.23328676910796248,
      "grad_norm": 3479.355422028937,
      "learning_rate": 1e-06,
      "loss": 0.3311,
      "step": 167
    },
    {
      "clip_ratio": 0.031411009441529004,
      "epoch": 0.234683695869088,
      "grad_norm": 211.96486688673437,
      "learning_rate": 1e-06,
      "loss": 0.0428,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3288.581760951451,
      "epoch": 0.23608062263021354,
      "grad_norm": 1.5809513920036224,
      "learning_rate": 1e-06,
      "loss": 0.0206,
      "num_tokens": 44495433.0,
      "reward": 0.3336522286491735,
      "reward_std": 0.35370165216071264,
      "rewards/reward_func": 0.3336522126836436,
      "step": 169
    },
    {
      "clip_ratio": 0.03352949349209666,
      "epoch": 0.23747754939133905,
      "grad_norm": 131.25750741813656,
      "learning_rate": 1e-06,
      "loss": 0.0411,
      "step": 170
    },
    {
      "clip_ratio": 0.033907221497169564,
      "epoch": 0.23887447615246457,
      "grad_norm": 38.44427070292949,
      "learning_rate": 1e-06,
      "loss": 0.0333,
      "step": 171
    },
    {
      "clip_ratio": 0.034381040039339235,
      "epoch": 0.24027140291359012,
      "grad_norm": 17107451499810.188,
      "learning_rate": 1e-06,
      "loss": 549430272.0,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3361.0919799804688,
      "epoch": 0.24166832967471563,
      "grad_norm": 1.637145998911673,
      "learning_rate": 1e-06,
      "loss": 0.0187,
      "num_tokens": 45243898.0,
      "reward": 0.394156134554318,
      "reward_std": 0.40689716062375475,
      "rewards/reward_func": 0.3941561228462628,
      "step": 173
    },
    {
      "clip_ratio": 0.032513607027275224,
      "epoch": 0.24306525643584115,
      "grad_norm": 800.8287273060004,
      "learning_rate": 1e-06,
      "loss": 1.6832,
      "step": 174
    },
    {
      "clip_ratio": 0.030495245408798968,
      "epoch": 0.24446218319696666,
      "grad_norm": 729.365666028725,
      "learning_rate": 1e-06,
      "loss": 0.1281,
      "step": 175
    },
    {
      "clip_ratio": 0.03067155448453767,
      "epoch": 0.2458591099580922,
      "grad_norm": 257.8929935202217,
      "learning_rate": 1e-06,
      "loss": 0.0783,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2669.938886369978,
      "epoch": 0.24725603671921773,
      "grad_norm": 1.6984457904054797,
      "learning_rate": 1e-06,
      "loss": -0.0017,
      "num_tokens": 45856081.0,
      "reward": 0.3272132841604097,
      "reward_std": 0.36115063620465143,
      "rewards/reward_func": 0.3272132767098291,
      "step": 177
    },
    {
      "clip_ratio": 0.03918117896786758,
      "epoch": 0.24865296348034324,
      "grad_norm": 459.0007426096092,
      "learning_rate": 1e-06,
      "loss": 0.0378,
      "step": 178
    },
    {
      "clip_ratio": 0.037612543175263066,
      "epoch": 0.2500498902414688,
      "grad_norm": 198.16000110545232,
      "learning_rate": 1e-06,
      "loss": 0.0357,
      "step": 179
    },
    {
      "clip_ratio": 0.03760030067392758,
      "epoch": 0.2514468170025943,
      "grad_norm": 193.31294839005466,
      "learning_rate": 1e-06,
      "loss": 0.0318,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3767.30623953683,
      "epoch": 0.2528437437637198,
      "grad_norm": 1.4505504547674242,
      "learning_rate": 1e-06,
      "loss": 0.0004,
      "num_tokens": 46683920.0,
      "reward": 0.3422575050166675,
      "reward_std": 0.3818031592028482,
      "rewards/reward_func": 0.3422574890511377,
      "step": 181
    },
    {
      "clip_ratio": 0.03315202798694372,
      "epoch": 0.25424067052484534,
      "grad_norm": 238.64962645549264,
      "learning_rate": 1e-06,
      "loss": 0.0462,
      "step": 182
    },
    {
      "clip_ratio": 0.03287409871284451,
      "epoch": 0.25563759728597085,
      "grad_norm": 6799.648792237924,
      "learning_rate": 1e-06,
      "loss": 0.0493,
      "step": 183
    },
    {
      "clip_ratio": 0.03349324116217239,
      "epoch": 0.25703452404709637,
      "grad_norm": 1395.3440659332787,
      "learning_rate": 1e-06,
      "loss": 0.1532,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3256.025630405971,
      "epoch": 0.2584314508082219,
      "grad_norm": 1.8519382626688905,
      "learning_rate": 1e-06,
      "loss": 0.0091,
      "num_tokens": 47409833.0,
      "reward": 0.32464135118893217,
      "reward_std": 0.3450521251985005,
      "rewards/reward_func": 0.324641336287771,
      "step": 185
    },
    {
      "clip_ratio": 0.034153553524187634,
      "epoch": 0.25982837756934746,
      "grad_norm": 55.33553677644012,
      "learning_rate": 1e-06,
      "loss": 0.0322,
      "step": 186
    },
    {
      "clip_ratio": 0.03325902218265193,
      "epoch": 0.261225304330473,
      "grad_norm": 807.7840491437855,
      "learning_rate": 1e-06,
      "loss": 0.0771,
      "step": 187
    },
    {
      "clip_ratio": 0.032821542982544215,
      "epoch": 0.2626222310915985,
      "grad_norm": 1395.5436294619333,
      "learning_rate": 1e-06,
      "loss": 0.0888,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3214.1480887276784,
      "epoch": 0.264019157852724,
      "grad_norm": 1.7683494272507614,
      "learning_rate": 1e-06,
      "loss": 0.0112,
      "num_tokens": 48129036.0,
      "reward": 0.31755988619157244,
      "reward_std": 0.3383456287639482,
      "rewards/reward_func": 0.31755986117890905,
      "step": 189
    },
    {
      "clip_ratio": 0.03975264209189585,
      "epoch": 0.2654160846138495,
      "grad_norm": 45184.22194084358,
      "learning_rate": 1e-06,
      "loss": 0.4443,
      "step": 190
    },
    {
      "clip_ratio": 0.03915290268404143,
      "epoch": 0.26681301137497504,
      "grad_norm": 1242.8816256939144,
      "learning_rate": 1e-06,
      "loss": 0.1447,
      "step": 191
    },
    {
      "clip_ratio": 0.03953878129167216,
      "epoch": 0.26820993813610056,
      "grad_norm": 1479.8321627428613,
      "learning_rate": 1e-06,
      "loss": 0.1168,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3272.2399553571427,
      "epoch": 0.26960686489722613,
      "grad_norm": 2.177506823211759,
      "learning_rate": 1e-06,
      "loss": 0.0092,
      "num_tokens": 48860342.0,
      "reward": 0.34522383979388643,
      "reward_std": 0.3796129695006779,
      "rewards/reward_func": 0.34522382276398794,
      "step": 193
    },
    {
      "clip_ratio": 0.03925856229450021,
      "epoch": 0.27100379165835164,
      "grad_norm": 513.8290574797221,
      "learning_rate": 1e-06,
      "loss": 0.0613,
      "step": 194
    },
    {
      "clip_ratio": 0.03799283770578248,
      "epoch": 0.27240071841947716,
      "grad_norm": 8983.081819712857,
      "learning_rate": 1e-06,
      "loss": 0.3161,
      "step": 195
    },
    {
      "clip_ratio": 0.036926204205623696,
      "epoch": 0.2737976451806027,
      "grad_norm": 1027.0780848048523,
      "learning_rate": 1e-06,
      "loss": 0.1274,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4028.9797712053573,
      "epoch": 0.2751945719417282,
      "grad_norm": 1.3698484721094724,
      "learning_rate": 1e-06,
      "loss": 0.018,
      "num_tokens": 49737682.0,
      "reward": 0.2579219543508121,
      "reward_std": 0.32817436435392927,
      "rewards/reward_func": 0.25792194423930986,
      "step": 197
    },
    {
      "clip_ratio": 0.03714690676757267,
      "epoch": 0.2765914987028537,
      "grad_norm": 759.0060595947047,
      "learning_rate": 1e-06,
      "loss": 0.0657,
      "step": 198
    },
    {
      "clip_ratio": 0.0380573890038899,
      "epoch": 0.2779884254639792,
      "grad_norm": 7489.486866588998,
      "learning_rate": 1e-06,
      "loss": 0.9186,
      "step": 199
    },
    {
      "clip_ratio": 0.03898970350357039,
      "epoch": 0.27938535222510474,
      "grad_norm": 403.4437322167033,
      "learning_rate": 1e-06,
      "loss": 0.1357,
      "step": 200
    }
  ],
  "logging_steps": 1,
  "max_steps": 715,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}