{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8810068649885584,
  "eval_steps": 500,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 228.8928680419922,
      "epoch": 0.0005720823798627002,
      "grad_norm": 1564325.25,
      "kl": 102312.453125,
      "learning_rate": 2.8571428571428575e-08,
      "loss": 4092.4983,
      "reward": 1.1400492191314697,
      "reward_std": 0.8813798427581787,
      "rewards/reward_function": 1.1400492191314697,
      "step": 1
    },
    {
      "completion_length": 89.25000762939453,
      "epoch": 0.0011441647597254005,
      "grad_norm": 154684.796875,
      "kl": 6795.4912109375,
      "learning_rate": 5.714285714285715e-08,
      "loss": 271.8196,
      "reward": 0.8071163892745972,
      "reward_std": 0.8583706021308899,
      "rewards/reward_function": 0.8071163892745972,
      "step": 2
    },
    {
      "completion_length": 96.96428680419922,
      "epoch": 0.0017162471395881006,
      "grad_norm": 547000.625,
      "kl": 85112.78125,
      "learning_rate": 8.571428571428573e-08,
      "loss": 3404.5115,
      "reward": 0.8311154842376709,
      "reward_std": 1.0693038702011108,
      "rewards/reward_function": 0.8311154842376709,
      "step": 3
    },
    {
      "completion_length": 98.39286041259766,
      "epoch": 0.002288329519450801,
      "grad_norm": 1676.3565673828125,
      "kl": 37.668617248535156,
      "learning_rate": 1.142857142857143e-07,
      "loss": 1.5067,
      "reward": 0.7354946732521057,
      "reward_std": 0.8702083826065063,
      "rewards/reward_function": 0.7354946732521057,
      "step": 4
    },
    {
      "completion_length": 77.42857360839844,
      "epoch": 0.002860411899313501,
      "grad_norm": 11284369.0,
      "kl": 777478.8125,
      "learning_rate": 1.4285714285714287e-07,
      "loss": 31099.1523,
      "reward": 0.7547075748443604,
      "reward_std": 0.9791582822799683,
      "rewards/reward_function": 0.7547075748443604,
      "step": 5
    },
    {
      "completion_length": 73.5714340209961,
      "epoch": 0.003432494279176201,
      "grad_norm": 6150391.5,
      "kl": 22783.47265625,
      "learning_rate": 1.7142857142857146e-07,
      "loss": 911.3389,
      "reward": 0.8155083060264587,
      "reward_std": 0.9278892278671265,
      "rewards/reward_function": 0.8155083060264587,
      "step": 6
    },
    {
      "completion_length": 149.21429443359375,
      "epoch": 0.004004576659038902,
      "grad_norm": 35080.0625,
      "kl": 1262.63623046875,
      "learning_rate": 2.0000000000000002e-07,
      "loss": 50.5054,
      "reward": 0.6627997159957886,
      "reward_std": 0.5749834775924683,
      "rewards/reward_function": 0.6627997159957886,
      "step": 7
    },
    {
      "completion_length": 71.92857360839844,
      "epoch": 0.004576659038901602,
      "grad_norm": 2994343.25,
      "kl": 907139.5,
      "learning_rate": 2.285714285714286e-07,
      "loss": 36285.5781,
      "reward": 0.7719243764877319,
      "reward_std": 0.6365665197372437,
      "rewards/reward_function": 0.7719243764877319,
      "step": 8
    },
    {
      "completion_length": 66.10714721679688,
      "epoch": 0.005148741418764302,
      "grad_norm": 2527576.0,
      "kl": 365474.625,
      "learning_rate": 2.5714285714285716e-07,
      "loss": 14618.9844,
      "reward": 0.7183529734611511,
      "reward_std": 0.8470447063446045,
      "rewards/reward_function": 0.7183529734611511,
      "step": 9
    },
    {
      "completion_length": 102.71428680419922,
      "epoch": 0.005720823798627002,
      "grad_norm": 651075328.0,
      "kl": 10634912.0,
      "learning_rate": 2.8571428571428575e-07,
      "loss": 425396.6562,
      "reward": 0.9092226624488831,
      "reward_std": 0.7284191250801086,
      "rewards/reward_function": 0.9092226624488831,
      "step": 10
    },
    {
      "completion_length": 71.25,
      "epoch": 0.006292906178489702,
      "grad_norm": 3888834.0,
      "kl": 397511.28125,
      "learning_rate": 3.1428571428571433e-07,
      "loss": 15900.4541,
      "reward": 0.6656758785247803,
      "reward_std": 0.7943789958953857,
      "rewards/reward_function": 0.6656758785247803,
      "step": 11
    },
    {
      "completion_length": 76.71428680419922,
      "epoch": 0.006864988558352402,
      "grad_norm": 3.834186315536499,
      "kl": 0.8590617179870605,
      "learning_rate": 3.428571428571429e-07,
      "loss": 0.0344,
      "reward": 0.9295981526374817,
      "reward_std": 0.9486422538757324,
      "rewards/reward_function": 0.9295981526374817,
      "step": 12
    },
    {
      "completion_length": 106.28572082519531,
      "epoch": 0.007437070938215103,
      "grad_norm": 434489.84375,
      "kl": 11238.5986328125,
      "learning_rate": 3.7142857142857145e-07,
      "loss": 449.544,
      "reward": 1.2047741413116455,
      "reward_std": 1.1723436117172241,
      "rewards/reward_function": 1.2047741413116455,
      "step": 13
    },
    {
      "completion_length": 88.75000762939453,
      "epoch": 0.008009153318077803,
      "grad_norm": 1111384.875,
      "kl": 151343.296875,
      "learning_rate": 4.0000000000000003e-07,
      "loss": 6053.7314,
      "reward": 0.7786172032356262,
      "reward_std": 1.085495114326477,
      "rewards/reward_function": 0.7786172032356262,
      "step": 14
    },
    {
      "completion_length": 72.03572082519531,
      "epoch": 0.008581235697940504,
      "grad_norm": 24670.48828125,
      "kl": 2378.1279296875,
      "learning_rate": 4.285714285714286e-07,
      "loss": 95.1251,
      "reward": 1.03531014919281,
      "reward_std": 0.985435962677002,
      "rewards/reward_function": 1.03531014919281,
      "step": 15
    },
    {
      "completion_length": 106.53572082519531,
      "epoch": 0.009153318077803204,
      "grad_norm": 15368923.0,
      "kl": 654015.25,
      "learning_rate": 4.571428571428572e-07,
      "loss": 26160.6133,
      "reward": 0.8058643937110901,
      "reward_std": 0.9300833344459534,
      "rewards/reward_function": 0.8058643937110901,
      "step": 16
    },
    {
      "completion_length": 116.17857360839844,
      "epoch": 0.009725400457665904,
      "grad_norm": 3736480.75,
      "kl": 45342.09765625,
      "learning_rate": 4.857142857142857e-07,
      "loss": 1813.6838,
      "reward": 0.6415729522705078,
      "reward_std": 0.6860412955284119,
      "rewards/reward_function": 0.6415729522705078,
      "step": 17
    },
    {
      "completion_length": 118.50000762939453,
      "epoch": 0.010297482837528604,
      "grad_norm": 14.7220458984375,
      "kl": 2.0087039470672607,
      "learning_rate": 5.142857142857143e-07,
      "loss": 0.0803,
      "reward": 0.8128255605697632,
      "reward_std": 0.8992515802383423,
      "rewards/reward_function": 0.8128255605697632,
      "step": 18
    },
    {
      "completion_length": 73.75,
      "epoch": 0.010869565217391304,
      "grad_norm": 13757.4423828125,
      "kl": 416.51214599609375,
      "learning_rate": 5.428571428571429e-07,
      "loss": 16.6605,
      "reward": 0.5336253046989441,
      "reward_std": 0.7795614004135132,
      "rewards/reward_function": 0.5336253046989441,
      "step": 19
    },
    {
      "completion_length": 65.3214340209961,
      "epoch": 0.011441647597254004,
      "grad_norm": 3077649.5,
      "kl": 519223.09375,
      "learning_rate": 5.714285714285715e-07,
      "loss": 20768.9258,
      "reward": 0.5448503494262695,
      "reward_std": 0.6515836715698242,
      "rewards/reward_function": 0.5448503494262695,
      "step": 20
    },
    {
      "completion_length": 86.17857360839844,
      "epoch": 0.012013729977116704,
      "grad_norm": 106360.8984375,
      "kl": 24441.162109375,
      "learning_rate": 6.000000000000001e-07,
      "loss": 977.6466,
      "reward": 0.41388723254203796,
      "reward_std": 0.5603397488594055,
      "rewards/reward_function": 0.41388723254203796,
      "step": 21
    },
    {
      "completion_length": 125.5714340209961,
      "epoch": 0.012585812356979404,
      "grad_norm": 32312.447265625,
      "kl": 1363.8094482421875,
      "learning_rate": 6.285714285714287e-07,
      "loss": 54.5524,
      "reward": 1.0343550443649292,
      "reward_std": 1.0715274810791016,
      "rewards/reward_function": 1.0343550443649292,
      "step": 22
    },
    {
      "completion_length": 126.28572082519531,
      "epoch": 0.013157894736842105,
      "grad_norm": 4250140.5,
      "kl": 31725.962890625,
      "learning_rate": 6.571428571428571e-07,
      "loss": 1269.0383,
      "reward": 0.6536315083503723,
      "reward_std": 0.7068561911582947,
      "rewards/reward_function": 0.6536315083503723,
      "step": 23
    },
    {
      "completion_length": 70.89286041259766,
      "epoch": 0.013729977116704805,
      "grad_norm": 1623422.125,
      "kl": 160032.3125,
      "learning_rate": 6.857142857142858e-07,
      "loss": 6401.292,
      "reward": 0.632136344909668,
      "reward_std": 0.749646008014679,
      "rewards/reward_function": 0.632136344909668,
      "step": 24
    },
    {
      "completion_length": 79.8214340209961,
      "epoch": 0.014302059496567507,
      "grad_norm": 40139060.0,
      "kl": 561254.0625,
      "learning_rate": 7.142857142857143e-07,
      "loss": 22450.1621,
      "reward": 0.5180789232254028,
      "reward_std": 0.5655960440635681,
      "rewards/reward_function": 0.5180789232254028,
      "step": 25
    },
    {
      "completion_length": 60.750003814697266,
      "epoch": 0.014874141876430207,
      "grad_norm": 3678240.25,
      "kl": 31498.375,
      "learning_rate": 7.428571428571429e-07,
      "loss": 1259.9349,
      "reward": 0.7351226210594177,
      "reward_std": 0.6401946544647217,
      "rewards/reward_function": 0.7351226210594177,
      "step": 26
    },
    {
      "completion_length": 61.53571701049805,
      "epoch": 0.015446224256292907,
      "grad_norm": 561489472.0,
      "kl": 2291197.25,
      "learning_rate": 7.714285714285715e-07,
      "loss": 91647.8984,
      "reward": 0.9715724587440491,
      "reward_std": 0.8716418147087097,
      "rewards/reward_function": 0.9715724587440491,
      "step": 27
    },
    {
      "completion_length": 68.60714721679688,
      "epoch": 0.016018306636155607,
      "grad_norm": 2399563.75,
      "kl": 257526.890625,
      "learning_rate": 8.000000000000001e-07,
      "loss": 10301.0752,
      "reward": 0.5327023863792419,
      "reward_std": 0.5703479647636414,
      "rewards/reward_function": 0.5327023863792419,
      "step": 28
    },
    {
      "completion_length": 82.85714721679688,
      "epoch": 0.016590389016018305,
      "grad_norm": 311356.25,
      "kl": 12673.7744140625,
      "learning_rate": 8.285714285714285e-07,
      "loss": 506.9509,
      "reward": 0.39342233538627625,
      "reward_std": 0.6884366869926453,
      "rewards/reward_function": 0.39342233538627625,
      "step": 29
    },
    {
      "completion_length": 99.42857360839844,
      "epoch": 0.017162471395881007,
      "grad_norm": 888257.75,
      "kl": 29379.40234375,
      "learning_rate": 8.571428571428572e-07,
      "loss": 1175.1763,
      "reward": 0.7146863341331482,
      "reward_std": 0.8734264969825745,
      "rewards/reward_function": 0.7146863341331482,
      "step": 30
    },
    {
      "completion_length": 55.96428680419922,
      "epoch": 0.017734553775743706,
      "grad_norm": 1192864.125,
      "kl": 93294.296875,
      "learning_rate": 8.857142857142857e-07,
      "loss": 3731.772,
      "reward": 1.0370844602584839,
      "reward_std": 0.9122040271759033,
      "rewards/reward_function": 1.0370844602584839,
      "step": 31
    },
    {
      "completion_length": 117.78572082519531,
      "epoch": 0.018306636155606407,
      "grad_norm": 1077003.0,
      "kl": 114313.1640625,
      "learning_rate": 9.142857142857144e-07,
      "loss": 4572.5269,
      "reward": 0.4757004380226135,
      "reward_std": 0.4821663200855255,
      "rewards/reward_function": 0.4757004380226135,
      "step": 32
    },
    {
      "completion_length": 52.42857360839844,
      "epoch": 0.01887871853546911,
      "grad_norm": 473145.1875,
      "kl": 85542.546875,
      "learning_rate": 9.42857142857143e-07,
      "loss": 3421.7021,
      "reward": 1.176900863647461,
      "reward_std": 1.1432535648345947,
      "rewards/reward_function": 1.176900863647461,
      "step": 33
    },
    {
      "completion_length": 97.0714340209961,
      "epoch": 0.019450800915331808,
      "grad_norm": 604604.75,
      "kl": 108438.515625,
      "learning_rate": 9.714285714285715e-07,
      "loss": 4337.54,
      "reward": 0.9204012751579285,
      "reward_std": 0.9002698659896851,
      "rewards/reward_function": 0.9204012751579285,
      "step": 34
    },
    {
      "completion_length": 89.42857360839844,
      "epoch": 0.02002288329519451,
      "grad_norm": 980339.0,
      "kl": 68370.3671875,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 2734.8149,
      "reward": 0.8670087456703186,
      "reward_std": 1.0319786071777344,
      "rewards/reward_function": 0.8670087456703186,
      "step": 35
    },
    {
      "completion_length": 64.35714721679688,
      "epoch": 0.020594965675057208,
      "grad_norm": 2.0410149097442627,
      "kl": 0.542355477809906,
      "learning_rate": 1.0285714285714286e-06,
      "loss": 0.0217,
      "reward": 1.1713099479675293,
      "reward_std": 1.0274691581726074,
      "rewards/reward_function": 1.1713099479675293,
      "step": 36
    },
    {
      "completion_length": 91.92857360839844,
      "epoch": 0.02116704805491991,
      "grad_norm": 7319958.0,
      "kl": 396432.84375,
      "learning_rate": 1.0571428571428573e-06,
      "loss": 15857.3145,
      "reward": 0.8260324001312256,
      "reward_std": 1.0479296445846558,
      "rewards/reward_function": 0.8260324001312256,
      "step": 37
    },
    {
      "completion_length": 118.42857360839844,
      "epoch": 0.021739130434782608,
      "grad_norm": 136772.625,
      "kl": 11540.68359375,
      "learning_rate": 1.0857142857142858e-06,
      "loss": 461.6273,
      "reward": 0.7798692584037781,
      "reward_std": 0.9083630442619324,
      "rewards/reward_function": 0.7798692584037781,
      "step": 38
    },
    {
      "completion_length": 93.00000762939453,
      "epoch": 0.02231121281464531,
      "grad_norm": 1537851.0,
      "kl": 285428.6875,
      "learning_rate": 1.1142857142857145e-06,
      "loss": 11417.1475,
      "reward": 1.280434489250183,
      "reward_std": 1.063579797744751,
      "rewards/reward_function": 1.280434489250183,
      "step": 39
    },
    {
      "completion_length": 76.03572082519531,
      "epoch": 0.02288329519450801,
      "grad_norm": 635.2694091796875,
      "kl": 9.865652084350586,
      "learning_rate": 1.142857142857143e-06,
      "loss": 0.3946,
      "reward": 1.3114697933197021,
      "reward_std": 1.1318063735961914,
      "rewards/reward_function": 1.3114697933197021,
      "step": 40
    },
    {
      "completion_length": 117.8214340209961,
      "epoch": 0.02345537757437071,
      "grad_norm": 14.028284072875977,
      "kl": 0.8193120360374451,
      "learning_rate": 1.1714285714285715e-06,
      "loss": 0.0328,
      "reward": 0.6799558401107788,
      "reward_std": 0.3601263463497162,
      "rewards/reward_function": 0.6799558401107788,
      "step": 41
    },
    {
      "completion_length": 104.92857360839844,
      "epoch": 0.02402745995423341,
      "grad_norm": 3042845.25,
      "kl": 12223.900390625,
      "learning_rate": 1.2000000000000002e-06,
      "loss": 488.9562,
      "reward": 0.6414692401885986,
      "reward_std": 0.6692579388618469,
      "rewards/reward_function": 0.6414692401885986,
      "step": 42
    },
    {
      "completion_length": 96.42857360839844,
      "epoch": 0.02459954233409611,
      "grad_norm": 427927.53125,
      "kl": 74239.5625,
      "learning_rate": 1.2285714285714286e-06,
      "loss": 2969.5828,
      "reward": 0.9474552273750305,
      "reward_std": 0.8861908912658691,
      "rewards/reward_function": 0.9474552273750305,
      "step": 43
    },
    {
      "completion_length": 96.89286041259766,
      "epoch": 0.02517162471395881,
      "grad_norm": 934487.6875,
      "kl": 124002.3359375,
      "learning_rate": 1.2571428571428573e-06,
      "loss": 4960.0923,
      "reward": 0.9418355822563171,
      "reward_std": 1.046279788017273,
      "rewards/reward_function": 0.9418355822563171,
      "step": 44
    },
    {
      "completion_length": 73.28572082519531,
      "epoch": 0.02574370709382151,
      "grad_norm": 7058849.0,
      "kl": 291908.75,
      "learning_rate": 1.2857142857142856e-06,
      "loss": 11676.3477,
      "reward": 0.5734997987747192,
      "reward_std": 0.9853571057319641,
      "rewards/reward_function": 0.5734997987747192,
      "step": 45
    },
    {
      "completion_length": 120.28572082519531,
      "epoch": 0.02631578947368421,
      "grad_norm": 108833.953125,
      "kl": 11022.5263671875,
      "learning_rate": 1.3142857142857143e-06,
      "loss": 440.9012,
      "reward": 0.3059252202510834,
      "reward_std": 0.49025431275367737,
      "rewards/reward_function": 0.3059252202510834,
      "step": 46
    },
    {
      "completion_length": 81.25,
      "epoch": 0.02688787185354691,
      "grad_norm": 4449.6162109375,
      "kl": 240.4727325439453,
      "learning_rate": 1.342857142857143e-06,
      "loss": 9.6189,
      "reward": 0.8813316226005554,
      "reward_std": 0.7435001730918884,
      "rewards/reward_function": 0.8813316226005554,
      "step": 47
    },
    {
      "completion_length": 127.00000762939453,
      "epoch": 0.02745995423340961,
      "grad_norm": 1.3961389064788818,
      "kl": 0.40921276807785034,
      "learning_rate": 1.3714285714285717e-06,
      "loss": 0.0164,
      "reward": 1.0761361122131348,
      "reward_std": 1.1622653007507324,
      "rewards/reward_function": 1.0761361122131348,
      "step": 48
    },
    {
      "completion_length": 79.75,
      "epoch": 0.02803203661327231,
      "grad_norm": 253153.953125,
      "kl": 19037.72265625,
      "learning_rate": 1.4000000000000001e-06,
      "loss": 761.509,
      "reward": 1.0196136236190796,
      "reward_std": 0.7807164788246155,
      "rewards/reward_function": 1.0196136236190796,
      "step": 49
    },
    {
      "completion_length": 67.10714721679688,
      "epoch": 0.028604118993135013,
      "grad_norm": 4083.54638671875,
      "kl": 51.461673736572266,
      "learning_rate": 1.4285714285714286e-06,
      "loss": 2.0585,
      "reward": 1.1423027515411377,
      "reward_std": 1.0599474906921387,
      "rewards/reward_function": 1.1423027515411377,
      "step": 50
    },
    {
      "completion_length": 91.75000762939453,
      "epoch": 0.02917620137299771,
      "grad_norm": 3132.056640625,
      "kl": 332.4888916015625,
      "learning_rate": 1.4571428571428573e-06,
      "loss": 13.2996,
      "reward": 1.0447752475738525,
      "reward_std": 1.1316096782684326,
      "rewards/reward_function": 1.0447752475738525,
      "step": 51
    },
    {
      "completion_length": 112.03572082519531,
      "epoch": 0.029748283752860413,
      "grad_norm": 1351855.625,
      "kl": 236515.40625,
      "learning_rate": 1.4857142857142858e-06,
      "loss": 9460.6152,
      "reward": 0.7726398706436157,
      "reward_std": 0.9718583226203918,
      "rewards/reward_function": 0.7726398706436157,
      "step": 52
    },
    {
      "completion_length": 74.0714340209961,
      "epoch": 0.03032036613272311,
      "grad_norm": 365871.90625,
      "kl": 86840.4609375,
      "learning_rate": 1.5142857142857145e-06,
      "loss": 3473.6182,
      "reward": 0.7167432308197021,
      "reward_std": 0.7652686238288879,
      "rewards/reward_function": 0.7167432308197021,
      "step": 53
    },
    {
      "completion_length": 73.71428680419922,
      "epoch": 0.030892448512585814,
      "grad_norm": 300117.0,
      "kl": 37508.125,
      "learning_rate": 1.542857142857143e-06,
      "loss": 1500.3251,
      "reward": 1.061155080795288,
      "reward_std": 0.5769102573394775,
      "rewards/reward_function": 1.061155080795288,
      "step": 54
    },
    {
      "completion_length": 94.78572082519531,
      "epoch": 0.031464530892448515,
      "grad_norm": 5525644.0,
      "kl": 191403.546875,
      "learning_rate": 1.5714285714285714e-06,
      "loss": 7656.1421,
      "reward": 0.9869399070739746,
      "reward_std": 1.0555799007415771,
      "rewards/reward_function": 0.9869399070739746,
      "step": 55
    },
    {
      "completion_length": 99.92857360839844,
      "epoch": 0.032036613272311214,
      "grad_norm": 39059.734375,
      "kl": 1068.976806640625,
      "learning_rate": 1.6000000000000001e-06,
      "loss": 42.7591,
      "reward": 0.7690447568893433,
      "reward_std": 0.6354593634605408,
      "rewards/reward_function": 0.7690447568893433,
      "step": 56
    },
    {
      "completion_length": 76.92857360839844,
      "epoch": 0.03260869565217391,
      "grad_norm": 1733.0419921875,
      "kl": 109.82354736328125,
      "learning_rate": 1.6285714285714288e-06,
      "loss": 4.3929,
      "reward": 0.7044306993484497,
      "reward_std": 0.8395044207572937,
      "rewards/reward_function": 0.7044306993484497,
      "step": 57
    },
    {
      "completion_length": 108.67857360839844,
      "epoch": 0.03318077803203661,
      "grad_norm": 718138.6875,
      "kl": 18385.6328125,
      "learning_rate": 1.657142857142857e-06,
      "loss": 735.4251,
      "reward": 0.5066463947296143,
      "reward_std": 0.7008506655693054,
      "rewards/reward_function": 0.5066463947296143,
      "step": 58
    },
    {
      "completion_length": 74.21428680419922,
      "epoch": 0.033752860411899316,
      "grad_norm": 3092098.75,
      "kl": 21698.197265625,
      "learning_rate": 1.6857142857142858e-06,
      "loss": 867.9279,
      "reward": 0.8928393125534058,
      "reward_std": 0.9691706895828247,
      "rewards/reward_function": 0.8928393125534058,
      "step": 59
    },
    {
      "completion_length": 69.21428680419922,
      "epoch": 0.034324942791762014,
      "grad_norm": 6776.443359375,
      "kl": 122.99604034423828,
      "learning_rate": 1.7142857142857145e-06,
      "loss": 4.9198,
      "reward": 1.1115070581436157,
      "reward_std": 1.0321422815322876,
      "rewards/reward_function": 1.1115070581436157,
      "step": 60
    },
    {
      "completion_length": 68.96428680419922,
      "epoch": 0.03489702517162471,
      "grad_norm": 3487237.75,
      "kl": 99870.28125,
      "learning_rate": 1.7428571428571432e-06,
      "loss": 3994.8113,
      "reward": 0.42442554235458374,
      "reward_std": 0.6797846555709839,
      "rewards/reward_function": 0.42442554235458374,
      "step": 61
    },
    {
      "completion_length": 66.5,
      "epoch": 0.03546910755148741,
      "grad_norm": 14173367.0,
      "kl": 392340.15625,
      "learning_rate": 1.7714285714285714e-06,
      "loss": 15693.6045,
      "reward": 0.8915873169898987,
      "reward_std": 0.9666271805763245,
      "rewards/reward_function": 0.8915873169898987,
      "step": 62
    },
    {
      "completion_length": 73.53572082519531,
      "epoch": 0.036041189931350116,
      "grad_norm": 32805.0546875,
      "kl": 798.0703735351562,
      "learning_rate": 1.8000000000000001e-06,
      "loss": 31.9228,
      "reward": 1.4147207736968994,
      "reward_std": 0.9207350611686707,
      "rewards/reward_function": 1.4147207736968994,
      "step": 63
    },
    {
      "completion_length": 76.10714721679688,
      "epoch": 0.036613272311212815,
      "grad_norm": 112268.9296875,
      "kl": 22303.8515625,
      "learning_rate": 1.8285714285714288e-06,
      "loss": 892.1541,
      "reward": 0.6782101392745972,
      "reward_std": 0.6700085401535034,
      "rewards/reward_function": 0.6782101392745972,
      "step": 64
    },
    {
      "completion_length": 72.14286041259766,
      "epoch": 0.03718535469107551,
      "grad_norm": 178114.5625,
      "kl": 35136.1875,
      "learning_rate": 1.8571428571428573e-06,
      "loss": 1405.4476,
      "reward": 0.9143344163894653,
      "reward_std": 1.0868008136749268,
      "rewards/reward_function": 0.9143344163894653,
      "step": 65
    },
    {
      "completion_length": 80.35714721679688,
      "epoch": 0.03775743707093822,
      "grad_norm": 37138316.0,
      "kl": 207868.578125,
      "learning_rate": 1.885714285714286e-06,
      "loss": 8314.7422,
      "reward": 1.0537182092666626,
      "reward_std": 1.1045030355453491,
      "rewards/reward_function": 1.0537182092666626,
      "step": 66
    },
    {
      "completion_length": 76.53572082519531,
      "epoch": 0.03832951945080092,
      "grad_norm": 5537.51513671875,
      "kl": 18.85404396057129,
      "learning_rate": 1.9142857142857145e-06,
      "loss": 0.7542,
      "reward": 0.5689067244529724,
      "reward_std": 0.3695144057273865,
      "rewards/reward_function": 0.5689067244529724,
      "step": 67
    },
    {
      "completion_length": 85.85714721679688,
      "epoch": 0.038901601830663615,
      "grad_norm": 15454.76171875,
      "kl": 270.26312255859375,
      "learning_rate": 1.942857142857143e-06,
      "loss": 10.8105,
      "reward": 0.49577897787094116,
      "reward_std": 0.38729485869407654,
      "rewards/reward_function": 0.49577897787094116,
      "step": 68
    },
    {
      "completion_length": 58.750003814697266,
      "epoch": 0.039473684210526314,
      "grad_norm": 140.70729064941406,
      "kl": 10.838787078857422,
      "learning_rate": 1.9714285714285714e-06,
      "loss": 0.4336,
      "reward": 1.039588451385498,
      "reward_std": 0.8005112409591675,
      "rewards/reward_function": 1.039588451385498,
      "step": 69
    },
    {
      "completion_length": 89.25000762939453,
      "epoch": 0.04004576659038902,
      "grad_norm": 204606.203125,
      "kl": 47173.26953125,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 1886.9307,
      "reward": 0.9541910886764526,
      "reward_std": 0.7928133606910706,
      "rewards/reward_function": 0.9541910886764526,
      "step": 70
    },
    {
      "completion_length": 55.357147216796875,
      "epoch": 0.04061784897025172,
      "grad_norm": 1358112.875,
      "kl": 35529.9609375,
      "learning_rate": 2.028571428571429e-06,
      "loss": 1421.1984,
      "reward": 1.3281930685043335,
      "reward_std": 1.029097318649292,
      "rewards/reward_function": 1.3281930685043335,
      "step": 71
    },
    {
      "completion_length": 70.14286041259766,
      "epoch": 0.041189931350114416,
      "grad_norm": 33247878.0,
      "kl": 603424.4375,
      "learning_rate": 2.0571428571428573e-06,
      "loss": 24136.9824,
      "reward": 0.8211567401885986,
      "reward_std": 0.822792112827301,
      "rewards/reward_function": 0.8211567401885986,
      "step": 72
    },
    {
      "completion_length": 55.32143020629883,
      "epoch": 0.041762013729977114,
      "grad_norm": 208937.671875,
      "kl": 9806.916015625,
      "learning_rate": 2.0857142857142858e-06,
      "loss": 392.2766,
      "reward": 0.7869198322296143,
      "reward_std": 0.7801079154014587,
      "rewards/reward_function": 0.7869198322296143,
      "step": 73
    },
    {
      "completion_length": 53.10714340209961,
      "epoch": 0.04233409610983982,
      "grad_norm": 2.8001291751861572,
      "kl": 0.7485305666923523,
      "learning_rate": 2.1142857142857147e-06,
      "loss": 0.0299,
      "reward": 0.9195212125778198,
      "reward_std": 0.8575438261032104,
      "rewards/reward_function": 0.9195212125778198,
      "step": 74
    },
    {
      "completion_length": 47.142860412597656,
      "epoch": 0.04290617848970252,
      "grad_norm": 40520.71875,
      "kl": 4130.67431640625,
      "learning_rate": 2.1428571428571427e-06,
      "loss": 165.227,
      "reward": 0.9767449498176575,
      "reward_std": 0.7389394044876099,
      "rewards/reward_function": 0.9767449498176575,
      "step": 75
    },
    {
      "completion_length": 60.53571701049805,
      "epoch": 0.043478260869565216,
      "grad_norm": 148915.625,
      "kl": 15134.6435546875,
      "learning_rate": 2.1714285714285716e-06,
      "loss": 605.3857,
      "reward": 0.8881747126579285,
      "reward_std": 1.0513893365859985,
      "rewards/reward_function": 0.8881747126579285,
      "step": 76
    },
    {
      "completion_length": 100.46428680419922,
      "epoch": 0.044050343249427915,
      "grad_norm": 6.646983623504639,
      "kl": 1.2588237524032593,
      "learning_rate": 2.2e-06,
      "loss": 0.0504,
      "reward": 0.9205050468444824,
      "reward_std": 1.0281927585601807,
      "rewards/reward_function": 0.9205050468444824,
      "step": 77
    },
    {
      "completion_length": 95.25000762939453,
      "epoch": 0.04462242562929062,
      "grad_norm": 2652.0048828125,
      "kl": 60.936073303222656,
      "learning_rate": 2.228571428571429e-06,
      "loss": 2.4374,
      "reward": 1.1903152465820312,
      "reward_std": 1.0934501886367798,
      "rewards/reward_function": 1.1903152465820312,
      "step": 78
    },
    {
      "completion_length": 75.96428680419922,
      "epoch": 0.04519450800915332,
      "grad_norm": 1470919.875,
      "kl": 48120.0078125,
      "learning_rate": 2.257142857142857e-06,
      "loss": 1924.8004,
      "reward": 1.1153239011764526,
      "reward_std": 0.8738850355148315,
      "rewards/reward_function": 1.1153239011764526,
      "step": 79
    },
    {
      "completion_length": 69.89286041259766,
      "epoch": 0.04576659038901602,
      "grad_norm": 30842.7890625,
      "kl": 2231.918701171875,
      "learning_rate": 2.285714285714286e-06,
      "loss": 89.2767,
      "reward": 1.0487995147705078,
      "reward_std": 1.078444004058838,
      "rewards/reward_function": 1.0487995147705078,
      "step": 80
    },
    {
      "completion_length": 96.14286041259766,
      "epoch": 0.04633867276887872,
      "grad_norm": 683805.3125,
      "kl": 2687.82275390625,
      "learning_rate": 2.3142857142857145e-06,
      "loss": 107.5129,
      "reward": 1.2307084798812866,
      "reward_std": 0.6299479603767395,
      "rewards/reward_function": 1.2307084798812866,
      "step": 81
    },
    {
      "completion_length": 86.71428680419922,
      "epoch": 0.04691075514874142,
      "grad_norm": 90158.0703125,
      "kl": 11469.1552734375,
      "learning_rate": 2.342857142857143e-06,
      "loss": 458.7663,
      "reward": 1.1188867092132568,
      "reward_std": 0.9243437051773071,
      "rewards/reward_function": 1.1188867092132568,
      "step": 82
    },
    {
      "completion_length": 64.92857360839844,
      "epoch": 0.04748283752860412,
      "grad_norm": 68.14144897460938,
      "kl": 4.270777702331543,
      "learning_rate": 2.371428571428572e-06,
      "loss": 0.1708,
      "reward": 1.1227463483810425,
      "reward_std": 1.041912317276001,
      "rewards/reward_function": 1.1227463483810425,
      "step": 83
    },
    {
      "completion_length": 69.17857360839844,
      "epoch": 0.04805491990846682,
      "grad_norm": 11569.748046875,
      "kl": 704.378662109375,
      "learning_rate": 2.4000000000000003e-06,
      "loss": 28.1751,
      "reward": 0.9703347682952881,
      "reward_std": 1.0495824813842773,
      "rewards/reward_function": 0.9703347682952881,
      "step": 84
    },
    {
      "completion_length": 65.28572082519531,
      "epoch": 0.04862700228832952,
      "grad_norm": 623.8413696289062,
      "kl": 103.28960418701172,
      "learning_rate": 2.428571428571429e-06,
      "loss": 4.1316,
      "reward": 0.7567501664161682,
      "reward_std": 0.6725633144378662,
      "rewards/reward_function": 0.7567501664161682,
      "step": 85
    },
    {
      "completion_length": 63.000003814697266,
      "epoch": 0.04919908466819222,
      "grad_norm": 107993528.0,
      "kl": 872167.625,
      "learning_rate": 2.4571428571428573e-06,
      "loss": 34886.7031,
      "reward": 0.6225532293319702,
      "reward_std": 0.7273246645927429,
      "rewards/reward_function": 0.6225532293319702,
      "step": 86
    },
    {
      "completion_length": 84.10714721679688,
      "epoch": 0.04977116704805492,
      "grad_norm": 9392.5419921875,
      "kl": 2435.8994140625,
      "learning_rate": 2.485714285714286e-06,
      "loss": 97.436,
      "reward": 0.8569318652153015,
      "reward_std": 0.9703346490859985,
      "rewards/reward_function": 0.8569318652153015,
      "step": 87
    },
    {
      "completion_length": 81.8214340209961,
      "epoch": 0.05034324942791762,
      "grad_norm": 5.535726547241211,
      "kl": 0.6003254055976868,
      "learning_rate": 2.5142857142857147e-06,
      "loss": 0.024,
      "reward": 0.7826415300369263,
      "reward_std": 0.6194065809249878,
      "rewards/reward_function": 0.7826415300369263,
      "step": 88
    },
    {
      "completion_length": 60.357147216796875,
      "epoch": 0.05091533180778032,
      "grad_norm": 278910.25,
      "kl": 24218.13671875,
      "learning_rate": 2.542857142857143e-06,
      "loss": 968.7254,
      "reward": 0.686827540397644,
      "reward_std": 0.6423742175102234,
      "rewards/reward_function": 0.686827540397644,
      "step": 89
    },
    {
      "completion_length": 77.89286041259766,
      "epoch": 0.05148741418764302,
      "grad_norm": 466197.09375,
      "kl": 12107.435546875,
      "learning_rate": 2.571428571428571e-06,
      "loss": 484.2975,
      "reward": 0.557130753993988,
      "reward_std": 0.6012104749679565,
      "rewards/reward_function": 0.557130753993988,
      "step": 90
    },
    {
      "completion_length": 63.21428680419922,
      "epoch": 0.05205949656750572,
      "grad_norm": 231667.78125,
      "kl": 1100.36279296875,
      "learning_rate": 2.6e-06,
      "loss": 44.0145,
      "reward": 0.9704992771148682,
      "reward_std": 0.7451456785202026,
      "rewards/reward_function": 0.9704992771148682,
      "step": 91
    },
    {
      "completion_length": 94.75000762939453,
      "epoch": 0.05263157894736842,
      "grad_norm": 244107.015625,
      "kl": 1554.5328369140625,
      "learning_rate": 2.6285714285714286e-06,
      "loss": 62.1813,
      "reward": 0.8950142860412598,
      "reward_std": 0.7484855055809021,
      "rewards/reward_function": 0.8950142860412598,
      "step": 92
    },
    {
      "completion_length": 64.10714721679688,
      "epoch": 0.05320366132723112,
      "grad_norm": 37978248.0,
      "kl": 702552.3125,
      "learning_rate": 2.6571428571428575e-06,
      "loss": 28102.0859,
      "reward": 1.08194899559021,
      "reward_std": 0.8591198921203613,
      "rewards/reward_function": 1.08194899559021,
      "step": 93
    },
    {
      "completion_length": 73.10714721679688,
      "epoch": 0.05377574370709382,
      "grad_norm": 370171.28125,
      "kl": 6021.37451171875,
      "learning_rate": 2.685714285714286e-06,
      "loss": 240.855,
      "reward": 1.0200893878936768,
      "reward_std": 0.8128372430801392,
      "rewards/reward_function": 1.0200893878936768,
      "step": 94
    },
    {
      "completion_length": 81.53572082519531,
      "epoch": 0.05434782608695652,
      "grad_norm": 1765.568359375,
      "kl": 20.953439712524414,
      "learning_rate": 2.7142857142857144e-06,
      "loss": 0.8381,
      "reward": 0.8016756176948547,
      "reward_std": 0.6009109020233154,
      "rewards/reward_function": 0.8016756176948547,
      "step": 95
    },
    {
      "completion_length": 69.14286041259766,
      "epoch": 0.05491990846681922,
      "grad_norm": 378837.90625,
      "kl": 11462.9970703125,
      "learning_rate": 2.7428571428571433e-06,
      "loss": 458.5199,
      "reward": 0.8851485252380371,
      "reward_std": 0.8089224100112915,
      "rewards/reward_function": 0.8851485252380371,
      "step": 96
    },
    {
      "completion_length": 93.17857360839844,
      "epoch": 0.055491990846681924,
      "grad_norm": 1664503.75,
      "kl": 23976.85546875,
      "learning_rate": 2.7714285714285714e-06,
      "loss": 959.0743,
      "reward": 1.3467190265655518,
      "reward_std": 1.0991709232330322,
      "rewards/reward_function": 1.3467190265655518,
      "step": 97
    },
    {
      "completion_length": 96.78572082519531,
      "epoch": 0.05606407322654462,
      "grad_norm": 15987.314453125,
      "kl": 3734.636962890625,
      "learning_rate": 2.8000000000000003e-06,
      "loss": 149.3855,
      "reward": 0.40695470571517944,
      "reward_std": 0.4793854057788849,
      "rewards/reward_function": 0.40695470571517944,
      "step": 98
    },
    {
      "completion_length": 83.35714721679688,
      "epoch": 0.05663615560640732,
      "grad_norm": 237164.53125,
      "kl": 13279.5849609375,
      "learning_rate": 2.8285714285714288e-06,
      "loss": 531.1834,
      "reward": 0.6734562516212463,
      "reward_std": 0.7493541240692139,
      "rewards/reward_function": 0.6734562516212463,
      "step": 99
    },
    {
      "completion_length": 66.21428680419922,
      "epoch": 0.057208237986270026,
      "grad_norm": 101.31340789794922,
      "kl": 5.006754398345947,
      "learning_rate": 2.8571428571428573e-06,
      "loss": 0.2003,
      "reward": 0.993274986743927,
      "reward_std": 0.6643393635749817,
      "rewards/reward_function": 0.993274986743927,
      "step": 100
    },
    {
      "completion_length": 77.64286041259766,
      "epoch": 0.057780320366132724,
      "grad_norm": 995936.5,
      "kl": 93690.4609375,
      "learning_rate": 2.885714285714286e-06,
      "loss": 3747.6182,
      "reward": 1.3292661905288696,
      "reward_std": 1.1285889148712158,
      "rewards/reward_function": 1.3292661905288696,
      "step": 101
    },
    {
      "completion_length": 69.5,
      "epoch": 0.05835240274599542,
      "grad_norm": 63318.06640625,
      "kl": 9366.99609375,
      "learning_rate": 2.9142857142857146e-06,
      "loss": 374.6798,
      "reward": 0.709989607334137,
      "reward_std": 0.589360237121582,
      "rewards/reward_function": 0.709989607334137,
      "step": 102
    },
    {
      "completion_length": 87.85714721679688,
      "epoch": 0.05892448512585812,
      "grad_norm": 952.90625,
      "kl": 11.19540786743164,
      "learning_rate": 2.9428571428571427e-06,
      "loss": 0.4478,
      "reward": 0.7803307175636292,
      "reward_std": 0.9410938024520874,
      "rewards/reward_function": 0.7803307175636292,
      "step": 103
    },
    {
      "completion_length": 61.03571701049805,
      "epoch": 0.059496567505720827,
      "grad_norm": 286.7213134765625,
      "kl": 6.7517619132995605,
      "learning_rate": 2.9714285714285716e-06,
      "loss": 0.2701,
      "reward": 1.2457932233810425,
      "reward_std": 0.8239418268203735,
      "rewards/reward_function": 1.2457932233810425,
      "step": 104
    },
    {
      "completion_length": 65.5,
      "epoch": 0.060068649885583525,
      "grad_norm": 2438651.25,
      "kl": 29405.662109375,
      "learning_rate": 3e-06,
      "loss": 1176.2268,
      "reward": 1.019971251487732,
      "reward_std": 1.0803383588790894,
      "rewards/reward_function": 1.019971251487732,
      "step": 105
    },
    {
      "completion_length": 64.5,
      "epoch": 0.06064073226544622,
      "grad_norm": 6.428584098815918,
      "kl": 1.217685580253601,
      "learning_rate": 3.028571428571429e-06,
      "loss": 0.0487,
      "reward": 1.3005273342132568,
      "reward_std": 0.9520907402038574,
      "rewards/reward_function": 1.3005273342132568,
      "step": 106
    },
    {
      "completion_length": 76.0,
      "epoch": 0.06121281464530892,
      "grad_norm": 75623.8203125,
      "kl": 4889.19775390625,
      "learning_rate": 3.0571428571428575e-06,
      "loss": 195.5678,
      "reward": 0.38282328844070435,
      "reward_std": 0.3451842665672302,
      "rewards/reward_function": 0.38282328844070435,
      "step": 107
    },
    {
      "completion_length": 80.92857360839844,
      "epoch": 0.06178489702517163,
      "grad_norm": 493.4114685058594,
      "kl": 8.082494735717773,
      "learning_rate": 3.085714285714286e-06,
      "loss": 0.3233,
      "reward": 0.7522178888320923,
      "reward_std": 0.698627769947052,
      "rewards/reward_function": 0.7522178888320923,
      "step": 108
    },
    {
      "completion_length": 71.60714721679688,
      "epoch": 0.062356979405034325,
      "grad_norm": 7672.40771484375,
      "kl": 127.68350219726562,
      "learning_rate": 3.114285714285715e-06,
      "loss": 5.1073,
      "reward": 1.0444462299346924,
      "reward_std": 0.9674249887466431,
      "rewards/reward_function": 1.0444462299346924,
      "step": 109
    },
    {
      "completion_length": 88.50000762939453,
      "epoch": 0.06292906178489703,
      "grad_norm": 133487.328125,
      "kl": 8653.7978515625,
      "learning_rate": 3.142857142857143e-06,
      "loss": 346.1519,
      "reward": 0.9829298853874207,
      "reward_std": 0.9203434586524963,
      "rewards/reward_function": 0.9829298853874207,
      "step": 110
    },
    {
      "completion_length": 71.71428680419922,
      "epoch": 0.06350114416475973,
      "grad_norm": 1.860447645187378,
      "kl": 0.5260595679283142,
      "learning_rate": 3.1714285714285714e-06,
      "loss": 0.021,
      "reward": 1.2717598676681519,
      "reward_std": 0.9790816903114319,
      "rewards/reward_function": 1.2717598676681519,
      "step": 111
    },
    {
      "completion_length": 90.67857360839844,
      "epoch": 0.06407322654462243,
      "grad_norm": 78477.1875,
      "kl": 13047.017578125,
      "learning_rate": 3.2000000000000003e-06,
      "loss": 521.8807,
      "reward": 1.0069576501846313,
      "reward_std": 0.6037776470184326,
      "rewards/reward_function": 1.0069576501846313,
      "step": 112
    },
    {
      "completion_length": 64.25,
      "epoch": 0.06464530892448513,
      "grad_norm": 603.7963256835938,
      "kl": 94.09223937988281,
      "learning_rate": 3.2285714285714288e-06,
      "loss": 3.7637,
      "reward": 0.31649214029312134,
      "reward_std": 0.29072389006614685,
      "rewards/reward_function": 0.31649214029312134,
      "step": 113
    },
    {
      "completion_length": 68.67857360839844,
      "epoch": 0.06521739130434782,
      "grad_norm": 200319.09375,
      "kl": 3849.8876953125,
      "learning_rate": 3.2571428571428577e-06,
      "loss": 153.9955,
      "reward": 0.4277487099170685,
      "reward_std": 0.34380677342414856,
      "rewards/reward_function": 0.4277487099170685,
      "step": 114
    },
    {
      "completion_length": 105.0714340209961,
      "epoch": 0.06578947368421052,
      "grad_norm": 533921.625,
      "kl": 10240.3037109375,
      "learning_rate": 3.285714285714286e-06,
      "loss": 409.6122,
      "reward": 1.142284870147705,
      "reward_std": 0.9854078888893127,
      "rewards/reward_function": 1.142284870147705,
      "step": 115
    },
    {
      "completion_length": 68.17857360839844,
      "epoch": 0.06636155606407322,
      "grad_norm": 406653376.0,
      "kl": 1482348.375,
      "learning_rate": 3.314285714285714e-06,
      "loss": 59293.9414,
      "reward": 0.5077195167541504,
      "reward_std": 0.5461395382881165,
      "rewards/reward_function": 0.5077195167541504,
      "step": 116
    },
    {
      "completion_length": 86.39286041259766,
      "epoch": 0.06693363844393593,
      "grad_norm": 13590.787109375,
      "kl": 28.27956771850586,
      "learning_rate": 3.342857142857143e-06,
      "loss": 1.1312,
      "reward": 1.1498898267745972,
      "reward_std": 1.0251048803329468,
      "rewards/reward_function": 1.1498898267745972,
      "step": 117
    },
    {
      "completion_length": 66.03572082519531,
      "epoch": 0.06750572082379863,
      "grad_norm": 1.2044802904129028,
      "kl": 0.4519920349121094,
      "learning_rate": 3.3714285714285716e-06,
      "loss": 0.0181,
      "reward": 0.6916710138320923,
      "reward_std": 0.7077118754386902,
      "rewards/reward_function": 0.6916710138320923,
      "step": 118
    },
    {
      "completion_length": 106.03572082519531,
      "epoch": 0.06807780320366133,
      "grad_norm": 53712.6484375,
      "kl": 2227.35400390625,
      "learning_rate": 3.4000000000000005e-06,
      "loss": 89.0942,
      "reward": 0.9127389788627625,
      "reward_std": 1.035290002822876,
      "rewards/reward_function": 0.9127389788627625,
      "step": 119
    },
    {
      "completion_length": 99.3214340209961,
      "epoch": 0.06864988558352403,
      "grad_norm": 14194080.0,
      "kl": 155773.203125,
      "learning_rate": 3.428571428571429e-06,
      "loss": 6230.9272,
      "reward": 0.8683643937110901,
      "reward_std": 0.6231727600097656,
      "rewards/reward_function": 0.8683643937110901,
      "step": 120
    },
    {
      "completion_length": 78.10714721679688,
      "epoch": 0.06922196796338673,
      "grad_norm": 389000.6875,
      "kl": 6245.36083984375,
      "learning_rate": 3.4571428571428574e-06,
      "loss": 249.8144,
      "reward": 0.7813895344734192,
      "reward_std": 0.955848217010498,
      "rewards/reward_function": 0.7813895344734192,
      "step": 121
    },
    {
      "completion_length": 79.5,
      "epoch": 0.06979405034324943,
      "grad_norm": 5831.4912109375,
      "kl": 180.41339111328125,
      "learning_rate": 3.4857142857142863e-06,
      "loss": 7.2165,
      "reward": 0.8601799607276917,
      "reward_std": 0.8076702356338501,
      "rewards/reward_function": 0.8601799607276917,
      "step": 122
    },
    {
      "completion_length": 70.53572082519531,
      "epoch": 0.07036613272311212,
      "grad_norm": 874913.5,
      "kl": 1734.0245361328125,
      "learning_rate": 3.5142857142857144e-06,
      "loss": 69.361,
      "reward": 0.461391806602478,
      "reward_std": 0.5364981889724731,
      "rewards/reward_function": 0.461391806602478,
      "step": 123
    },
    {
      "completion_length": 63.32143020629883,
      "epoch": 0.07093821510297482,
      "grad_norm": 4.4475603103637695,
      "kl": 0.569288969039917,
      "learning_rate": 3.542857142857143e-06,
      "loss": 0.0228,
      "reward": 0.9151536226272583,
      "reward_std": 0.6034038066864014,
      "rewards/reward_function": 0.9151536226272583,
      "step": 124
    },
    {
      "completion_length": 77.96428680419922,
      "epoch": 0.07151029748283753,
      "grad_norm": 11923.443359375,
      "kl": 523.2318725585938,
      "learning_rate": 3.5714285714285718e-06,
      "loss": 20.9293,
      "reward": 1.048008918762207,
      "reward_std": 1.079918622970581,
      "rewards/reward_function": 1.048008918762207,
      "step": 125
    },
    {
      "completion_length": 100.3214340209961,
      "epoch": 0.07208237986270023,
      "grad_norm": 503.06268310546875,
      "kl": 19.8065242767334,
      "learning_rate": 3.6000000000000003e-06,
      "loss": 0.7923,
      "reward": 0.7785278558731079,
      "reward_std": 0.9978376626968384,
      "rewards/reward_function": 0.7785278558731079,
      "step": 126
    },
    {
      "completion_length": 112.96429443359375,
      "epoch": 0.07265446224256293,
      "grad_norm": 83295.21875,
      "kl": 5862.4111328125,
      "learning_rate": 3.628571428571429e-06,
      "loss": 234.4964,
      "reward": 0.5598422288894653,
      "reward_std": 0.7298100590705872,
      "rewards/reward_function": 0.5598422288894653,
      "step": 127
    },
    {
      "completion_length": 80.28572082519531,
      "epoch": 0.07322654462242563,
      "grad_norm": 129503.1796875,
      "kl": 22259.490234375,
      "learning_rate": 3.6571428571428576e-06,
      "loss": 890.3797,
      "reward": 0.9316549897193909,
      "reward_std": 0.9885486364364624,
      "rewards/reward_function": 0.9316549897193909,
      "step": 128
    },
    {
      "completion_length": 68.03572082519531,
      "epoch": 0.07379862700228833,
      "grad_norm": 7.6474456787109375,
      "kl": 0.8037868142127991,
      "learning_rate": 3.6857142857142857e-06,
      "loss": 0.0322,
      "reward": 0.6243561506271362,
      "reward_std": 0.5376760959625244,
      "rewards/reward_function": 0.6243561506271362,
      "step": 129
    },
    {
      "completion_length": 96.85714721679688,
      "epoch": 0.07437070938215103,
      "grad_norm": 495505.0,
      "kl": 29576.54296875,
      "learning_rate": 3.7142857142857146e-06,
      "loss": 1183.0618,
      "reward": 0.8316521048545837,
      "reward_std": 0.7780259251594543,
      "rewards/reward_function": 0.8316521048545837,
      "step": 130
    },
    {
      "completion_length": 67.46428680419922,
      "epoch": 0.07494279176201372,
      "grad_norm": 27308.521484375,
      "kl": 351.64202880859375,
      "learning_rate": 3.742857142857143e-06,
      "loss": 14.0657,
      "reward": 0.8038970232009888,
      "reward_std": 0.8452528715133667,
      "rewards/reward_function": 0.8038970232009888,
      "step": 131
    },
    {
      "completion_length": 84.8214340209961,
      "epoch": 0.07551487414187644,
      "grad_norm": 78730.859375,
      "kl": 2872.984619140625,
      "learning_rate": 3.771428571428572e-06,
      "loss": 114.9194,
      "reward": 1.3303391933441162,
      "reward_std": 0.713073194026947,
      "rewards/reward_function": 1.3303391933441162,
      "step": 132
    },
    {
      "completion_length": 80.60714721679688,
      "epoch": 0.07608695652173914,
      "grad_norm": 7094.54736328125,
      "kl": 315.4158630371094,
      "learning_rate": 3.8000000000000005e-06,
      "loss": 12.6166,
      "reward": 1.0466818809509277,
      "reward_std": 1.0060451030731201,
      "rewards/reward_function": 1.0466818809509277,
      "step": 133
    },
    {
      "completion_length": 65.21428680419922,
      "epoch": 0.07665903890160183,
      "grad_norm": 1843674.625,
      "kl": 9586.375,
      "learning_rate": 3.828571428571429e-06,
      "loss": 383.455,
      "reward": 1.1340609788894653,
      "reward_std": 0.790274441242218,
      "rewards/reward_function": 1.1340609788894653,
      "step": 134
    },
    {
      "completion_length": 72.39286041259766,
      "epoch": 0.07723112128146453,
      "grad_norm": 72.35924530029297,
      "kl": 2.4562935829162598,
      "learning_rate": 3.857142857142858e-06,
      "loss": 0.0983,
      "reward": 0.6875572800636292,
      "reward_std": 0.7661923766136169,
      "rewards/reward_function": 0.6875572800636292,
      "step": 135
    },
    {
      "completion_length": 83.3214340209961,
      "epoch": 0.07780320366132723,
      "grad_norm": 66.0393295288086,
      "kl": 7.746999263763428,
      "learning_rate": 3.885714285714286e-06,
      "loss": 0.3099,
      "reward": 0.766676664352417,
      "reward_std": 0.7378672957420349,
      "rewards/reward_function": 0.766676664352417,
      "step": 136
    },
    {
      "completion_length": 103.42857360839844,
      "epoch": 0.07837528604118993,
      "grad_norm": 857859.3125,
      "kl": 71858.03125,
      "learning_rate": 3.914285714285714e-06,
      "loss": 2874.322,
      "reward": 1.444486141204834,
      "reward_std": 1.118514060974121,
      "rewards/reward_function": 1.444486141204834,
      "step": 137
    },
    {
      "completion_length": 57.67857360839844,
      "epoch": 0.07894736842105263,
      "grad_norm": 5.886438369750977,
      "kl": 0.6165660619735718,
      "learning_rate": 3.942857142857143e-06,
      "loss": 0.0247,
      "reward": 0.99308180809021,
      "reward_std": 0.5322183966636658,
      "rewards/reward_function": 0.99308180809021,
      "step": 138
    },
    {
      "completion_length": 64.28572082519531,
      "epoch": 0.07951945080091533,
      "grad_norm": 48290.0859375,
      "kl": 3181.72802734375,
      "learning_rate": 3.971428571428572e-06,
      "loss": 127.2691,
      "reward": 0.9975389242172241,
      "reward_std": 0.8097325563430786,
      "rewards/reward_function": 0.9975389242172241,
      "step": 139
    },
    {
      "completion_length": 57.892860412597656,
      "epoch": 0.08009153318077804,
      "grad_norm": 1.422910451889038,
      "kl": 0.4653211832046509,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0186,
      "reward": 0.7076358199119568,
      "reward_std": 0.7446828484535217,
      "rewards/reward_function": 0.7076358199119568,
      "step": 140
    },
    {
      "completion_length": 70.53572082519531,
      "epoch": 0.08066361556064074,
      "grad_norm": 137620.546875,
      "kl": 3363.561279296875,
      "learning_rate": 4.028571428571429e-06,
      "loss": 134.5424,
      "reward": 1.0708597898483276,
      "reward_std": 0.9894415736198425,
      "rewards/reward_function": 1.0708597898483276,
      "step": 141
    },
    {
      "completion_length": 67.5,
      "epoch": 0.08123569794050343,
      "grad_norm": 11.5463285446167,
      "kl": 1.222627878189087,
      "learning_rate": 4.057142857142858e-06,
      "loss": 0.0489,
      "reward": 0.9060318470001221,
      "reward_std": 0.790241539478302,
      "rewards/reward_function": 0.9060318470001221,
      "step": 142
    },
    {
      "completion_length": 61.857147216796875,
      "epoch": 0.08180778032036613,
      "grad_norm": 894.0208129882812,
      "kl": 80.05220794677734,
      "learning_rate": 4.0857142857142865e-06,
      "loss": 3.2021,
      "reward": 1.4151250123977661,
      "reward_std": 0.9148633480072021,
      "rewards/reward_function": 1.4151250123977661,
      "step": 143
    },
    {
      "completion_length": 94.03572082519531,
      "epoch": 0.08237986270022883,
      "grad_norm": 1100.770263671875,
      "kl": 116.94873046875,
      "learning_rate": 4.114285714285715e-06,
      "loss": 4.6779,
      "reward": 0.618543267250061,
      "reward_std": 0.7632157802581787,
      "rewards/reward_function": 0.618543267250061,
      "step": 144
    },
    {
      "completion_length": 67.8214340209961,
      "epoch": 0.08295194508009153,
      "grad_norm": 1.8865011930465698,
      "kl": 0.5115684866905212,
      "learning_rate": 4.1428571428571435e-06,
      "loss": 0.0205,
      "reward": 0.9246187210083008,
      "reward_std": 0.7462993860244751,
      "rewards/reward_function": 0.9246187210083008,
      "step": 145
    },
    {
      "completion_length": 90.78572082519531,
      "epoch": 0.08352402745995423,
      "grad_norm": 18.561193466186523,
      "kl": 1.08840012550354,
      "learning_rate": 4.1714285714285715e-06,
      "loss": 0.0435,
      "reward": 0.9311470985412598,
      "reward_std": 0.9502328038215637,
      "rewards/reward_function": 0.9311470985412598,
      "step": 146
    },
    {
      "completion_length": 64.17857360839844,
      "epoch": 0.08409610983981694,
      "grad_norm": 14483.7060546875,
      "kl": 754.9644165039062,
      "learning_rate": 4.2000000000000004e-06,
      "loss": 30.1986,
      "reward": 0.5451472997665405,
      "reward_std": 0.542201042175293,
      "rewards/reward_function": 0.5451472997665405,
      "step": 147
    },
    {
      "completion_length": 76.35714721679688,
      "epoch": 0.08466819221967964,
      "grad_norm": 14091.421875,
      "kl": 1191.689208984375,
      "learning_rate": 4.228571428571429e-06,
      "loss": 47.6676,
      "reward": 1.4179259538650513,
      "reward_std": 1.0224483013153076,
      "rewards/reward_function": 1.4179259538650513,
      "step": 148
    },
    {
      "completion_length": 85.35714721679688,
      "epoch": 0.08524027459954234,
      "grad_norm": 55.49179458618164,
      "kl": 3.497796058654785,
      "learning_rate": 4.257142857142857e-06,
      "loss": 0.1399,
      "reward": 1.0712175369262695,
      "reward_std": 0.9481403231620789,
      "rewards/reward_function": 1.0712175369262695,
      "step": 149
    },
    {
      "completion_length": 80.25,
      "epoch": 0.08581235697940504,
      "grad_norm": 1732.9415283203125,
      "kl": 222.05569458007812,
      "learning_rate": 4.2857142857142855e-06,
      "loss": 8.8822,
      "reward": 0.7208712697029114,
      "reward_std": 0.7181576490402222,
      "rewards/reward_function": 0.7208712697029114,
      "step": 150
    },
    {
      "completion_length": 84.3214340209961,
      "epoch": 0.08638443935926773,
      "grad_norm": 105920.0546875,
      "kl": 9658.7958984375,
      "learning_rate": 4.314285714285714e-06,
      "loss": 386.3517,
      "reward": 0.8224695324897766,
      "reward_std": 0.7860068678855896,
      "rewards/reward_function": 0.8224695324897766,
      "step": 151
    },
    {
      "completion_length": 64.42857360839844,
      "epoch": 0.08695652173913043,
      "grad_norm": 7130.55419921875,
      "kl": 1530.49853515625,
      "learning_rate": 4.342857142857143e-06,
      "loss": 61.22,
      "reward": 1.0754064321517944,
      "reward_std": 0.9805803894996643,
      "rewards/reward_function": 1.0754064321517944,
      "step": 152
    },
    {
      "completion_length": 75.42857360839844,
      "epoch": 0.08752860411899313,
      "grad_norm": 147.3867645263672,
      "kl": 7.4129438400268555,
      "learning_rate": 4.371428571428572e-06,
      "loss": 0.2965,
      "reward": 0.9033632874488831,
      "reward_std": 0.9660179018974304,
      "rewards/reward_function": 0.9033632874488831,
      "step": 153
    },
    {
      "completion_length": 81.03572082519531,
      "epoch": 0.08810068649885583,
      "grad_norm": 11980.23828125,
      "kl": 286.21148681640625,
      "learning_rate": 4.4e-06,
      "loss": 11.4485,
      "reward": 0.998909056186676,
      "reward_std": 0.9239910840988159,
      "rewards/reward_function": 0.998909056186676,
      "step": 154
    },
    {
      "completion_length": 91.03572082519531,
      "epoch": 0.08867276887871854,
      "grad_norm": 28285.22265625,
      "kl": 377.3843078613281,
      "learning_rate": 4.428571428571429e-06,
      "loss": 15.0954,
      "reward": 1.0054373741149902,
      "reward_std": 0.7861291170120239,
      "rewards/reward_function": 1.0054373741149902,
      "step": 155
    },
    {
      "completion_length": 81.0,
      "epoch": 0.08924485125858124,
      "grad_norm": 50.57877731323242,
      "kl": 5.108626842498779,
      "learning_rate": 4.457142857142858e-06,
      "loss": 0.2043,
      "reward": 1.3511475324630737,
      "reward_std": 1.0390514135360718,
      "rewards/reward_function": 1.3511475324630737,
      "step": 156
    },
    {
      "completion_length": 52.250003814697266,
      "epoch": 0.08981693363844394,
      "grad_norm": 277.97015380859375,
      "kl": 15.708544731140137,
      "learning_rate": 4.485714285714286e-06,
      "loss": 0.6283,
      "reward": 0.7682435512542725,
      "reward_std": 0.7497861981391907,
      "rewards/reward_function": 0.7682435512542725,
      "step": 157
    },
    {
      "completion_length": 69.42857360839844,
      "epoch": 0.09038901601830664,
      "grad_norm": 58.2374153137207,
      "kl": 13.843992233276367,
      "learning_rate": 4.514285714285714e-06,
      "loss": 0.5538,
      "reward": 0.9975533485412598,
      "reward_std": 0.9383617639541626,
      "rewards/reward_function": 0.9975533485412598,
      "step": 158
    },
    {
      "completion_length": 88.14286041259766,
      "epoch": 0.09096109839816933,
      "grad_norm": 11.953612327575684,
      "kl": 0.5367157459259033,
      "learning_rate": 4.542857142857143e-06,
      "loss": 0.0215,
      "reward": 1.0708742141723633,
      "reward_std": 0.7348182201385498,
      "rewards/reward_function": 1.0708742141723633,
      "step": 159
    },
    {
      "completion_length": 65.21428680419922,
      "epoch": 0.09153318077803203,
      "grad_norm": 10747.470703125,
      "kl": 57.00440979003906,
      "learning_rate": 4.571428571428572e-06,
      "loss": 2.2802,
      "reward": 1.1665701866149902,
      "reward_std": 0.6126526594161987,
      "rewards/reward_function": 1.1665701866149902,
      "step": 160
    },
    {
      "completion_length": 63.750003814697266,
      "epoch": 0.09210526315789473,
      "grad_norm": 1.6591835021972656,
      "kl": 0.4152368903160095,
      "learning_rate": 4.600000000000001e-06,
      "loss": 0.0166,
      "reward": 1.22443425655365,
      "reward_std": 0.7108891606330872,
      "rewards/reward_function": 1.22443425655365,
      "step": 161
    },
    {
      "completion_length": 76.71428680419922,
      "epoch": 0.09267734553775744,
      "grad_norm": 39.48670959472656,
      "kl": 1.8404861688613892,
      "learning_rate": 4.628571428571429e-06,
      "loss": 0.0736,
      "reward": 1.224462866783142,
      "reward_std": 1.0679142475128174,
      "rewards/reward_function": 1.224462866783142,
      "step": 162
    },
    {
      "completion_length": 65.3214340209961,
      "epoch": 0.09324942791762014,
      "grad_norm": 80.05848693847656,
      "kl": 2.4652302265167236,
      "learning_rate": 4.657142857142857e-06,
      "loss": 0.0986,
      "reward": 1.0068539381027222,
      "reward_std": 0.724425196647644,
      "rewards/reward_function": 1.0068539381027222,
      "step": 163
    },
    {
      "completion_length": 71.8214340209961,
      "epoch": 0.09382151029748284,
      "grad_norm": 17.541017532348633,
      "kl": 0.8244562745094299,
      "learning_rate": 4.685714285714286e-06,
      "loss": 0.033,
      "reward": 0.6696106791496277,
      "reward_std": 0.7001439929008484,
      "rewards/reward_function": 0.6696106791496277,
      "step": 164
    },
    {
      "completion_length": 78.5714340209961,
      "epoch": 0.09439359267734554,
      "grad_norm": 2717793.75,
      "kl": 26785.45703125,
      "learning_rate": 4.714285714285715e-06,
      "loss": 1071.4181,
      "reward": 0.7027172446250916,
      "reward_std": 0.8397053480148315,
      "rewards/reward_function": 0.7027172446250916,
      "step": 165
    },
    {
      "completion_length": 95.78572082519531,
      "epoch": 0.09496567505720824,
      "grad_norm": 2365.997314453125,
      "kl": 44.28641891479492,
      "learning_rate": 4.742857142857144e-06,
      "loss": 1.7715,
      "reward": 1.557921290397644,
      "reward_std": 0.9112396240234375,
      "rewards/reward_function": 1.557921290397644,
      "step": 166
    },
    {
      "completion_length": 85.39286041259766,
      "epoch": 0.09553775743707094,
      "grad_norm": 2472.313720703125,
      "kl": 25.077659606933594,
      "learning_rate": 4.771428571428572e-06,
      "loss": 1.0031,
      "reward": 0.93460613489151,
      "reward_std": 1.0219250917434692,
      "rewards/reward_function": 0.93460613489151,
      "step": 167
    },
    {
      "completion_length": 53.67857360839844,
      "epoch": 0.09610983981693363,
      "grad_norm": 23.110355377197266,
      "kl": 1.082519292831421,
      "learning_rate": 4.800000000000001e-06,
      "loss": 0.0433,
      "reward": 1.4396891593933105,
      "reward_std": 1.067988395690918,
      "rewards/reward_function": 1.4396891593933105,
      "step": 168
    },
    {
      "completion_length": 72.42857360839844,
      "epoch": 0.09668192219679633,
      "grad_norm": 3838.635986328125,
      "kl": 208.36264038085938,
      "learning_rate": 4.8285714285714295e-06,
      "loss": 8.3345,
      "reward": 0.8054172396659851,
      "reward_std": 0.7945077419281006,
      "rewards/reward_function": 0.8054172396659851,
      "step": 169
    },
    {
      "completion_length": 125.71429443359375,
      "epoch": 0.09725400457665904,
      "grad_norm": 1402.7926025390625,
      "kl": 94.01925659179688,
      "learning_rate": 4.857142857142858e-06,
      "loss": 3.7608,
      "reward": 1.8190248012542725,
      "reward_std": 0.9309060573577881,
      "rewards/reward_function": 1.8190248012542725,
      "step": 170
    },
    {
      "completion_length": 80.53572082519531,
      "epoch": 0.09782608695652174,
      "grad_norm": 5824.87451171875,
      "kl": 343.9230041503906,
      "learning_rate": 4.885714285714286e-06,
      "loss": 13.7569,
      "reward": 1.057026982307434,
      "reward_std": 0.9848878383636475,
      "rewards/reward_function": 1.057026982307434,
      "step": 171
    },
    {
      "completion_length": 100.53572082519531,
      "epoch": 0.09839816933638444,
      "grad_norm": 16446.216796875,
      "kl": 768.9603881835938,
      "learning_rate": 4.9142857142857145e-06,
      "loss": 30.7584,
      "reward": 0.7897814512252808,
      "reward_std": 0.7102627158164978,
      "rewards/reward_function": 0.7897814512252808,
      "step": 172
    },
    {
      "completion_length": 103.5714340209961,
      "epoch": 0.09897025171624714,
      "grad_norm": 974687.0,
      "kl": 18556.869140625,
      "learning_rate": 4.9428571428571435e-06,
      "loss": 742.2748,
      "reward": 0.7110770344734192,
      "reward_std": 0.8649537563323975,
      "rewards/reward_function": 0.7110770344734192,
      "step": 173
    },
    {
      "completion_length": 97.14286041259766,
      "epoch": 0.09954233409610984,
      "grad_norm": 379862.03125,
      "kl": 10978.4814453125,
      "learning_rate": 4.971428571428572e-06,
      "loss": 439.1393,
      "reward": 0.5969444513320923,
      "reward_std": 0.7491056323051453,
      "rewards/reward_function": 0.5969444513320923,
      "step": 174
    },
    {
      "completion_length": 82.21428680419922,
      "epoch": 0.10011441647597254,
      "grad_norm": 4166.083984375,
      "kl": 28.75504493713379,
      "learning_rate": 5e-06,
      "loss": 1.1502,
      "reward": 1.0659698247909546,
      "reward_std": 0.6791222095489502,
      "rewards/reward_function": 1.0659698247909546,
      "step": 175
    },
    {
      "completion_length": 71.89286041259766,
      "epoch": 0.10068649885583524,
      "grad_norm": 2179.628662109375,
      "kl": 144.59950256347656,
      "learning_rate": 4.99999501400121e-06,
      "loss": 5.784,
      "reward": 1.2910478115081787,
      "reward_std": 0.823914110660553,
      "rewards/reward_function": 1.2910478115081787,
      "step": 176
    },
    {
      "completion_length": 73.92857360839844,
      "epoch": 0.10125858123569793,
      "grad_norm": 479066.8125,
      "kl": 20347.021484375,
      "learning_rate": 4.999980056024728e-06,
      "loss": 813.8809,
      "reward": 1.221840739250183,
      "reward_std": 0.996203601360321,
      "rewards/reward_function": 1.221840739250183,
      "step": 177
    },
    {
      "completion_length": 79.96428680419922,
      "epoch": 0.10183066361556065,
      "grad_norm": 481.5789794921875,
      "kl": 2.926368236541748,
      "learning_rate": 4.999955126130217e-06,
      "loss": 0.1171,
      "reward": 0.9862995147705078,
      "reward_std": 0.8244569301605225,
      "rewards/reward_function": 0.9862995147705078,
      "step": 178
    },
    {
      "completion_length": 107.00000762939453,
      "epoch": 0.10240274599542334,
      "grad_norm": 8741.9775390625,
      "kl": 492.6190185546875,
      "learning_rate": 4.9999202244171195e-06,
      "loss": 19.7048,
      "reward": 0.8891870379447937,
      "reward_std": 0.7700316905975342,
      "rewards/reward_function": 0.8891870379447937,
      "step": 179
    },
    {
      "completion_length": 124.50000762939453,
      "epoch": 0.10297482837528604,
      "grad_norm": 1073.474365234375,
      "kl": 60.3995361328125,
      "learning_rate": 4.99987535102465e-06,
      "loss": 2.416,
      "reward": 1.0054516792297363,
      "reward_std": 0.7528944611549377,
      "rewards/reward_function": 1.0054516792297363,
      "step": 180
    },
    {
      "completion_length": 91.25000762939453,
      "epoch": 0.10354691075514874,
      "grad_norm": 1217.821044921875,
      "kl": 7.022561550140381,
      "learning_rate": 4.9998205061318e-06,
      "loss": 0.2809,
      "reward": 0.615159273147583,
      "reward_std": 0.5513620376586914,
      "rewards/reward_function": 0.615159273147583,
      "step": 181
    },
    {
      "completion_length": 65.39286041259766,
      "epoch": 0.10411899313501144,
      "grad_norm": 28.1700439453125,
      "kl": 1.7268632650375366,
      "learning_rate": 4.999755689957334e-06,
      "loss": 0.0691,
      "reward": 1.252636432647705,
      "reward_std": 0.7366302609443665,
      "rewards/reward_function": 1.252636432647705,
      "step": 182
    },
    {
      "completion_length": 64.0714340209961,
      "epoch": 0.10469107551487414,
      "grad_norm": 1012.9512939453125,
      "kl": 43.76435470581055,
      "learning_rate": 4.999680902759792e-06,
      "loss": 1.7506,
      "reward": 0.681744396686554,
      "reward_std": 0.6758476495742798,
      "rewards/reward_function": 0.681744396686554,
      "step": 183
    },
    {
      "completion_length": 73.5,
      "epoch": 0.10526315789473684,
      "grad_norm": 17568.150390625,
      "kl": 1626.864013671875,
      "learning_rate": 4.999596144837484e-06,
      "loss": 65.0746,
      "reward": 0.8539949655532837,
      "reward_std": 0.7934482097625732,
      "rewards/reward_function": 0.8539949655532837,
      "step": 184
    },
    {
      "completion_length": 71.64286041259766,
      "epoch": 0.10583524027459955,
      "grad_norm": 1.0253714323043823,
      "kl": 0.4322874844074249,
      "learning_rate": 4.999501416528493e-06,
      "loss": 0.0173,
      "reward": 0.7698962092399597,
      "reward_std": 0.6341173052787781,
      "rewards/reward_function": 0.7698962092399597,
      "step": 185
    },
    {
      "completion_length": 95.42857360839844,
      "epoch": 0.10640732265446225,
      "grad_norm": 1096.26220703125,
      "kl": 57.60231018066406,
      "learning_rate": 4.999396718210671e-06,
      "loss": 2.3041,
      "reward": 1.4765803813934326,
      "reward_std": 0.7597432732582092,
      "rewards/reward_function": 1.4765803813934326,
      "step": 186
    },
    {
      "completion_length": 83.03572082519531,
      "epoch": 0.10697940503432495,
      "grad_norm": 102.68016052246094,
      "kl": 4.27730131149292,
      "learning_rate": 4.999282050301638e-06,
      "loss": 0.1711,
      "reward": 0.8683788180351257,
      "reward_std": 0.5856199264526367,
      "rewards/reward_function": 0.8683788180351257,
      "step": 187
    },
    {
      "completion_length": 125.10714721679688,
      "epoch": 0.10755148741418764,
      "grad_norm": 428.9923400878906,
      "kl": 12.814004898071289,
      "learning_rate": 4.999157413258782e-06,
      "loss": 0.5126,
      "reward": 0.652408242225647,
      "reward_std": 0.7201288342475891,
      "rewards/reward_function": 0.652408242225647,
      "step": 188
    },
    {
      "completion_length": 84.92857360839844,
      "epoch": 0.10812356979405034,
      "grad_norm": 1725.2889404296875,
      "kl": 16.937419891357422,
      "learning_rate": 4.999022807579254e-06,
      "loss": 0.6775,
      "reward": 0.9787410497665405,
      "reward_std": 0.9816575050354004,
      "rewards/reward_function": 0.9787410497665405,
      "step": 189
    },
    {
      "completion_length": 89.92857360839844,
      "epoch": 0.10869565217391304,
      "grad_norm": 25.972246170043945,
      "kl": 1.7363359928131104,
      "learning_rate": 4.99887823379997e-06,
      "loss": 0.0695,
      "reward": 1.4062535762786865,
      "reward_std": 0.5764892101287842,
      "rewards/reward_function": 1.4062535762786865,
      "step": 190
    },
    {
      "completion_length": 91.53572082519531,
      "epoch": 0.10926773455377574,
      "grad_norm": 180.07806396484375,
      "kl": 4.618396759033203,
      "learning_rate": 4.9987236924976065e-06,
      "loss": 0.1847,
      "reward": 1.244584321975708,
      "reward_std": 1.016752004623413,
      "rewards/reward_function": 1.244584321975708,
      "step": 191
    },
    {
      "completion_length": 78.0714340209961,
      "epoch": 0.10983981693363844,
      "grad_norm": 1.8729695081710815,
      "kl": 0.40956297516822815,
      "learning_rate": 4.998559184288596e-06,
      "loss": 0.0164,
      "reward": 0.6941750049591064,
      "reward_std": 0.5893157720565796,
      "rewards/reward_function": 0.6941750049591064,
      "step": 192
    },
    {
      "completion_length": 83.28572082519531,
      "epoch": 0.11041189931350115,
      "grad_norm": 14.813088417053223,
      "kl": 0.9863696098327637,
      "learning_rate": 4.998384709829129e-06,
      "loss": 0.0395,
      "reward": 1.0009515285491943,
      "reward_std": 0.78996741771698,
      "rewards/reward_function": 1.0009515285491943,
      "step": 193
    },
    {
      "completion_length": 94.0714340209961,
      "epoch": 0.11098398169336385,
      "grad_norm": 94.07627868652344,
      "kl": 1.9381582736968994,
      "learning_rate": 4.9982002698151505e-06,
      "loss": 0.0775,
      "reward": 0.77436763048172,
      "reward_std": 0.7886732816696167,
      "rewards/reward_function": 0.77436763048172,
      "step": 194
    },
    {
      "completion_length": 75.0,
      "epoch": 0.11155606407322655,
      "grad_norm": 14.137426376342773,
      "kl": 0.7862874865531921,
      "learning_rate": 4.998005864982354e-06,
      "loss": 0.0315,
      "reward": 1.1896427869796753,
      "reward_std": 0.78732830286026,
      "rewards/reward_function": 1.1896427869796753,
      "step": 195
    },
    {
      "completion_length": 78.78572082519531,
      "epoch": 0.11212814645308924,
      "grad_norm": 9137.9970703125,
      "kl": 397.71356201171875,
      "learning_rate": 4.997801496106179e-06,
      "loss": 15.9085,
      "reward": 0.7418583631515503,
      "reward_std": 0.46573546528816223,
      "rewards/reward_function": 0.7418583631515503,
      "step": 196
    },
    {
      "completion_length": 78.21428680419922,
      "epoch": 0.11270022883295194,
      "grad_norm": 5157.177734375,
      "kl": 139.95361328125,
      "learning_rate": 4.997587164001815e-06,
      "loss": 5.5981,
      "reward": 1.199959397315979,
      "reward_std": 0.9939581751823425,
      "rewards/reward_function": 1.199959397315979,
      "step": 197
    },
    {
      "completion_length": 79.35714721679688,
      "epoch": 0.11327231121281464,
      "grad_norm": 5407.041015625,
      "kl": 868.0248413085938,
      "learning_rate": 4.9973628695241886e-06,
      "loss": 34.721,
      "reward": 0.7420980930328369,
      "reward_std": 0.8618729710578918,
      "rewards/reward_function": 0.7420980930328369,
      "step": 198
    },
    {
      "completion_length": 65.42857360839844,
      "epoch": 0.11384439359267734,
      "grad_norm": 1.4323068857192993,
      "kl": 0.38493773341178894,
      "learning_rate": 4.9971286135679656e-06,
      "loss": 0.0154,
      "reward": 1.2123005390167236,
      "reward_std": 0.56341153383255,
      "rewards/reward_function": 1.2123005390167236,
      "step": 199
    },
    {
      "completion_length": 58.78571701049805,
      "epoch": 0.11441647597254005,
      "grad_norm": 225.90330505371094,
      "kl": 1.9128228425979614,
      "learning_rate": 4.996884397067545e-06,
      "loss": 0.0765,
      "reward": 0.7615792751312256,
      "reward_std": 0.46201032400131226,
      "rewards/reward_function": 0.7615792751312256,
      "step": 200
    },
    {
      "completion_length": 75.92857360839844,
      "epoch": 0.11498855835240275,
      "grad_norm": 230199.125,
      "kl": 3328.978759765625,
      "learning_rate": 4.996630220997058e-06,
      "loss": 133.1591,
      "reward": 0.8519238233566284,
      "reward_std": 0.5434860587120056,
      "rewards/reward_function": 0.8519238233566284,
      "step": 201
    },
    {
      "completion_length": 106.96428680419922,
      "epoch": 0.11556064073226545,
      "grad_norm": 43.92433547973633,
      "kl": 2.2816171646118164,
      "learning_rate": 4.996366086370362e-06,
      "loss": 0.0913,
      "reward": 1.1624706983566284,
      "reward_std": 0.8627653121948242,
      "rewards/reward_function": 1.1624706983566284,
      "step": 202
    },
    {
      "completion_length": 116.00000762939453,
      "epoch": 0.11613272311212815,
      "grad_norm": 1.7102408409118652,
      "kl": 0.35919514298439026,
      "learning_rate": 4.996091994241037e-06,
      "loss": 0.0144,
      "reward": 1.157223105430603,
      "reward_std": 0.8623934388160706,
      "rewards/reward_function": 1.157223105430603,
      "step": 203
    },
    {
      "completion_length": 79.85714721679688,
      "epoch": 0.11670480549199085,
      "grad_norm": 1.7143895626068115,
      "kl": 0.42739465832710266,
      "learning_rate": 4.995807945702381e-06,
      "loss": 0.0171,
      "reward": 1.1716963052749634,
      "reward_std": 1.053682804107666,
      "rewards/reward_function": 1.1716963052749634,
      "step": 204
    },
    {
      "completion_length": 78.0,
      "epoch": 0.11727688787185354,
      "grad_norm": 33.80851745605469,
      "kl": 2.6532204151153564,
      "learning_rate": 4.995513941887406e-06,
      "loss": 0.1061,
      "reward": 1.0898795127868652,
      "reward_std": 0.7144097685813904,
      "rewards/reward_function": 1.0898795127868652,
      "step": 205
    },
    {
      "completion_length": 103.71428680419922,
      "epoch": 0.11784897025171624,
      "grad_norm": 17.931734085083008,
      "kl": 2.4464528560638428,
      "learning_rate": 4.995209983968836e-06,
      "loss": 0.0979,
      "reward": 1.221601128578186,
      "reward_std": 0.7010052800178528,
      "rewards/reward_function": 1.221601128578186,
      "step": 206
    },
    {
      "completion_length": 72.35714721679688,
      "epoch": 0.11842105263157894,
      "grad_norm": 58.693782806396484,
      "kl": 0.6712306141853333,
      "learning_rate": 4.994896073159096e-06,
      "loss": 0.0268,
      "reward": 0.6765860915184021,
      "reward_std": 0.5144326090812683,
      "rewards/reward_function": 0.6765860915184021,
      "step": 207
    },
    {
      "completion_length": 90.5714340209961,
      "epoch": 0.11899313501144165,
      "grad_norm": 1447.0665283203125,
      "kl": 8.516731262207031,
      "learning_rate": 4.994572210710315e-06,
      "loss": 0.3407,
      "reward": 1.0723336935043335,
      "reward_std": 0.7867205142974854,
      "rewards/reward_function": 1.0723336935043335,
      "step": 208
    },
    {
      "completion_length": 89.10714721679688,
      "epoch": 0.11956521739130435,
      "grad_norm": 0.882321298122406,
      "kl": 0.281232625246048,
      "learning_rate": 4.994238397914314e-06,
      "loss": 0.0112,
      "reward": 0.9848078489303589,
      "reward_std": 0.9518094658851624,
      "rewards/reward_function": 0.9848078489303589,
      "step": 209
    },
    {
      "completion_length": 81.71428680419922,
      "epoch": 0.12013729977116705,
      "grad_norm": 279.3052062988281,
      "kl": 12.843462944030762,
      "learning_rate": 4.9938946361026055e-06,
      "loss": 0.5137,
      "reward": 1.2567787170410156,
      "reward_std": 0.9645555019378662,
      "rewards/reward_function": 1.2567787170410156,
      "step": 210
    },
    {
      "completion_length": 90.3214340209961,
      "epoch": 0.12070938215102975,
      "grad_norm": 1.5117827653884888,
      "kl": 0.4550014138221741,
      "learning_rate": 4.993540926646385e-06,
      "loss": 0.0182,
      "reward": 1.3314125537872314,
      "reward_std": 0.5124015212059021,
      "rewards/reward_function": 1.3314125537872314,
      "step": 211
    },
    {
      "completion_length": 91.17857360839844,
      "epoch": 0.12128146453089245,
      "grad_norm": 338987.5625,
      "kl": 4706.6171875,
      "learning_rate": 4.993177270956531e-06,
      "loss": 188.2647,
      "reward": 1.0056304931640625,
      "reward_std": 0.7894412875175476,
      "rewards/reward_function": 1.0056304931640625,
      "step": 212
    },
    {
      "completion_length": 96.03572082519531,
      "epoch": 0.12185354691075514,
      "grad_norm": 37.42105484008789,
      "kl": 2.108281373977661,
      "learning_rate": 4.992803670483591e-06,
      "loss": 0.0843,
      "reward": 0.5953633189201355,
      "reward_std": 0.5422486662864685,
      "rewards/reward_function": 0.5953633189201355,
      "step": 213
    },
    {
      "completion_length": 96.21428680419922,
      "epoch": 0.12242562929061784,
      "grad_norm": 298.7342529296875,
      "kl": 13.79680061340332,
      "learning_rate": 4.992420126717784e-06,
      "loss": 0.5519,
      "reward": 1.4866859912872314,
      "reward_std": 1.0063490867614746,
      "rewards/reward_function": 1.4866859912872314,
      "step": 214
    },
    {
      "completion_length": 100.00000762939453,
      "epoch": 0.12299771167048056,
      "grad_norm": 23.14501953125,
      "kl": 1.1113550662994385,
      "learning_rate": 4.992026641188987e-06,
      "loss": 0.0445,
      "reward": 1.521327257156372,
      "reward_std": 1.0560986995697021,
      "rewards/reward_function": 1.521327257156372,
      "step": 215
    },
    {
      "completion_length": 72.67857360839844,
      "epoch": 0.12356979405034325,
      "grad_norm": 24.533599853515625,
      "kl": 1.7391393184661865,
      "learning_rate": 4.991623215466735e-06,
      "loss": 0.0696,
      "reward": 1.409623384475708,
      "reward_std": 0.9904702305793762,
      "rewards/reward_function": 1.409623384475708,
      "step": 216
    },
    {
      "completion_length": 93.64286041259766,
      "epoch": 0.12414187643020595,
      "grad_norm": 188.65565490722656,
      "kl": 6.780441761016846,
      "learning_rate": 4.991209851160213e-06,
      "loss": 0.2712,
      "reward": 0.8537552952766418,
      "reward_std": 0.898422122001648,
      "rewards/reward_function": 0.8537552952766418,
      "step": 217
    },
    {
      "completion_length": 80.53572082519531,
      "epoch": 0.12471395881006865,
      "grad_norm": 1.5046577453613281,
      "kl": 0.4850654900074005,
      "learning_rate": 4.990786549918248e-06,
      "loss": 0.0194,
      "reward": 1.2043555974960327,
      "reward_std": 0.9640652537345886,
      "rewards/reward_function": 1.2043555974960327,
      "step": 218
    },
    {
      "completion_length": 91.21428680419922,
      "epoch": 0.12528604118993136,
      "grad_norm": 105.4818344116211,
      "kl": 10.869367599487305,
      "learning_rate": 4.9903533134293035e-06,
      "loss": 0.4348,
      "reward": 0.9868647456169128,
      "reward_std": 0.8155920505523682,
      "rewards/reward_function": 0.9868647456169128,
      "step": 219
    },
    {
      "completion_length": 80.78572082519531,
      "epoch": 0.12585812356979406,
      "grad_norm": 1.7031464576721191,
      "kl": 0.4270274043083191,
      "learning_rate": 4.989910143421473e-06,
      "loss": 0.0171,
      "reward": 0.9041681885719299,
      "reward_std": 0.5183762311935425,
      "rewards/reward_function": 0.9041681885719299,
      "step": 220
    },
    {
      "completion_length": 98.89286041259766,
      "epoch": 0.12643020594965676,
      "grad_norm": 26158.5,
      "kl": 824.1007690429688,
      "learning_rate": 4.989457041662471e-06,
      "loss": 32.964,
      "reward": 0.7817007303237915,
      "reward_std": 0.9004895687103271,
      "rewards/reward_function": 0.7817007303237915,
      "step": 221
    },
    {
      "completion_length": 100.14286041259766,
      "epoch": 0.12700228832951946,
      "grad_norm": 408.6023864746094,
      "kl": 14.287392616271973,
      "learning_rate": 4.988994009959631e-06,
      "loss": 0.5715,
      "reward": 0.9572316408157349,
      "reward_std": 0.8652568459510803,
      "rewards/reward_function": 0.9572316408157349,
      "step": 222
    },
    {
      "completion_length": 116.21429443359375,
      "epoch": 0.12757437070938216,
      "grad_norm": 869728.6875,
      "kl": 12126.4697265625,
      "learning_rate": 4.988521050159893e-06,
      "loss": 485.0588,
      "reward": 1.1678364276885986,
      "reward_std": 0.7653435468673706,
      "rewards/reward_function": 1.1678364276885986,
      "step": 223
    },
    {
      "completion_length": 105.5714340209961,
      "epoch": 0.12814645308924486,
      "grad_norm": 4562.818359375,
      "kl": 58.675865173339844,
      "learning_rate": 4.9880381641498e-06,
      "loss": 2.347,
      "reward": 0.9914255738258362,
      "reward_std": 1.00206458568573,
      "rewards/reward_function": 0.9914255738258362,
      "step": 224
    },
    {
      "completion_length": 101.21428680419922,
      "epoch": 0.12871853546910755,
      "grad_norm": 5.8026533126831055,
      "kl": 0.6773678064346313,
      "learning_rate": 4.987545353855484e-06,
      "loss": 0.0271,
      "reward": 1.085168480873108,
      "reward_std": 0.7249619960784912,
      "rewards/reward_function": 1.085168480873108,
      "step": 225
    },
    {
      "completion_length": 92.96428680419922,
      "epoch": 0.12929061784897025,
      "grad_norm": 139.0247039794922,
      "kl": 2.510270833969116,
      "learning_rate": 4.987042621242669e-06,
      "loss": 0.1004,
      "reward": 1.043104648590088,
      "reward_std": 0.7776756882667542,
      "rewards/reward_function": 1.043104648590088,
      "step": 226
    },
    {
      "completion_length": 102.42857360839844,
      "epoch": 0.12986270022883295,
      "grad_norm": 8.008402824401855,
      "kl": 0.4531140625476837,
      "learning_rate": 4.986529968316654e-06,
      "loss": 0.0181,
      "reward": 1.0167948007583618,
      "reward_std": 0.8750170469284058,
      "rewards/reward_function": 1.0167948007583618,
      "step": 227
    },
    {
      "completion_length": 107.53572082519531,
      "epoch": 0.13043478260869565,
      "grad_norm": 2306410.0,
      "kl": 4489.3857421875,
      "learning_rate": 4.986007397122307e-06,
      "loss": 179.5754,
      "reward": 1.3755329847335815,
      "reward_std": 1.0602939128875732,
      "rewards/reward_function": 1.3755329847335815,
      "step": 228
    },
    {
      "completion_length": 108.21428680419922,
      "epoch": 0.13100686498855835,
      "grad_norm": 5579.63623046875,
      "kl": 123.02890014648438,
      "learning_rate": 4.985474909744061e-06,
      "loss": 4.9212,
      "reward": 0.7249850034713745,
      "reward_std": 0.5045235753059387,
      "rewards/reward_function": 0.7249850034713745,
      "step": 229
    },
    {
      "completion_length": 91.28572082519531,
      "epoch": 0.13157894736842105,
      "grad_norm": 11.090805053710938,
      "kl": 0.8526528477668762,
      "learning_rate": 4.9849325083059e-06,
      "loss": 0.0341,
      "reward": 1.1562392711639404,
      "reward_std": 0.7468379139900208,
      "rewards/reward_function": 1.1562392711639404,
      "step": 230
    },
    {
      "completion_length": 94.39286041259766,
      "epoch": 0.13215102974828374,
      "grad_norm": 489.3533935546875,
      "kl": 59.64825439453125,
      "learning_rate": 4.984380194971355e-06,
      "loss": 2.3859,
      "reward": 0.8291910290718079,
      "reward_std": 0.7907097935676575,
      "rewards/reward_function": 0.8291910290718079,
      "step": 231
    },
    {
      "completion_length": 76.85714721679688,
      "epoch": 0.13272311212814644,
      "grad_norm": 1.8216077089309692,
      "kl": 0.2894367277622223,
      "learning_rate": 4.983817971943494e-06,
      "loss": 0.0116,
      "reward": 0.5805181264877319,
      "reward_std": 0.6363489627838135,
      "rewards/reward_function": 0.5805181264877319,
      "step": 232
    },
    {
      "completion_length": 110.39286041259766,
      "epoch": 0.13329519450800914,
      "grad_norm": 2479.962890625,
      "kl": 94.89442443847656,
      "learning_rate": 4.983245841464909e-06,
      "loss": 3.7958,
      "reward": 1.028899908065796,
      "reward_std": 1.0094155073165894,
      "rewards/reward_function": 1.028899908065796,
      "step": 233
    },
    {
      "completion_length": 84.5,
      "epoch": 0.13386727688787187,
      "grad_norm": 356.8840637207031,
      "kl": 5.213452339172363,
      "learning_rate": 4.982663805817716e-06,
      "loss": 0.2085,
      "reward": 1.0306742191314697,
      "reward_std": 0.9967160820960999,
      "rewards/reward_function": 1.0306742191314697,
      "step": 234
    },
    {
      "completion_length": 113.75000762939453,
      "epoch": 0.13443935926773457,
      "grad_norm": 2296.607666015625,
      "kl": 203.90089416503906,
      "learning_rate": 4.982071867323536e-06,
      "loss": 8.156,
      "reward": 0.7031930088996887,
      "reward_std": 0.5385299921035767,
      "rewards/reward_function": 0.7031930088996887,
      "step": 235
    },
    {
      "completion_length": 112.60714721679688,
      "epoch": 0.13501144164759726,
      "grad_norm": 55792.1484375,
      "kl": 2183.993896484375,
      "learning_rate": 4.9814700283434945e-06,
      "loss": 87.3597,
      "reward": 0.7961739301681519,
      "reward_std": 0.9227615594863892,
      "rewards/reward_function": 0.7961739301681519,
      "step": 236
    },
    {
      "completion_length": 73.10714721679688,
      "epoch": 0.13558352402745996,
      "grad_norm": 8.157552719116211,
      "kl": 0.5079297423362732,
      "learning_rate": 4.980858291278206e-06,
      "loss": 0.0203,
      "reward": 1.466861367225647,
      "reward_std": 0.9552510976791382,
      "rewards/reward_function": 1.466861367225647,
      "step": 237
    },
    {
      "completion_length": 95.3214340209961,
      "epoch": 0.13615560640732266,
      "grad_norm": 12.380412101745605,
      "kl": 0.6952613592147827,
      "learning_rate": 4.980236658567766e-06,
      "loss": 0.0278,
      "reward": 1.1904332637786865,
      "reward_std": 1.0007613897323608,
      "rewards/reward_function": 1.1904332637786865,
      "step": 238
    },
    {
      "completion_length": 103.03572082519531,
      "epoch": 0.13672768878718536,
      "grad_norm": 13.851837158203125,
      "kl": 0.7385976910591125,
      "learning_rate": 4.979605132691743e-06,
      "loss": 0.0295,
      "reward": 1.2258042097091675,
      "reward_std": 1.0423922538757324,
      "rewards/reward_function": 1.2258042097091675,
      "step": 239
    },
    {
      "completion_length": 95.3214340209961,
      "epoch": 0.13729977116704806,
      "grad_norm": 1.9024226665496826,
      "kl": 0.3461795449256897,
      "learning_rate": 4.978963716169166e-06,
      "loss": 0.0138,
      "reward": 1.354263424873352,
      "reward_std": 0.9741514921188354,
      "rewards/reward_function": 1.354263424873352,
      "step": 240
    },
    {
      "completion_length": 90.89286041259766,
      "epoch": 0.13787185354691076,
      "grad_norm": 3.90116286277771,
      "kl": 0.35509175062179565,
      "learning_rate": 4.978312411558518e-06,
      "loss": 0.0142,
      "reward": 1.4460673332214355,
      "reward_std": 1.015272617340088,
      "rewards/reward_function": 1.4460673332214355,
      "step": 241
    },
    {
      "completion_length": 73.78572082519531,
      "epoch": 0.13844393592677345,
      "grad_norm": 1524.3804931640625,
      "kl": 42.827449798583984,
      "learning_rate": 4.977651221457721e-06,
      "loss": 1.7131,
      "reward": 1.0643601417541504,
      "reward_std": 0.7378596067428589,
      "rewards/reward_function": 1.0643601417541504,
      "step": 242
    },
    {
      "completion_length": 88.89286041259766,
      "epoch": 0.13901601830663615,
      "grad_norm": 1.2958269119262695,
      "kl": 0.4230957627296448,
      "learning_rate": 4.97698014850413e-06,
      "loss": 0.0169,
      "reward": 1.9117838144302368,
      "reward_std": 0.5956181883811951,
      "rewards/reward_function": 1.9117838144302368,
      "step": 243
    },
    {
      "completion_length": 86.89286041259766,
      "epoch": 0.13958810068649885,
      "grad_norm": 47921.9765625,
      "kl": 52.790714263916016,
      "learning_rate": 4.97629919537452e-06,
      "loss": 2.1116,
      "reward": 1.0124270915985107,
      "reward_std": 0.9017977714538574,
      "rewards/reward_function": 1.0124270915985107,
      "step": 244
    },
    {
      "completion_length": 99.75000762939453,
      "epoch": 0.14016018306636155,
      "grad_norm": 1340.606201171875,
      "kl": 3.926396131515503,
      "learning_rate": 4.975608364785075e-06,
      "loss": 0.1571,
      "reward": 1.6005358695983887,
      "reward_std": 1.0291508436203003,
      "rewards/reward_function": 1.6005358695983887,
      "step": 245
    },
    {
      "completion_length": 93.3214340209961,
      "epoch": 0.14073226544622425,
      "grad_norm": 25186.396484375,
      "kl": 543.6224365234375,
      "learning_rate": 4.974907659491382e-06,
      "loss": 21.7449,
      "reward": 0.9564553499221802,
      "reward_std": 0.7461889386177063,
      "rewards/reward_function": 0.9564553499221802,
      "step": 246
    },
    {
      "completion_length": 97.78572082519531,
      "epoch": 0.14130434782608695,
      "grad_norm": 18127.091796875,
      "kl": 363.3118896484375,
      "learning_rate": 4.974197082288411e-06,
      "loss": 14.5325,
      "reward": 1.2273244857788086,
      "reward_std": 1.0314843654632568,
      "rewards/reward_function": 1.2273244857788086,
      "step": 247
    },
    {
      "completion_length": 96.10714721679688,
      "epoch": 0.14187643020594964,
      "grad_norm": 1204.4371337890625,
      "kl": 72.43836975097656,
      "learning_rate": 4.9734766360105145e-06,
      "loss": 2.8975,
      "reward": 1.242184042930603,
      "reward_std": 1.0600757598876953,
      "rewards/reward_function": 1.242184042930603,
      "step": 248
    },
    {
      "completion_length": 79.75,
      "epoch": 0.14244851258581237,
      "grad_norm": 6.639447212219238,
      "kl": 0.2844906747341156,
      "learning_rate": 4.972746323531404e-06,
      "loss": 0.0114,
      "reward": 1.3965953588485718,
      "reward_std": 1.1259105205535889,
      "rewards/reward_function": 1.3965953588485718,
      "step": 249
    },
    {
      "completion_length": 88.75000762939453,
      "epoch": 0.14302059496567507,
      "grad_norm": 4223.62744140625,
      "kl": 97.76451873779297,
      "learning_rate": 4.972006147764154e-06,
      "loss": 3.9106,
      "reward": 1.3338271379470825,
      "reward_std": 1.008004069328308,
      "rewards/reward_function": 1.3338271379470825,
      "step": 250
    },
    {
      "completion_length": 102.25000762939453,
      "epoch": 0.14359267734553777,
      "grad_norm": 37255.89453125,
      "kl": 277.9356994628906,
      "learning_rate": 4.971256111661173e-06,
      "loss": 11.1174,
      "reward": 1.0953024625778198,
      "reward_std": 0.8216311931610107,
      "rewards/reward_function": 1.0953024625778198,
      "step": 251
    },
    {
      "completion_length": 69.89286041259766,
      "epoch": 0.14416475972540047,
      "grad_norm": 4370797.5,
      "kl": 24565.435546875,
      "learning_rate": 4.970496218214205e-06,
      "loss": 982.6174,
      "reward": 0.8514910340309143,
      "reward_std": 0.9069300293922424,
      "rewards/reward_function": 0.8514910340309143,
      "step": 252
    },
    {
      "completion_length": 85.60714721679688,
      "epoch": 0.14473684210526316,
      "grad_norm": 1.0103343725204468,
      "kl": 0.22769084572792053,
      "learning_rate": 4.9697264704543135e-06,
      "loss": 0.0091,
      "reward": 0.7552441358566284,
      "reward_std": 0.7587482929229736,
      "rewards/reward_function": 0.7552441358566284,
      "step": 253
    },
    {
      "completion_length": 84.5714340209961,
      "epoch": 0.14530892448512586,
      "grad_norm": 134.44012451171875,
      "kl": 3.1210896968841553,
      "learning_rate": 4.968946871451866e-06,
      "loss": 0.1248,
      "reward": 0.8278352618217468,
      "reward_std": 0.8056066632270813,
      "rewards/reward_function": 0.8278352618217468,
      "step": 254
    },
    {
      "completion_length": 91.14286041259766,
      "epoch": 0.14588100686498856,
      "grad_norm": 11.201982498168945,
      "kl": 0.6643458604812622,
      "learning_rate": 4.9681574243165284e-06,
      "loss": 0.0266,
      "reward": 1.6722328662872314,
      "reward_std": 0.9818896055221558,
      "rewards/reward_function": 1.6722328662872314,
      "step": 255
    },
    {
      "completion_length": 92.17857360839844,
      "epoch": 0.14645308924485126,
      "grad_norm": 49.81162643432617,
      "kl": 0.5053438544273376,
      "learning_rate": 4.967358132197245e-06,
      "loss": 0.0202,
      "reward": 1.3441864252090454,
      "reward_std": 0.6979497075080872,
      "rewards/reward_function": 1.3441864252090454,
      "step": 256
    },
    {
      "completion_length": 79.14286041259766,
      "epoch": 0.14702517162471396,
      "grad_norm": 1.6182328462600708,
      "kl": 0.37519535422325134,
      "learning_rate": 4.966548998282232e-06,
      "loss": 0.015,
      "reward": 1.392138123512268,
      "reward_std": 0.737883448600769,
      "rewards/reward_function": 1.392138123512268,
      "step": 257
    },
    {
      "completion_length": 70.8214340209961,
      "epoch": 0.14759725400457666,
      "grad_norm": 2.1169230937957764,
      "kl": 0.29316335916519165,
      "learning_rate": 4.965730025798963e-06,
      "loss": 0.0117,
      "reward": 1.0762255191802979,
      "reward_std": 0.9009786248207092,
      "rewards/reward_function": 1.0762255191802979,
      "step": 258
    },
    {
      "completion_length": 104.75000762939453,
      "epoch": 0.14816933638443935,
      "grad_norm": 505.0931396484375,
      "kl": 31.460376739501953,
      "learning_rate": 4.964901218014152e-06,
      "loss": 1.2584,
      "reward": 0.6613653898239136,
      "reward_std": 0.2979689836502075,
      "rewards/reward_function": 0.6613653898239136,
      "step": 259
    },
    {
      "completion_length": 110.78572082519531,
      "epoch": 0.14874141876430205,
      "grad_norm": 58.86721420288086,
      "kl": 1.1068408489227295,
      "learning_rate": 4.964062578233748e-06,
      "loss": 0.0443,
      "reward": 0.8729681968688965,
      "reward_std": 0.6025736927986145,
      "rewards/reward_function": 0.8729681968688965,
      "step": 260
    },
    {
      "completion_length": 99.21428680419922,
      "epoch": 0.14931350114416475,
      "grad_norm": 1.3944075107574463,
      "kl": 0.33645883202552795,
      "learning_rate": 4.963214109802918e-06,
      "loss": 0.0135,
      "reward": 1.3210065364837646,
      "reward_std": 1.053885817527771,
      "rewards/reward_function": 1.3210065364837646,
      "step": 261
    },
    {
      "completion_length": 127.17857360839844,
      "epoch": 0.14988558352402745,
      "grad_norm": 12.137589454650879,
      "kl": 0.9205886125564575,
      "learning_rate": 4.962355816106031e-06,
      "loss": 0.0368,
      "reward": 0.9787266850471497,
      "reward_std": 0.682168185710907,
      "rewards/reward_function": 0.9787266850471497,
      "step": 262
    },
    {
      "completion_length": 87.85714721679688,
      "epoch": 0.15045766590389015,
      "grad_norm": 4902.63623046875,
      "kl": 16.2197265625,
      "learning_rate": 4.961487700566646e-06,
      "loss": 0.6488,
      "reward": 0.989815890789032,
      "reward_std": 0.5463213324546814,
      "rewards/reward_function": 0.989815890789032,
      "step": 263
    },
    {
      "completion_length": 116.60714721679688,
      "epoch": 0.15102974828375287,
      "grad_norm": 5.428101062774658,
      "kl": 0.534171462059021,
      "learning_rate": 4.960609766647505e-06,
      "loss": 0.0214,
      "reward": 1.2290666103363037,
      "reward_std": 0.8211463093757629,
      "rewards/reward_function": 1.2290666103363037,
      "step": 264
    },
    {
      "completion_length": 104.96428680419922,
      "epoch": 0.15160183066361557,
      "grad_norm": 3.8458914756774902,
      "kl": 0.33722323179244995,
      "learning_rate": 4.959722017850508e-06,
      "loss": 0.0135,
      "reward": 0.7259830236434937,
      "reward_std": 0.2614331841468811,
      "rewards/reward_function": 0.7259830236434937,
      "step": 265
    },
    {
      "completion_length": 135.96429443359375,
      "epoch": 0.15217391304347827,
      "grad_norm": 148.5657958984375,
      "kl": 9.977492332458496,
      "learning_rate": 4.958824457716707e-06,
      "loss": 0.3991,
      "reward": 0.6672676801681519,
      "reward_std": 0.6122839450836182,
      "rewards/reward_function": 0.6672676801681519,
      "step": 266
    },
    {
      "completion_length": 116.00000762939453,
      "epoch": 0.15274599542334097,
      "grad_norm": 1.1991726160049438,
      "kl": 0.2640129327774048,
      "learning_rate": 4.957917089826288e-06,
      "loss": 0.0106,
      "reward": 0.7644982933998108,
      "reward_std": 0.284463495016098,
      "rewards/reward_function": 0.7644982933998108,
      "step": 267
    },
    {
      "completion_length": 95.39286041259766,
      "epoch": 0.15331807780320367,
      "grad_norm": 1.1044611930847168,
      "kl": 0.39538782835006714,
      "learning_rate": 4.95699991779856e-06,
      "loss": 0.0158,
      "reward": 1.0799815654754639,
      "reward_std": 0.801887571811676,
      "rewards/reward_function": 1.0799815654754639,
      "step": 268
    },
    {
      "completion_length": 83.92857360839844,
      "epoch": 0.15389016018306637,
      "grad_norm": 5.771021842956543,
      "kl": 0.5574456453323364,
      "learning_rate": 4.956072945291939e-06,
      "loss": 0.0223,
      "reward": 1.2450921535491943,
      "reward_std": 0.638893723487854,
      "rewards/reward_function": 1.2450921535491943,
      "step": 269
    },
    {
      "completion_length": 107.92857360839844,
      "epoch": 0.15446224256292906,
      "grad_norm": 93.71923828125,
      "kl": 0.4137488007545471,
      "learning_rate": 4.9551361760039295e-06,
      "loss": 0.0165,
      "reward": 1.3322315216064453,
      "reward_std": 0.9691293239593506,
      "rewards/reward_function": 1.3322315216064453,
      "step": 270
    },
    {
      "completion_length": 80.8214340209961,
      "epoch": 0.15503432494279176,
      "grad_norm": 2.184169054031372,
      "kl": 0.2893190085887909,
      "learning_rate": 4.954189613671119e-06,
      "loss": 0.0116,
      "reward": 1.4255130290985107,
      "reward_std": 1.0172398090362549,
      "rewards/reward_function": 1.4255130290985107,
      "step": 271
    },
    {
      "completion_length": 147.25,
      "epoch": 0.15560640732265446,
      "grad_norm": 694.9945678710938,
      "kl": 35.80674362182617,
      "learning_rate": 4.953233262069152e-06,
      "loss": 1.4323,
      "reward": 0.9864605069160461,
      "reward_std": 0.9973043203353882,
      "rewards/reward_function": 0.9864605069160461,
      "step": 272
    },
    {
      "completion_length": 81.89286041259766,
      "epoch": 0.15617848970251716,
      "grad_norm": 41.36872863769531,
      "kl": 0.5821775794029236,
      "learning_rate": 4.952267125012723e-06,
      "loss": 0.0233,
      "reward": 1.3034785985946655,
      "reward_std": 0.7711018323898315,
      "rewards/reward_function": 1.3034785985946655,
      "step": 273
    },
    {
      "completion_length": 103.0714340209961,
      "epoch": 0.15675057208237986,
      "grad_norm": 26.34965705871582,
      "kl": 0.8971932530403137,
      "learning_rate": 4.95129120635556e-06,
      "loss": 0.0359,
      "reward": 1.0492146015167236,
      "reward_std": 0.6724790930747986,
      "rewards/reward_function": 1.0492146015167236,
      "step": 274
    },
    {
      "completion_length": 108.71428680419922,
      "epoch": 0.15732265446224256,
      "grad_norm": 0.8714706301689148,
      "kl": 0.21655048429965973,
      "learning_rate": 4.9503055099904045e-06,
      "loss": 0.0087,
      "reward": 0.8876953721046448,
      "reward_std": 0.5752017498016357,
      "rewards/reward_function": 0.8876953721046448,
      "step": 275
    },
    {
      "completion_length": 97.8214340209961,
      "epoch": 0.15789473684210525,
      "grad_norm": 1.2017228603363037,
      "kl": 0.514641523361206,
      "learning_rate": 4.949310039849003e-06,
      "loss": 0.0206,
      "reward": 1.366515040397644,
      "reward_std": 0.9205434918403625,
      "rewards/reward_function": 1.366515040397644,
      "step": 276
    },
    {
      "completion_length": 109.71429443359375,
      "epoch": 0.15846681922196795,
      "grad_norm": 1.4029099941253662,
      "kl": 0.311221182346344,
      "learning_rate": 4.948304799902085e-06,
      "loss": 0.0124,
      "reward": 1.5679945945739746,
      "reward_std": 1.0709946155548096,
      "rewards/reward_function": 1.5679945945739746,
      "step": 277
    },
    {
      "completion_length": 80.85714721679688,
      "epoch": 0.15903890160183065,
      "grad_norm": 35.61316680908203,
      "kl": 0.31684204936027527,
      "learning_rate": 4.9472897941593515e-06,
      "loss": 0.0127,
      "reward": 1.0941113233566284,
      "reward_std": 0.7813193798065186,
      "rewards/reward_function": 1.0941113233566284,
      "step": 278
    },
    {
      "completion_length": 86.89286041259766,
      "epoch": 0.15961098398169338,
      "grad_norm": 0.6968557238578796,
      "kl": 0.27789306640625,
      "learning_rate": 4.9462650266694546e-06,
      "loss": 0.0111,
      "reward": 1.4548313617706299,
      "reward_std": 0.797633945941925,
      "rewards/reward_function": 1.4548313617706299,
      "step": 279
    },
    {
      "completion_length": 111.00000762939453,
      "epoch": 0.16018306636155608,
      "grad_norm": 0.5751184821128845,
      "kl": 0.25206154584884644,
      "learning_rate": 4.945230501519987e-06,
      "loss": 0.0101,
      "reward": 1.2786744832992554,
      "reward_std": 0.8077378869056702,
      "rewards/reward_function": 1.2786744832992554,
      "step": 280
    },
    {
      "completion_length": 98.00000762939453,
      "epoch": 0.16075514874141877,
      "grad_norm": 9.162391662597656,
      "kl": 0.41662901639938354,
      "learning_rate": 4.944186222837462e-06,
      "loss": 0.0167,
      "reward": 1.086538553237915,
      "reward_std": 0.896883487701416,
      "rewards/reward_function": 1.086538553237915,
      "step": 281
    },
    {
      "completion_length": 108.0714340209961,
      "epoch": 0.16132723112128147,
      "grad_norm": 1.7413784265518188,
      "kl": 0.49267593026161194,
      "learning_rate": 4.943132194787297e-06,
      "loss": 0.0197,
      "reward": 1.3660823106765747,
      "reward_std": 0.4949747622013092,
      "rewards/reward_function": 1.3660823106765747,
      "step": 282
    },
    {
      "completion_length": 93.03572082519531,
      "epoch": 0.16189931350114417,
      "grad_norm": 0.547335147857666,
      "kl": 0.23523350059986115,
      "learning_rate": 4.942068421573797e-06,
      "loss": 0.0094,
      "reward": 1.258849859237671,
      "reward_std": 0.7095308899879456,
      "rewards/reward_function": 1.258849859237671,
      "step": 283
    },
    {
      "completion_length": 94.14286041259766,
      "epoch": 0.16247139588100687,
      "grad_norm": 0.6841917037963867,
      "kl": 0.26628243923187256,
      "learning_rate": 4.940994907440141e-06,
      "loss": 0.0107,
      "reward": 1.0299265384674072,
      "reward_std": 0.9878777265548706,
      "rewards/reward_function": 1.0299265384674072,
      "step": 284
    },
    {
      "completion_length": 117.5714340209961,
      "epoch": 0.16304347826086957,
      "grad_norm": 4.666252136230469,
      "kl": 0.34584110975265503,
      "learning_rate": 4.939911656668361e-06,
      "loss": 0.0138,
      "reward": 1.4992202520370483,
      "reward_std": 0.7888539433479309,
      "rewards/reward_function": 1.4992202520370483,
      "step": 285
    },
    {
      "completion_length": 98.50000762939453,
      "epoch": 0.16361556064073227,
      "grad_norm": 1.1736968755722046,
      "kl": 0.3215252161026001,
      "learning_rate": 4.938818673579327e-06,
      "loss": 0.0129,
      "reward": 1.2624270915985107,
      "reward_std": 0.7479844689369202,
      "rewards/reward_function": 1.2624270915985107,
      "step": 286
    },
    {
      "completion_length": 106.46428680419922,
      "epoch": 0.16418764302059496,
      "grad_norm": 5421.74755859375,
      "kl": 12.437493324279785,
      "learning_rate": 4.937715962532727e-06,
      "loss": 0.4975,
      "reward": 1.1559710502624512,
      "reward_std": 0.7643110752105713,
      "rewards/reward_function": 1.1559710502624512,
      "step": 287
    },
    {
      "completion_length": 111.8214340209961,
      "epoch": 0.16475972540045766,
      "grad_norm": 0.9144102334976196,
      "kl": 0.3106268346309662,
      "learning_rate": 4.9366035279270565e-06,
      "loss": 0.0124,
      "reward": 1.23722243309021,
      "reward_std": 0.7896729707717896,
      "rewards/reward_function": 1.23722243309021,
      "step": 288
    },
    {
      "completion_length": 92.28572082519531,
      "epoch": 0.16533180778032036,
      "grad_norm": 5.091939449310303,
      "kl": 0.30111226439476013,
      "learning_rate": 4.935481374199591e-06,
      "loss": 0.012,
      "reward": 0.8788275718688965,
      "reward_std": 0.756310224533081,
      "rewards/reward_function": 0.8788275718688965,
      "step": 289
    },
    {
      "completion_length": 121.92857360839844,
      "epoch": 0.16590389016018306,
      "grad_norm": 19663.150390625,
      "kl": 1674.4891357421875,
      "learning_rate": 4.934349505826377e-06,
      "loss": 66.9796,
      "reward": 1.5011876821517944,
      "reward_std": 0.7711322903633118,
      "rewards/reward_function": 1.5011876821517944,
      "step": 290
    },
    {
      "completion_length": 134.6428680419922,
      "epoch": 0.16647597254004576,
      "grad_norm": 2610889.5,
      "kl": 19773.62890625,
      "learning_rate": 4.933207927322211e-06,
      "loss": 790.9452,
      "reward": 1.5749986171722412,
      "reward_std": 0.9599400162696838,
      "rewards/reward_function": 1.5749986171722412,
      "step": 291
    },
    {
      "completion_length": 120.85714721679688,
      "epoch": 0.16704805491990846,
      "grad_norm": 3734.353271484375,
      "kl": 281.1929016113281,
      "learning_rate": 4.9320566432406185e-06,
      "loss": 11.2477,
      "reward": 1.3421437740325928,
      "reward_std": 1.1167188882827759,
      "rewards/reward_function": 1.3421437740325928,
      "step": 292
    },
    {
      "completion_length": 118.75000762939453,
      "epoch": 0.16762013729977115,
      "grad_norm": 2.2610697746276855,
      "kl": 0.33379432559013367,
      "learning_rate": 4.930895658173842e-06,
      "loss": 0.0134,
      "reward": 1.4661136865615845,
      "reward_std": 0.8395535945892334,
      "rewards/reward_function": 1.4661136865615845,
      "step": 293
    },
    {
      "completion_length": 108.67857360839844,
      "epoch": 0.16819221967963388,
      "grad_norm": 0.6658067107200623,
      "kl": 0.1928369253873825,
      "learning_rate": 4.929724976752816e-06,
      "loss": 0.0077,
      "reward": 1.1212834119796753,
      "reward_std": 0.8246734142303467,
      "rewards/reward_function": 1.1212834119796753,
      "step": 294
    },
    {
      "completion_length": 146.92857360839844,
      "epoch": 0.16876430205949658,
      "grad_norm": 4.567567825317383,
      "kl": 0.2750953733921051,
      "learning_rate": 4.9285446036471565e-06,
      "loss": 0.011,
      "reward": 1.465591549873352,
      "reward_std": 0.9376941919326782,
      "rewards/reward_function": 1.465591549873352,
      "step": 295
    },
    {
      "completion_length": 84.67857360839844,
      "epoch": 0.16933638443935928,
      "grad_norm": 0.8427210450172424,
      "kl": 0.28738102316856384,
      "learning_rate": 4.927354543565131e-06,
      "loss": 0.0115,
      "reward": 0.9428478479385376,
      "reward_std": 0.7449436783790588,
      "rewards/reward_function": 0.9428478479385376,
      "step": 296
    },
    {
      "completion_length": 94.3214340209961,
      "epoch": 0.16990846681922198,
      "grad_norm": 1.6339555978775024,
      "kl": 0.24425235390663147,
      "learning_rate": 4.926154801253651e-06,
      "loss": 0.0098,
      "reward": 1.1564468145370483,
      "reward_std": 0.8209735751152039,
      "rewards/reward_function": 1.1564468145370483,
      "step": 297
    },
    {
      "completion_length": 116.35714721679688,
      "epoch": 0.17048054919908467,
      "grad_norm": 160536.3125,
      "kl": 3361.142822265625,
      "learning_rate": 4.924945381498249e-06,
      "loss": 134.4457,
      "reward": 1.154583215713501,
      "reward_std": 0.6225337982177734,
      "rewards/reward_function": 1.154583215713501,
      "step": 298
    },
    {
      "completion_length": 94.75000762939453,
      "epoch": 0.17105263157894737,
      "grad_norm": 0.7688279151916504,
      "kl": 0.287626713514328,
      "learning_rate": 4.923726289123055e-06,
      "loss": 0.0115,
      "reward": 1.5753742456436157,
      "reward_std": 1.0283485651016235,
      "rewards/reward_function": 1.5753742456436157,
      "step": 299
    },
    {
      "completion_length": 102.3214340209961,
      "epoch": 0.17162471395881007,
      "grad_norm": 0.8722249865531921,
      "kl": 0.17692206799983978,
      "learning_rate": 4.922497528990785e-06,
      "loss": 0.0071,
      "reward": 0.47204455733299255,
      "reward_std": 0.30633464455604553,
      "rewards/reward_function": 0.47204455733299255,
      "step": 300
    },
    {
      "completion_length": 151.60714721679688,
      "epoch": 0.17219679633867277,
      "grad_norm": 0.7495328783988953,
      "kl": 0.2350076138973236,
      "learning_rate": 4.921259106002717e-06,
      "loss": 0.0094,
      "reward": 1.4523847103118896,
      "reward_std": 0.7516869306564331,
      "rewards/reward_function": 1.4523847103118896,
      "step": 301
    },
    {
      "completion_length": 86.64286041259766,
      "epoch": 0.17276887871853547,
      "grad_norm": 0.6595125198364258,
      "kl": 0.25872719287872314,
      "learning_rate": 4.920011025098669e-06,
      "loss": 0.0103,
      "reward": 1.0650899410247803,
      "reward_std": 0.7399339079856873,
      "rewards/reward_function": 1.0650899410247803,
      "step": 302
    },
    {
      "completion_length": 116.8214340209961,
      "epoch": 0.17334096109839817,
      "grad_norm": 52.04047775268555,
      "kl": 0.22568394243717194,
      "learning_rate": 4.9187532912569876e-06,
      "loss": 0.009,
      "reward": 1.050391435623169,
      "reward_std": 0.7935701608657837,
      "rewards/reward_function": 1.050391435623169,
      "step": 303
    },
    {
      "completion_length": 111.50000762939453,
      "epoch": 0.17391304347826086,
      "grad_norm": 2.8706390857696533,
      "kl": 0.344206303358078,
      "learning_rate": 4.9174859094945185e-06,
      "loss": 0.0138,
      "reward": 1.3977255821228027,
      "reward_std": 0.9582617878913879,
      "rewards/reward_function": 1.3977255821228027,
      "step": 304
    },
    {
      "completion_length": 125.50000762939453,
      "epoch": 0.17448512585812356,
      "grad_norm": 417.60003662109375,
      "kl": 9.76838493347168,
      "learning_rate": 4.916208884866593e-06,
      "loss": 0.3907,
      "reward": 0.94913649559021,
      "reward_std": 0.8735306859016418,
      "rewards/reward_function": 0.94913649559021,
      "step": 305
    },
    {
      "completion_length": 165.3928680419922,
      "epoch": 0.17505720823798626,
      "grad_norm": 100640.578125,
      "kl": 1505.052490234375,
      "learning_rate": 4.914922222467006e-06,
      "loss": 60.2021,
      "reward": 1.0314611196517944,
      "reward_std": 0.3420330882072449,
      "rewards/reward_function": 1.0314611196517944,
      "step": 306
    },
    {
      "completion_length": 139.6428680419922,
      "epoch": 0.17562929061784896,
      "grad_norm": 2438.798828125,
      "kl": 67.08959197998047,
      "learning_rate": 4.913625927427996e-06,
      "loss": 2.6836,
      "reward": 0.890242338180542,
      "reward_std": 0.7164995670318604,
      "rewards/reward_function": 0.890242338180542,
      "step": 307
    },
    {
      "completion_length": 96.85714721679688,
      "epoch": 0.17620137299771166,
      "grad_norm": 0.9877712726593018,
      "kl": 0.2260640561580658,
      "learning_rate": 4.912320004920222e-06,
      "loss": 0.009,
      "reward": 1.0613911151885986,
      "reward_std": 0.8790560364723206,
      "rewards/reward_function": 1.0613911151885986,
      "step": 308
    },
    {
      "completion_length": 113.78572082519531,
      "epoch": 0.17677345537757438,
      "grad_norm": 5.199087619781494,
      "kl": 0.4060578942298889,
      "learning_rate": 4.911004460152748e-06,
      "loss": 0.0162,
      "reward": 1.6868526935577393,
      "reward_std": 1.1094470024108887,
      "rewards/reward_function": 1.6868526935577393,
      "step": 309
    },
    {
      "completion_length": 107.60714721679688,
      "epoch": 0.17734553775743708,
      "grad_norm": 1.6535966396331787,
      "kl": 0.29029783606529236,
      "learning_rate": 4.909679298373015e-06,
      "loss": 0.0116,
      "reward": 1.5695005655288696,
      "reward_std": 1.1027717590332031,
      "rewards/reward_function": 1.5695005655288696,
      "step": 310
    },
    {
      "completion_length": 138.6428680419922,
      "epoch": 0.17791762013729978,
      "grad_norm": 0.5062119960784912,
      "kl": 0.23820067942142487,
      "learning_rate": 4.90834452486683e-06,
      "loss": 0.0095,
      "reward": 1.6787145137786865,
      "reward_std": 0.9350864291191101,
      "rewards/reward_function": 1.6787145137786865,
      "step": 311
    },
    {
      "completion_length": 133.92857360839844,
      "epoch": 0.17848970251716248,
      "grad_norm": 0.45105525851249695,
      "kl": 0.21358622610569,
      "learning_rate": 4.907000144958336e-06,
      "loss": 0.0085,
      "reward": 0.8443188071250916,
      "reward_std": 0.920600414276123,
      "rewards/reward_function": 0.8443188071250916,
      "step": 312
    },
    {
      "completion_length": 107.42857360839844,
      "epoch": 0.17906178489702518,
      "grad_norm": 0.4901830852031708,
      "kl": 0.18663878738880157,
      "learning_rate": 4.905646164009993e-06,
      "loss": 0.0075,
      "reward": 1.1845883131027222,
      "reward_std": 0.6738457083702087,
      "rewards/reward_function": 1.1845883131027222,
      "step": 313
    },
    {
      "completion_length": 117.71429443359375,
      "epoch": 0.17963386727688788,
      "grad_norm": 1.761809229850769,
      "kl": 0.33902135491371155,
      "learning_rate": 4.904282587422559e-06,
      "loss": 0.0136,
      "reward": 1.3200085163116455,
      "reward_std": 0.5556520819664001,
      "rewards/reward_function": 1.3200085163116455,
      "step": 314
    },
    {
      "completion_length": 146.75,
      "epoch": 0.18020594965675057,
      "grad_norm": 9.769920349121094,
      "kl": 0.884746789932251,
      "learning_rate": 4.902909420635067e-06,
      "loss": 0.0354,
      "reward": 1.0625251531600952,
      "reward_std": 0.5510778427124023,
      "rewards/reward_function": 1.0625251531600952,
      "step": 315
    },
    {
      "completion_length": 98.25000762939453,
      "epoch": 0.18077803203661327,
      "grad_norm": 1.3788654804229736,
      "kl": 0.2406122237443924,
      "learning_rate": 4.901526669124803e-06,
      "loss": 0.0096,
      "reward": 0.9043004512786865,
      "reward_std": 0.7923168540000916,
      "rewards/reward_function": 0.9043004512786865,
      "step": 316
    },
    {
      "completion_length": 123.42857360839844,
      "epoch": 0.18135011441647597,
      "grad_norm": 2.9544782638549805,
      "kl": 0.272506982088089,
      "learning_rate": 4.900134338407286e-06,
      "loss": 0.0109,
      "reward": 1.727768063545227,
      "reward_std": 0.9375365972518921,
      "rewards/reward_function": 1.727768063545227,
      "step": 317
    },
    {
      "completion_length": 187.60714721679688,
      "epoch": 0.18192219679633867,
      "grad_norm": 63.76605224609375,
      "kl": 10.119579315185547,
      "learning_rate": 4.8987324340362445e-06,
      "loss": 0.4048,
      "reward": 1.4830586910247803,
      "reward_std": 0.5629326105117798,
      "rewards/reward_function": 1.4830586910247803,
      "step": 318
    },
    {
      "completion_length": 131.85714721679688,
      "epoch": 0.18249427917620137,
      "grad_norm": 7.3109941482543945,
      "kl": 0.28836148977279663,
      "learning_rate": 4.8973209616035896e-06,
      "loss": 0.0115,
      "reward": 1.5004115104675293,
      "reward_std": 0.766803503036499,
      "rewards/reward_function": 1.5004115104675293,
      "step": 319
    },
    {
      "completion_length": 125.00000762939453,
      "epoch": 0.18306636155606407,
      "grad_norm": 2.1318037509918213,
      "kl": 0.25991183519363403,
      "learning_rate": 4.895899926739404e-06,
      "loss": 0.0104,
      "reward": 1.1880295276641846,
      "reward_std": 0.7319925427436829,
      "rewards/reward_function": 1.1880295276641846,
      "step": 320
    },
    {
      "completion_length": 112.96429443359375,
      "epoch": 0.18363844393592677,
      "grad_norm": 0.6634427309036255,
      "kl": 0.25338053703308105,
      "learning_rate": 4.89446933511191e-06,
      "loss": 0.0101,
      "reward": 1.345016360282898,
      "reward_std": 0.6345396041870117,
      "rewards/reward_function": 1.345016360282898,
      "step": 321
    },
    {
      "completion_length": 127.3214340209961,
      "epoch": 0.18421052631578946,
      "grad_norm": 8226.607421875,
      "kl": 124.27462005615234,
      "learning_rate": 4.893029192427449e-06,
      "loss": 4.971,
      "reward": 1.5477229356765747,
      "reward_std": 0.8879736661911011,
      "rewards/reward_function": 1.5477229356765747,
      "step": 322
    },
    {
      "completion_length": 100.46428680419922,
      "epoch": 0.18478260869565216,
      "grad_norm": 40.619911193847656,
      "kl": 0.43373650312423706,
      "learning_rate": 4.891579504430461e-06,
      "loss": 0.0173,
      "reward": 1.5749413967132568,
      "reward_std": 1.0439107418060303,
      "rewards/reward_function": 1.5749413967132568,
      "step": 323
    },
    {
      "completion_length": 98.5714340209961,
      "epoch": 0.1853546910755149,
      "grad_norm": 0.6162534952163696,
      "kl": 0.22894205152988434,
      "learning_rate": 4.890120276903462e-06,
      "loss": 0.0092,
      "reward": 1.4754750728607178,
      "reward_std": 0.6490025520324707,
      "rewards/reward_function": 1.4754750728607178,
      "step": 324
    },
    {
      "completion_length": 120.5714340209961,
      "epoch": 0.1859267734553776,
      "grad_norm": 0.6174753904342651,
      "kl": 0.2344750314950943,
      "learning_rate": 4.888651515667015e-06,
      "loss": 0.0094,
      "reward": 1.3960121870040894,
      "reward_std": 0.9000580310821533,
      "rewards/reward_function": 1.3960121870040894,
      "step": 325
    },
    {
      "completion_length": 100.21428680419922,
      "epoch": 0.18649885583524028,
      "grad_norm": 12.51389217376709,
      "kl": 0.2455047070980072,
      "learning_rate": 4.887173226579713e-06,
      "loss": 0.0098,
      "reward": 0.8971139788627625,
      "reward_std": 0.8978429436683655,
      "rewards/reward_function": 0.8971139788627625,
      "step": 326
    },
    {
      "completion_length": 107.96428680419922,
      "epoch": 0.18707093821510298,
      "grad_norm": 4.140501976013184,
      "kl": 0.28204619884490967,
      "learning_rate": 4.885685415538156e-06,
      "loss": 0.0113,
      "reward": 1.037853479385376,
      "reward_std": 0.6879757642745972,
      "rewards/reward_function": 1.037853479385376,
      "step": 327
    },
    {
      "completion_length": 145.7857208251953,
      "epoch": 0.18764302059496568,
      "grad_norm": 51337.6953125,
      "kl": 3969.455322265625,
      "learning_rate": 4.8841880884769225e-06,
      "loss": 158.7782,
      "reward": 1.242938756942749,
      "reward_std": 0.8768441677093506,
      "rewards/reward_function": 1.242938756942749,
      "step": 328
    },
    {
      "completion_length": 105.5714340209961,
      "epoch": 0.18821510297482838,
      "grad_norm": 7.473899841308594,
      "kl": 0.5748904347419739,
      "learning_rate": 4.882681251368549e-06,
      "loss": 0.023,
      "reward": 1.6142184734344482,
      "reward_std": 1.0905870199203491,
      "rewards/reward_function": 1.6142184734344482,
      "step": 329
    },
    {
      "completion_length": 114.03572082519531,
      "epoch": 0.18878718535469108,
      "grad_norm": 32414.318359375,
      "kl": 425.3197326660156,
      "learning_rate": 4.8811649102235065e-06,
      "loss": 17.0128,
      "reward": 1.7128335237503052,
      "reward_std": 1.0610063076019287,
      "rewards/reward_function": 1.7128335237503052,
      "step": 330
    },
    {
      "completion_length": 116.5714340209961,
      "epoch": 0.18935926773455378,
      "grad_norm": 0.47712743282318115,
      "kl": 0.2337779998779297,
      "learning_rate": 4.879639071090174e-06,
      "loss": 0.0094,
      "reward": 1.708465814590454,
      "reward_std": 0.9931800365447998,
      "rewards/reward_function": 1.708465814590454,
      "step": 331
    },
    {
      "completion_length": 129.92857360839844,
      "epoch": 0.18993135011441648,
      "grad_norm": 960.82177734375,
      "kl": 70.38544464111328,
      "learning_rate": 4.878103740054819e-06,
      "loss": 2.8154,
      "reward": 0.5496902465820312,
      "reward_std": 0.4075881540775299,
      "rewards/reward_function": 0.5496902465820312,
      "step": 332
    },
    {
      "completion_length": 120.50000762939453,
      "epoch": 0.19050343249427917,
      "grad_norm": 1.6227664947509766,
      "kl": 0.22627940773963928,
      "learning_rate": 4.876558923241566e-06,
      "loss": 0.0091,
      "reward": 0.9918405413627625,
      "reward_std": 0.9269010424613953,
      "rewards/reward_function": 0.9918405413627625,
      "step": 333
    },
    {
      "completion_length": 98.67857360839844,
      "epoch": 0.19107551487414187,
      "grad_norm": 0.9276979565620422,
      "kl": 0.23648416996002197,
      "learning_rate": 4.875004626812381e-06,
      "loss": 0.0095,
      "reward": 1.0409120321273804,
      "reward_std": 0.4246746003627777,
      "rewards/reward_function": 1.0409120321273804,
      "step": 334
    },
    {
      "completion_length": 124.92857360839844,
      "epoch": 0.19164759725400457,
      "grad_norm": 0.7844218015670776,
      "kl": 0.24337854981422424,
      "learning_rate": 4.8734408569670395e-06,
      "loss": 0.0097,
      "reward": 1.3806556463241577,
      "reward_std": 0.8535459637641907,
      "rewards/reward_function": 1.3806556463241577,
      "step": 335
    },
    {
      "completion_length": 108.89286041259766,
      "epoch": 0.19221967963386727,
      "grad_norm": 1066.484130859375,
      "kl": 46.85482406616211,
      "learning_rate": 4.8718676199431045e-06,
      "loss": 1.8742,
      "reward": 1.2046953439712524,
      "reward_std": 0.8984759449958801,
      "rewards/reward_function": 1.2046953439712524,
      "step": 336
    },
    {
      "completion_length": 122.28572082519531,
      "epoch": 0.19279176201372997,
      "grad_norm": 2.764943838119507,
      "kl": 0.3607800602912903,
      "learning_rate": 4.870284922015902e-06,
      "loss": 0.0144,
      "reward": 1.5064175128936768,
      "reward_std": 0.8105327486991882,
      "rewards/reward_function": 1.5064175128936768,
      "step": 337
    },
    {
      "completion_length": 118.10714721679688,
      "epoch": 0.19336384439359267,
      "grad_norm": 3.3240928649902344,
      "kl": 0.251851350069046,
      "learning_rate": 4.8686927694984975e-06,
      "loss": 0.0101,
      "reward": 0.6989397406578064,
      "reward_std": 0.5251139402389526,
      "rewards/reward_function": 0.6989397406578064,
      "step": 338
    },
    {
      "completion_length": 117.78572082519531,
      "epoch": 0.19393592677345536,
      "grad_norm": 0.5495253205299377,
      "kl": 0.18054814636707306,
      "learning_rate": 4.867091168741666e-06,
      "loss": 0.0072,
      "reward": 1.2902965545654297,
      "reward_std": 0.5386127233505249,
      "rewards/reward_function": 1.2902965545654297,
      "step": 339
    },
    {
      "completion_length": 145.17857360839844,
      "epoch": 0.1945080091533181,
      "grad_norm": 765.4857788085938,
      "kl": 7.866365909576416,
      "learning_rate": 4.865480126133872e-06,
      "loss": 0.3147,
      "reward": 1.6899360418319702,
      "reward_std": 0.577396810054779,
      "rewards/reward_function": 1.6899360418319702,
      "step": 340
    },
    {
      "completion_length": 76.28572082519531,
      "epoch": 0.1950800915331808,
      "grad_norm": 0.8201074004173279,
      "kl": 0.22674302756786346,
      "learning_rate": 4.86385964810124e-06,
      "loss": 0.0091,
      "reward": 1.403764009475708,
      "reward_std": 0.7030969858169556,
      "rewards/reward_function": 1.403764009475708,
      "step": 341
    },
    {
      "completion_length": 140.5,
      "epoch": 0.1956521739130435,
      "grad_norm": 8.81520938873291,
      "kl": 0.42793723940849304,
      "learning_rate": 4.862229741107532e-06,
      "loss": 0.0171,
      "reward": 1.4787517786026,
      "reward_std": 1.1429989337921143,
      "rewards/reward_function": 1.4787517786026,
      "step": 342
    },
    {
      "completion_length": 111.53572082519531,
      "epoch": 0.19622425629290619,
      "grad_norm": 12.976065635681152,
      "kl": 0.2648286819458008,
      "learning_rate": 4.860590411654118e-06,
      "loss": 0.0106,
      "reward": 1.2827239036560059,
      "reward_std": 0.6830337047576904,
      "rewards/reward_function": 1.2827239036560059,
      "step": 343
    },
    {
      "completion_length": 128.75,
      "epoch": 0.19679633867276888,
      "grad_norm": 40.46461486816406,
      "kl": 0.8148645758628845,
      "learning_rate": 4.858941666279956e-06,
      "loss": 0.0326,
      "reward": 1.2540315389633179,
      "reward_std": 0.41325482726097107,
      "rewards/reward_function": 1.2540315389633179,
      "step": 344
    },
    {
      "completion_length": 86.42857360839844,
      "epoch": 0.19736842105263158,
      "grad_norm": 151.8027801513672,
      "kl": 7.630585193634033,
      "learning_rate": 4.857283511561557e-06,
      "loss": 0.3052,
      "reward": 0.6253970265388489,
      "reward_std": 0.42981648445129395,
      "rewards/reward_function": 0.6253970265388489,
      "step": 345
    },
    {
      "completion_length": 74.53572082519531,
      "epoch": 0.19794050343249428,
      "grad_norm": 2.0168356895446777,
      "kl": 0.3381984531879425,
      "learning_rate": 4.85561595411297e-06,
      "loss": 0.0135,
      "reward": 1.6335351467132568,
      "reward_std": 0.9874892234802246,
      "rewards/reward_function": 1.6335351467132568,
      "step": 346
    },
    {
      "completion_length": 117.53572082519531,
      "epoch": 0.19851258581235698,
      "grad_norm": 0.85274738073349,
      "kl": 0.29459846019744873,
      "learning_rate": 4.853939000585745e-06,
      "loss": 0.0118,
      "reward": 2.3510830402374268,
      "reward_std": 0.31042733788490295,
      "rewards/reward_function": 2.3510830402374268,
      "step": 347
    },
    {
      "completion_length": 101.00000762939453,
      "epoch": 0.19908466819221968,
      "grad_norm": 0.5284850597381592,
      "kl": 0.18485602736473083,
      "learning_rate": 4.852252657668911e-06,
      "loss": 0.0074,
      "reward": 1.6465917825698853,
      "reward_std": 0.9876177310943604,
      "rewards/reward_function": 1.6465917825698853,
      "step": 348
    },
    {
      "completion_length": 102.92857360839844,
      "epoch": 0.19965675057208238,
      "grad_norm": 0.976259171962738,
      "kl": 0.30213305354118347,
      "learning_rate": 4.850556932088954e-06,
      "loss": 0.0121,
      "reward": 1.9915868043899536,
      "reward_std": 0.6073095202445984,
      "rewards/reward_function": 1.9915868043899536,
      "step": 349
    },
    {
      "completion_length": 106.89286041259766,
      "epoch": 0.20022883295194507,
      "grad_norm": 0.7294902801513672,
      "kl": 0.25455406308174133,
      "learning_rate": 4.848851830609782e-06,
      "loss": 0.0102,
      "reward": 1.253212332725525,
      "reward_std": 0.7685462832450867,
      "rewards/reward_function": 1.253212332725525,
      "step": 350
    },
    {
      "completion_length": 95.85714721679688,
      "epoch": 0.20080091533180777,
      "grad_norm": 6.878642559051514,
      "kl": 0.26581838726997375,
      "learning_rate": 4.8471373600327e-06,
      "loss": 0.0106,
      "reward": 0.8053100109100342,
      "reward_std": 0.586913526058197,
      "rewards/reward_function": 0.8053100109100342,
      "step": 351
    },
    {
      "completion_length": 91.21428680419922,
      "epoch": 0.20137299771167047,
      "grad_norm": 2.1675381660461426,
      "kl": 0.278901070356369,
      "learning_rate": 4.845413527196389e-06,
      "loss": 0.0112,
      "reward": 1.1525691747665405,
      "reward_std": 1.0717148780822754,
      "rewards/reward_function": 1.1525691747665405,
      "step": 352
    },
    {
      "completion_length": 98.17857360839844,
      "epoch": 0.20194508009153317,
      "grad_norm": 5.118911266326904,
      "kl": 0.4234643578529358,
      "learning_rate": 4.84368033897687e-06,
      "loss": 0.0169,
      "reward": 1.4803471565246582,
      "reward_std": 0.3179745078086853,
      "rewards/reward_function": 1.4803471565246582,
      "step": 353
    },
    {
      "completion_length": 100.50000762939453,
      "epoch": 0.20251716247139587,
      "grad_norm": 0.35706350207328796,
      "kl": 0.19625622034072876,
      "learning_rate": 4.841937802287484e-06,
      "loss": 0.0079,
      "reward": 1.4014209508895874,
      "reward_std": 0.8151746988296509,
      "rewards/reward_function": 1.4014209508895874,
      "step": 354
    },
    {
      "completion_length": 94.8214340209961,
      "epoch": 0.2030892448512586,
      "grad_norm": 0.7310407161712646,
      "kl": 0.2751310467720032,
      "learning_rate": 4.840185924078858e-06,
      "loss": 0.011,
      "reward": 2.0471396446228027,
      "reward_std": 0.7316878437995911,
      "rewards/reward_function": 2.0471396446228027,
      "step": 355
    },
    {
      "completion_length": 119.10714721679688,
      "epoch": 0.2036613272311213,
      "grad_norm": 380.91162109375,
      "kl": 4.661480903625488,
      "learning_rate": 4.838424711338883e-06,
      "loss": 0.1865,
      "reward": 1.8280537128448486,
      "reward_std": 0.8202334642410278,
      "rewards/reward_function": 1.8280537128448486,
      "step": 356
    },
    {
      "completion_length": 92.25000762939453,
      "epoch": 0.204233409610984,
      "grad_norm": 3.7079362869262695,
      "kl": 0.3596739172935486,
      "learning_rate": 4.836654171092683e-06,
      "loss": 0.0144,
      "reward": 1.7137707471847534,
      "reward_std": 0.7782400250434875,
      "rewards/reward_function": 1.7137707471847534,
      "step": 357
    },
    {
      "completion_length": 98.50000762939453,
      "epoch": 0.2048054919908467,
      "grad_norm": 5.8861188888549805,
      "kl": 0.3887327313423157,
      "learning_rate": 4.8348743104025866e-06,
      "loss": 0.0155,
      "reward": 1.5379430055618286,
      "reward_std": 1.0187122821807861,
      "rewards/reward_function": 1.5379430055618286,
      "step": 358
    },
    {
      "completion_length": 88.60714721679688,
      "epoch": 0.2053775743707094,
      "grad_norm": 2.63096022605896,
      "kl": 0.3892691433429718,
      "learning_rate": 4.833085136368102e-06,
      "loss": 0.0156,
      "reward": 1.9189202785491943,
      "reward_std": 0.7943240404129028,
      "rewards/reward_function": 1.9189202785491943,
      "step": 359
    },
    {
      "completion_length": 93.67857360839844,
      "epoch": 0.20594965675057209,
      "grad_norm": 5.11529541015625,
      "kl": 0.4677744507789612,
      "learning_rate": 4.831286656125882e-06,
      "loss": 0.0187,
      "reward": 1.580589771270752,
      "reward_std": 1.0067200660705566,
      "rewards/reward_function": 1.580589771270752,
      "step": 360
    },
    {
      "completion_length": 87.14286041259766,
      "epoch": 0.20652173913043478,
      "grad_norm": 3.0058648586273193,
      "kl": 0.26843488216400146,
      "learning_rate": 4.829478876849705e-06,
      "loss": 0.0107,
      "reward": 1.156711459159851,
      "reward_std": 0.6100212335586548,
      "rewards/reward_function": 1.156711459159851,
      "step": 361
    },
    {
      "completion_length": 95.17857360839844,
      "epoch": 0.20709382151029748,
      "grad_norm": 4.568263530731201,
      "kl": 0.546653151512146,
      "learning_rate": 4.827661805750438e-06,
      "loss": 0.0219,
      "reward": 1.4922984838485718,
      "reward_std": 0.7943882346153259,
      "rewards/reward_function": 1.4922984838485718,
      "step": 362
    },
    {
      "completion_length": 101.21428680419922,
      "epoch": 0.20766590389016018,
      "grad_norm": 1.4180635213851929,
      "kl": 0.25450730323791504,
      "learning_rate": 4.825835450076014e-06,
      "loss": 0.0102,
      "reward": 0.5614340305328369,
      "reward_std": 0.3702699840068817,
      "rewards/reward_function": 0.5614340305328369,
      "step": 363
    },
    {
      "completion_length": 94.89286041259766,
      "epoch": 0.20823798627002288,
      "grad_norm": 18.62581443786621,
      "kl": 1.0665457248687744,
      "learning_rate": 4.823999817111396e-06,
      "loss": 0.0427,
      "reward": 1.6270498037338257,
      "reward_std": 0.5840937495231628,
      "rewards/reward_function": 1.6270498037338257,
      "step": 364
    },
    {
      "completion_length": 91.25000762939453,
      "epoch": 0.20881006864988558,
      "grad_norm": 0.5403398871421814,
      "kl": 0.2707400321960449,
      "learning_rate": 4.822154914178559e-06,
      "loss": 0.0108,
      "reward": 1.6503621339797974,
      "reward_std": 1.22114896774292,
      "rewards/reward_function": 1.6503621339797974,
      "step": 365
    },
    {
      "completion_length": 86.50000762939453,
      "epoch": 0.20938215102974828,
      "grad_norm": 3.8087520599365234,
      "kl": 0.4966406226158142,
      "learning_rate": 4.820300748636446e-06,
      "loss": 0.0199,
      "reward": 1.1725869178771973,
      "reward_std": 0.6223137378692627,
      "rewards/reward_function": 1.1725869178771973,
      "step": 366
    },
    {
      "completion_length": 91.14286041259766,
      "epoch": 0.20995423340961097,
      "grad_norm": 1.0787636041641235,
      "kl": 0.27567362785339355,
      "learning_rate": 4.818437327880954e-06,
      "loss": 0.011,
      "reward": 2.042174816131592,
      "reward_std": 1.005226731300354,
      "rewards/reward_function": 2.042174816131592,
      "step": 367
    },
    {
      "completion_length": 77.28572082519531,
      "epoch": 0.21052631578947367,
      "grad_norm": 0.625913143157959,
      "kl": 0.29806315898895264,
      "learning_rate": 4.816564659344891e-06,
      "loss": 0.0119,
      "reward": 2.1757922172546387,
      "reward_std": 0.6712394952774048,
      "rewards/reward_function": 2.1757922172546387,
      "step": 368
    },
    {
      "completion_length": 90.3214340209961,
      "epoch": 0.21109839816933637,
      "grad_norm": 0.5248486995697021,
      "kl": 0.2677677571773529,
      "learning_rate": 4.814682750497958e-06,
      "loss": 0.0107,
      "reward": 1.199655294418335,
      "reward_std": 0.8115776777267456,
      "rewards/reward_function": 1.199655294418335,
      "step": 369
    },
    {
      "completion_length": 86.78572082519531,
      "epoch": 0.2116704805491991,
      "grad_norm": 0.7393428683280945,
      "kl": 0.25870874524116516,
      "learning_rate": 4.812791608846709e-06,
      "loss": 0.0103,
      "reward": 1.7369791269302368,
      "reward_std": 1.1061533689498901,
      "rewards/reward_function": 1.7369791269302368,
      "step": 370
    },
    {
      "completion_length": 105.21428680419922,
      "epoch": 0.2122425629290618,
      "grad_norm": 1.3035908937454224,
      "kl": 0.29480499029159546,
      "learning_rate": 4.810891241934531e-06,
      "loss": 0.0118,
      "reward": 1.9215710163116455,
      "reward_std": 1.086393117904663,
      "rewards/reward_function": 1.9215710163116455,
      "step": 371
    },
    {
      "completion_length": 122.00000762939453,
      "epoch": 0.2128146453089245,
      "grad_norm": 0.4350615441799164,
      "kl": 0.2472151517868042,
      "learning_rate": 4.808981657341602e-06,
      "loss": 0.0099,
      "reward": 1.3613818883895874,
      "reward_std": 0.9854079484939575,
      "rewards/reward_function": 1.3613818883895874,
      "step": 372
    },
    {
      "completion_length": 83.78572082519531,
      "epoch": 0.2133867276887872,
      "grad_norm": 4.873141765594482,
      "kl": 0.44205692410469055,
      "learning_rate": 4.807062862684874e-06,
      "loss": 0.0177,
      "reward": 1.5324876308441162,
      "reward_std": 0.6392356157302856,
      "rewards/reward_function": 1.5324876308441162,
      "step": 373
    },
    {
      "completion_length": 86.3214340209961,
      "epoch": 0.2139588100686499,
      "grad_norm": 2.2842752933502197,
      "kl": 0.4985831677913666,
      "learning_rate": 4.805134865618031e-06,
      "loss": 0.0199,
      "reward": 1.5585867166519165,
      "reward_std": 0.6217808127403259,
      "rewards/reward_function": 1.5585867166519165,
      "step": 374
    },
    {
      "completion_length": 75.89286041259766,
      "epoch": 0.2145308924485126,
      "grad_norm": 0.4924401640892029,
      "kl": 0.2462616115808487,
      "learning_rate": 4.803197673831468e-06,
      "loss": 0.0099,
      "reward": 1.758846402168274,
      "reward_std": 0.6477039456367493,
      "rewards/reward_function": 1.758846402168274,
      "step": 375
    },
    {
      "completion_length": 96.17857360839844,
      "epoch": 0.2151029748283753,
      "grad_norm": 0.6760632395744324,
      "kl": 0.2665119171142578,
      "learning_rate": 4.801251295052253e-06,
      "loss": 0.0107,
      "reward": 1.5909026861190796,
      "reward_std": 0.699187159538269,
      "rewards/reward_function": 1.5909026861190796,
      "step": 376
    },
    {
      "completion_length": 98.75000762939453,
      "epoch": 0.21567505720823799,
      "grad_norm": 31.17388343811035,
      "kl": 0.83536696434021,
      "learning_rate": 4.799295737044098e-06,
      "loss": 0.0334,
      "reward": 2.108895778656006,
      "reward_std": 0.7955281734466553,
      "rewards/reward_function": 2.108895778656006,
      "step": 377
    },
    {
      "completion_length": 90.75000762939453,
      "epoch": 0.21624713958810068,
      "grad_norm": 1.2806651592254639,
      "kl": 0.27912774682044983,
      "learning_rate": 4.797331007607335e-06,
      "loss": 0.0112,
      "reward": 0.8935797810554504,
      "reward_std": 0.6387594938278198,
      "rewards/reward_function": 0.8935797810554504,
      "step": 378
    },
    {
      "completion_length": 78.10714721679688,
      "epoch": 0.21681922196796338,
      "grad_norm": 0.7805063724517822,
      "kl": 0.34524765610694885,
      "learning_rate": 4.79535711457887e-06,
      "loss": 0.0138,
      "reward": 1.6859869956970215,
      "reward_std": 0.6588106155395508,
      "rewards/reward_function": 1.6859869956970215,
      "step": 379
    },
    {
      "completion_length": 89.46428680419922,
      "epoch": 0.21739130434782608,
      "grad_norm": 2.030686140060425,
      "kl": 0.336612343788147,
      "learning_rate": 4.793374065832168e-06,
      "loss": 0.0135,
      "reward": 2.1843020915985107,
      "reward_std": 0.8292446732521057,
      "rewards/reward_function": 2.1843020915985107,
      "step": 380
    },
    {
      "completion_length": 86.75000762939453,
      "epoch": 0.21796338672768878,
      "grad_norm": 0.5339022278785706,
      "kl": 0.2844798266887665,
      "learning_rate": 4.791381869277213e-06,
      "loss": 0.0114,
      "reward": 1.4455559253692627,
      "reward_std": 0.33155199885368347,
      "rewards/reward_function": 1.4455559253692627,
      "step": 381
    },
    {
      "completion_length": 68.21428680419922,
      "epoch": 0.21853546910755148,
      "grad_norm": 0.42692357301712036,
      "kl": 0.28438687324523926,
      "learning_rate": 4.789380532860475e-06,
      "loss": 0.0114,
      "reward": 1.6835150718688965,
      "reward_std": 0.9036586284637451,
      "rewards/reward_function": 1.6835150718688965,
      "step": 382
    },
    {
      "completion_length": 90.71428680419922,
      "epoch": 0.21910755148741418,
      "grad_norm": 2.7690043449401855,
      "kl": 0.3457414507865906,
      "learning_rate": 4.787370064564884e-06,
      "loss": 0.0138,
      "reward": 1.7067487239837646,
      "reward_std": 0.7616198658943176,
      "rewards/reward_function": 1.7067487239837646,
      "step": 383
    },
    {
      "completion_length": 88.5714340209961,
      "epoch": 0.21967963386727687,
      "grad_norm": 0.6283729672431946,
      "kl": 0.2758108377456665,
      "learning_rate": 4.785350472409792e-06,
      "loss": 0.011,
      "reward": 1.2121752500534058,
      "reward_std": 0.5873774290084839,
      "rewards/reward_function": 1.2121752500534058,
      "step": 384
    },
    {
      "completion_length": 86.8214340209961,
      "epoch": 0.2202517162471396,
      "grad_norm": 10.63276195526123,
      "kl": 0.5512924194335938,
      "learning_rate": 4.783321764450948e-06,
      "loss": 0.0221,
      "reward": 1.5367945432662964,
      "reward_std": 0.8204678893089294,
      "rewards/reward_function": 1.5367945432662964,
      "step": 385
    },
    {
      "completion_length": 98.0714340209961,
      "epoch": 0.2208237986270023,
      "grad_norm": 91.5890884399414,
      "kl": 2.229886054992676,
      "learning_rate": 4.78128394878046e-06,
      "loss": 0.0892,
      "reward": 1.8513944149017334,
      "reward_std": 1.0696613788604736,
      "rewards/reward_function": 1.8513944149017334,
      "step": 386
    },
    {
      "completion_length": 82.0,
      "epoch": 0.221395881006865,
      "grad_norm": 18.003339767456055,
      "kl": 0.5304425954818726,
      "learning_rate": 4.7792370335267654e-06,
      "loss": 0.0212,
      "reward": 0.8652416467666626,
      "reward_std": 0.38886862993240356,
      "rewards/reward_function": 0.8652416467666626,
      "step": 387
    },
    {
      "completion_length": 108.0714340209961,
      "epoch": 0.2219679633867277,
      "grad_norm": 96.17278289794922,
      "kl": 0.8879557251930237,
      "learning_rate": 4.777181026854597e-06,
      "loss": 0.0355,
      "reward": 1.1058050394058228,
      "reward_std": 0.15085965394973755,
      "rewards/reward_function": 1.1058050394058228,
      "step": 388
    },
    {
      "completion_length": 100.5714340209961,
      "epoch": 0.2225400457665904,
      "grad_norm": 752.977294921875,
      "kl": 11.658669471740723,
      "learning_rate": 4.775115936964952e-06,
      "loss": 0.4663,
      "reward": 1.1423563957214355,
      "reward_std": 0.4262990355491638,
      "rewards/reward_function": 1.1423563957214355,
      "step": 389
    },
    {
      "completion_length": 90.25000762939453,
      "epoch": 0.2231121281464531,
      "grad_norm": 29.69986343383789,
      "kl": 0.4372935891151428,
      "learning_rate": 4.77304177209506e-06,
      "loss": 0.0175,
      "reward": 1.9974281787872314,
      "reward_std": 0.8076931238174438,
      "rewards/reward_function": 1.9974281787872314,
      "step": 390
    },
    {
      "completion_length": 74.67857360839844,
      "epoch": 0.2236842105263158,
      "grad_norm": 0.7693162560462952,
      "kl": 0.2657899260520935,
      "learning_rate": 4.770958540518348e-06,
      "loss": 0.0106,
      "reward": 1.901063323020935,
      "reward_std": 0.8529061675071716,
      "rewards/reward_function": 1.901063323020935,
      "step": 391
    },
    {
      "completion_length": 76.67857360839844,
      "epoch": 0.2242562929061785,
      "grad_norm": 1.1768600940704346,
      "kl": 0.28536632657051086,
      "learning_rate": 4.768866250544408e-06,
      "loss": 0.0114,
      "reward": 1.4158333539962769,
      "reward_std": 0.5593510866165161,
      "rewards/reward_function": 1.4158333539962769,
      "step": 392
    },
    {
      "completion_length": 89.3214340209961,
      "epoch": 0.2248283752860412,
      "grad_norm": 88.44566345214844,
      "kl": 2.2289884090423584,
      "learning_rate": 4.7667649105189625e-06,
      "loss": 0.0892,
      "reward": 2.2954087257385254,
      "reward_std": 0.5324976444244385,
      "rewards/reward_function": 2.2954087257385254,
      "step": 393
    },
    {
      "completion_length": 83.10714721679688,
      "epoch": 0.22540045766590389,
      "grad_norm": 0.5563766360282898,
      "kl": 0.2566133439540863,
      "learning_rate": 4.764654528823837e-06,
      "loss": 0.0103,
      "reward": 1.4182478189468384,
      "reward_std": 0.8663737773895264,
      "rewards/reward_function": 1.4182478189468384,
      "step": 394
    },
    {
      "completion_length": 100.39286041259766,
      "epoch": 0.22597254004576658,
      "grad_norm": 1.032266616821289,
      "kl": 0.2782370150089264,
      "learning_rate": 4.7625351138769175e-06,
      "loss": 0.0111,
      "reward": 2.209968090057373,
      "reward_std": 0.6227432489395142,
      "rewards/reward_function": 2.209968090057373,
      "step": 395
    },
    {
      "completion_length": 76.10714721679688,
      "epoch": 0.22654462242562928,
      "grad_norm": 0.7157876491546631,
      "kl": 0.2543988823890686,
      "learning_rate": 4.760406674132126e-06,
      "loss": 0.0102,
      "reward": 1.270636796951294,
      "reward_std": 0.34292271733283997,
      "rewards/reward_function": 1.270636796951294,
      "step": 396
    },
    {
      "completion_length": 85.0,
      "epoch": 0.22711670480549198,
      "grad_norm": 2.0377113819122314,
      "kl": 0.31901073455810547,
      "learning_rate": 4.75826921807938e-06,
      "loss": 0.0128,
      "reward": 1.4636204242706299,
      "reward_std": 0.8106599450111389,
      "rewards/reward_function": 1.4636204242706299,
      "step": 397
    },
    {
      "completion_length": 93.5714340209961,
      "epoch": 0.22768878718535468,
      "grad_norm": 0.6330264210700989,
      "kl": 0.27173715829849243,
      "learning_rate": 4.756122754244564e-06,
      "loss": 0.0109,
      "reward": 1.8590388298034668,
      "reward_std": 0.5645667910575867,
      "rewards/reward_function": 1.8590388298034668,
      "step": 398
    },
    {
      "completion_length": 82.03572082519531,
      "epoch": 0.22826086956521738,
      "grad_norm": 1.8609567880630493,
      "kl": 0.3475543260574341,
      "learning_rate": 4.753967291189489e-06,
      "loss": 0.0139,
      "reward": 1.4840424060821533,
      "reward_std": 0.779403030872345,
      "rewards/reward_function": 1.4840424060821533,
      "step": 399
    },
    {
      "completion_length": 80.39286041259766,
      "epoch": 0.2288329519450801,
      "grad_norm": 1172.561767578125,
      "kl": 48.733680725097656,
      "learning_rate": 4.751802837511863e-06,
      "loss": 1.9493,
      "reward": 1.434777855873108,
      "reward_std": 0.5135812163352966,
      "rewards/reward_function": 1.434777855873108,
      "step": 400
    },
    {
      "completion_length": 103.10714721679688,
      "epoch": 0.2294050343249428,
      "grad_norm": 339.6544189453125,
      "kl": 14.223200798034668,
      "learning_rate": 4.74962940184526e-06,
      "loss": 0.5689,
      "reward": 1.9467899799346924,
      "reward_std": 0.8953436613082886,
      "rewards/reward_function": 1.9467899799346924,
      "step": 401
    },
    {
      "completion_length": 97.25000762939453,
      "epoch": 0.2299771167048055,
      "grad_norm": 18.386478424072266,
      "kl": 0.30007362365722656,
      "learning_rate": 4.747446992859074e-06,
      "loss": 0.012,
      "reward": 1.9401723146438599,
      "reward_std": 0.5966084599494934,
      "rewards/reward_function": 1.9401723146438599,
      "step": 402
    },
    {
      "completion_length": 87.14286041259766,
      "epoch": 0.2305491990846682,
      "grad_norm": 6.20054292678833,
      "kl": 0.42262837290763855,
      "learning_rate": 4.745255619258499e-06,
      "loss": 0.0169,
      "reward": 2.0430831909179688,
      "reward_std": 0.4998953342437744,
      "rewards/reward_function": 2.0430831909179688,
      "step": 403
    },
    {
      "completion_length": 90.71428680419922,
      "epoch": 0.2311212814645309,
      "grad_norm": 4.387509822845459,
      "kl": 0.703170120716095,
      "learning_rate": 4.743055289784484e-06,
      "loss": 0.0281,
      "reward": 1.4662281274795532,
      "reward_std": 0.6927710175514221,
      "rewards/reward_function": 1.4662281274795532,
      "step": 404
    },
    {
      "completion_length": 85.0714340209961,
      "epoch": 0.2316933638443936,
      "grad_norm": 31.442174911499023,
      "kl": 0.6550614833831787,
      "learning_rate": 4.740846013213699e-06,
      "loss": 0.0262,
      "reward": 1.331348180770874,
      "reward_std": 0.7701842784881592,
      "rewards/reward_function": 1.331348180770874,
      "step": 405
    },
    {
      "completion_length": 76.75,
      "epoch": 0.2322654462242563,
      "grad_norm": 5.179102420806885,
      "kl": 0.27645981311798096,
      "learning_rate": 4.738627798358506e-06,
      "loss": 0.0111,
      "reward": 1.4602221250534058,
      "reward_std": 0.6575344204902649,
      "rewards/reward_function": 1.4602221250534058,
      "step": 406
    },
    {
      "completion_length": 81.60714721679688,
      "epoch": 0.232837528604119,
      "grad_norm": 0.9643022418022156,
      "kl": 0.36583027243614197,
      "learning_rate": 4.7364006540669174e-06,
      "loss": 0.0146,
      "reward": 1.8711153268814087,
      "reward_std": 0.8894098401069641,
      "rewards/reward_function": 1.8711153268814087,
      "step": 407
    },
    {
      "completion_length": 77.85714721679688,
      "epoch": 0.2334096109839817,
      "grad_norm": 3.2671875953674316,
      "kl": 0.2599635124206543,
      "learning_rate": 4.734164589222564e-06,
      "loss": 0.0104,
      "reward": 2.069110631942749,
      "reward_std": 0.6894646286964417,
      "rewards/reward_function": 2.069110631942749,
      "step": 408
    },
    {
      "completion_length": 120.8214340209961,
      "epoch": 0.2339816933638444,
      "grad_norm": 0.4681394398212433,
      "kl": 0.2989475429058075,
      "learning_rate": 4.73191961274466e-06,
      "loss": 0.012,
      "reward": 1.788457989692688,
      "reward_std": 0.8215076923370361,
      "rewards/reward_function": 1.788457989692688,
      "step": 409
    },
    {
      "completion_length": 76.75,
      "epoch": 0.2345537757437071,
      "grad_norm": 2.460574150085449,
      "kl": 0.627575159072876,
      "learning_rate": 4.729665733587964e-06,
      "loss": 0.0251,
      "reward": 1.554816484451294,
      "reward_std": 0.48234719038009644,
      "rewards/reward_function": 1.554816484451294,
      "step": 410
    },
    {
      "completion_length": 85.28572082519531,
      "epoch": 0.2351258581235698,
      "grad_norm": 0.4454432725906372,
      "kl": 0.24153156578540802,
      "learning_rate": 4.727402960742748e-06,
      "loss": 0.0097,
      "reward": 1.9054596424102783,
      "reward_std": 0.37957942485809326,
      "rewards/reward_function": 1.9054596424102783,
      "step": 411
    },
    {
      "completion_length": 78.92857360839844,
      "epoch": 0.23569794050343248,
      "grad_norm": 1.2332837581634521,
      "kl": 0.25924667716026306,
      "learning_rate": 4.725131303234758e-06,
      "loss": 0.0104,
      "reward": 1.6480799913406372,
      "reward_std": 0.49614834785461426,
      "rewards/reward_function": 1.6480799913406372,
      "step": 412
    },
    {
      "completion_length": 94.00000762939453,
      "epoch": 0.23627002288329518,
      "grad_norm": 0.6715325117111206,
      "kl": 0.24642853438854218,
      "learning_rate": 4.72285077012518e-06,
      "loss": 0.0099,
      "reward": 1.5412482023239136,
      "reward_std": 0.29151731729507446,
      "rewards/reward_function": 1.5412482023239136,
      "step": 413
    },
    {
      "completion_length": 95.8214340209961,
      "epoch": 0.23684210526315788,
      "grad_norm": 1.2342755794525146,
      "kl": 0.4432714283466339,
      "learning_rate": 4.7205613705106e-06,
      "loss": 0.0177,
      "reward": 1.5326344966888428,
      "reward_std": 1.1544361114501953,
      "rewards/reward_function": 1.5326344966888428,
      "step": 414
    },
    {
      "completion_length": 91.50000762939453,
      "epoch": 0.2374141876430206,
      "grad_norm": 3.310192584991455,
      "kl": 0.376022607088089,
      "learning_rate": 4.7182631135229765e-06,
      "loss": 0.015,
      "reward": 1.3115806579589844,
      "reward_std": 0.5413358807563782,
      "rewards/reward_function": 1.3115806579589844,
      "step": 415
    },
    {
      "completion_length": 121.50000762939453,
      "epoch": 0.2379862700228833,
      "grad_norm": 6.0497331619262695,
      "kl": 0.44697874784469604,
      "learning_rate": 4.7159560083295914e-06,
      "loss": 0.0179,
      "reward": 1.980383038520813,
      "reward_std": 0.7374697923660278,
      "rewards/reward_function": 1.980383038520813,
      "step": 416
    },
    {
      "completion_length": 93.89286041259766,
      "epoch": 0.238558352402746,
      "grad_norm": 0.4856164753437042,
      "kl": 0.18916834890842438,
      "learning_rate": 4.7136400641330245e-06,
      "loss": 0.0076,
      "reward": 1.8930397033691406,
      "reward_std": 0.702826201915741,
      "rewards/reward_function": 1.8930397033691406,
      "step": 417
    },
    {
      "completion_length": 92.42857360839844,
      "epoch": 0.2391304347826087,
      "grad_norm": 0.4841381013393402,
      "kl": 0.23405098915100098,
      "learning_rate": 4.711315290171114e-06,
      "loss": 0.0094,
      "reward": 2.1857151985168457,
      "reward_std": 0.3084292411804199,
      "rewards/reward_function": 2.1857151985168457,
      "step": 418
    },
    {
      "completion_length": 110.25000762939453,
      "epoch": 0.2397025171624714,
      "grad_norm": 0.43921491503715515,
      "kl": 0.23089519143104553,
      "learning_rate": 4.708981695716913e-06,
      "loss": 0.0092,
      "reward": 1.8337126970291138,
      "reward_std": 0.6248344779014587,
      "rewards/reward_function": 1.8337126970291138,
      "step": 419
    },
    {
      "completion_length": 107.14286041259766,
      "epoch": 0.2402745995423341,
      "grad_norm": 0.44122299551963806,
      "kl": 0.25949031114578247,
      "learning_rate": 4.706639290078662e-06,
      "loss": 0.0104,
      "reward": 1.981044888496399,
      "reward_std": 1.0096957683563232,
      "rewards/reward_function": 1.981044888496399,
      "step": 420
    },
    {
      "completion_length": 106.03572082519531,
      "epoch": 0.2408466819221968,
      "grad_norm": 0.47791242599487305,
      "kl": 0.21771159768104553,
      "learning_rate": 4.704288082599747e-06,
      "loss": 0.0087,
      "reward": 1.1879042387008667,
      "reward_std": 0.6692604422569275,
      "rewards/reward_function": 1.1879042387008667,
      "step": 421
    },
    {
      "completion_length": 120.17857360839844,
      "epoch": 0.2414187643020595,
      "grad_norm": 0.40663835406303406,
      "kl": 0.23152506351470947,
      "learning_rate": 4.701928082658661e-06,
      "loss": 0.0093,
      "reward": 1.7743425369262695,
      "reward_std": 0.8183586001396179,
      "rewards/reward_function": 1.7743425369262695,
      "step": 422
    },
    {
      "completion_length": 99.96428680419922,
      "epoch": 0.2419908466819222,
      "grad_norm": 0.45650094747543335,
      "kl": 0.31917786598205566,
      "learning_rate": 4.69955929966897e-06,
      "loss": 0.0128,
      "reward": 1.3225053548812866,
      "reward_std": 0.7764025926589966,
      "rewards/reward_function": 1.3225053548812866,
      "step": 423
    },
    {
      "completion_length": 105.85714721679688,
      "epoch": 0.2425629290617849,
      "grad_norm": 67.00717163085938,
      "kl": 3.664088249206543,
      "learning_rate": 4.697181743079275e-06,
      "loss": 0.1466,
      "reward": 1.810014009475708,
      "reward_std": 0.7500709891319275,
      "rewards/reward_function": 1.810014009475708,
      "step": 424
    },
    {
      "completion_length": 121.17857360839844,
      "epoch": 0.2431350114416476,
      "grad_norm": 1.6223152875900269,
      "kl": 0.2968105673789978,
      "learning_rate": 4.694795422373167e-06,
      "loss": 0.0119,
      "reward": 1.5528417825698853,
      "reward_std": 0.8591582775115967,
      "rewards/reward_function": 1.5528417825698853,
      "step": 425
    },
    {
      "completion_length": 111.75000762939453,
      "epoch": 0.2437070938215103,
      "grad_norm": 5.66955041885376,
      "kl": 0.5461820960044861,
      "learning_rate": 4.692400347069203e-06,
      "loss": 0.0218,
      "reward": 1.7042269706726074,
      "reward_std": 0.7835066914558411,
      "rewards/reward_function": 1.7042269706726074,
      "step": 426
    },
    {
      "completion_length": 126.46429443359375,
      "epoch": 0.244279176201373,
      "grad_norm": 3.357473373413086,
      "kl": 0.3209938406944275,
      "learning_rate": 4.689996526720858e-06,
      "loss": 0.0128,
      "reward": 1.3936620950698853,
      "reward_std": 0.5698928833007812,
      "rewards/reward_function": 1.3936620950698853,
      "step": 427
    },
    {
      "completion_length": 124.42857360839844,
      "epoch": 0.2448512585812357,
      "grad_norm": 0.36040380597114563,
      "kl": 0.2681467533111572,
      "learning_rate": 4.687583970916487e-06,
      "loss": 0.0107,
      "reward": 1.2467663288116455,
      "reward_std": 0.3478240370750427,
      "rewards/reward_function": 1.2467663288116455,
      "step": 428
    },
    {
      "completion_length": 113.5714340209961,
      "epoch": 0.24542334096109839,
      "grad_norm": 2.021986246109009,
      "kl": 0.29533758759498596,
      "learning_rate": 4.685162689279289e-06,
      "loss": 0.0118,
      "reward": 1.5271004438400269,
      "reward_std": 0.9200970530509949,
      "rewards/reward_function": 1.5271004438400269,
      "step": 429
    },
    {
      "completion_length": 113.21429443359375,
      "epoch": 0.2459954233409611,
      "grad_norm": 25.71526527404785,
      "kl": 0.7713311910629272,
      "learning_rate": 4.68273269146727e-06,
      "loss": 0.0309,
      "reward": 1.9503672122955322,
      "reward_std": 0.7304391264915466,
      "rewards/reward_function": 1.9503672122955322,
      "step": 430
    },
    {
      "completion_length": 103.25000762939453,
      "epoch": 0.2465675057208238,
      "grad_norm": 3.641395092010498,
      "kl": 0.4205818772315979,
      "learning_rate": 4.680293987173206e-06,
      "loss": 0.0168,
      "reward": 1.7949577569961548,
      "reward_std": 0.737669050693512,
      "rewards/reward_function": 1.7949577569961548,
      "step": 431
    },
    {
      "completion_length": 114.78572082519531,
      "epoch": 0.2471395881006865,
      "grad_norm": 0.36016321182250977,
      "kl": 0.2446749359369278,
      "learning_rate": 4.677846586124595e-06,
      "loss": 0.0098,
      "reward": 1.9222829341888428,
      "reward_std": 0.55290687084198,
      "rewards/reward_function": 1.9222829341888428,
      "step": 432
    },
    {
      "completion_length": 104.85714721679688,
      "epoch": 0.2477116704805492,
      "grad_norm": 12.957935333251953,
      "kl": 0.7442026734352112,
      "learning_rate": 4.675390498083629e-06,
      "loss": 0.0298,
      "reward": 2.0385475158691406,
      "reward_std": 0.3837388753890991,
      "rewards/reward_function": 2.0385475158691406,
      "step": 433
    },
    {
      "completion_length": 104.0714340209961,
      "epoch": 0.2482837528604119,
      "grad_norm": 0.4100499749183655,
      "kl": 0.2027246356010437,
      "learning_rate": 4.672925732847149e-06,
      "loss": 0.0081,
      "reward": 0.7448167204856873,
      "reward_std": 0.4852374494075775,
      "rewards/reward_function": 0.7448167204856873,
      "step": 434
    },
    {
      "completion_length": 126.75000762939453,
      "epoch": 0.2488558352402746,
      "grad_norm": 0.7695804238319397,
      "kl": 0.29976460337638855,
      "learning_rate": 4.67045230024661e-06,
      "loss": 0.012,
      "reward": 1.007572889328003,
      "reward_std": 0.7547060251235962,
      "rewards/reward_function": 1.007572889328003,
      "step": 435
    },
    {
      "completion_length": 105.60714721679688,
      "epoch": 0.2494279176201373,
      "grad_norm": 0.9089599251747131,
      "kl": 0.19642899930477142,
      "learning_rate": 4.667970210148036e-06,
      "loss": 0.0079,
      "reward": 1.4760797023773193,
      "reward_std": 0.551485002040863,
      "rewards/reward_function": 1.4760797023773193,
      "step": 436
    },
    {
      "completion_length": 132.57144165039062,
      "epoch": 0.25,
      "grad_norm": 1.815375804901123,
      "kl": 0.46152934432029724,
      "learning_rate": 4.665479472451987e-06,
      "loss": 0.0185,
      "reward": 1.8902781009674072,
      "reward_std": 0.6516996622085571,
      "rewards/reward_function": 1.8902781009674072,
      "step": 437
    },
    {
      "completion_length": 103.60714721679688,
      "epoch": 0.2505720823798627,
      "grad_norm": 0.4215809106826782,
      "kl": 0.24391865730285645,
      "learning_rate": 4.662980097093513e-06,
      "loss": 0.0098,
      "reward": 1.390682339668274,
      "reward_std": 0.4854310154914856,
      "rewards/reward_function": 1.390682339668274,
      "step": 438
    },
    {
      "completion_length": 104.75000762939453,
      "epoch": 0.2511441647597254,
      "grad_norm": 0.7631384134292603,
      "kl": 0.23280882835388184,
      "learning_rate": 4.660472094042121e-06,
      "loss": 0.0093,
      "reward": 1.6446030139923096,
      "reward_std": 0.7807363867759705,
      "rewards/reward_function": 1.6446030139923096,
      "step": 439
    },
    {
      "completion_length": 95.85714721679688,
      "epoch": 0.2517162471395881,
      "grad_norm": 0.47005242109298706,
      "kl": 0.22131504118442535,
      "learning_rate": 4.657955473301732e-06,
      "loss": 0.0089,
      "reward": 1.7947502136230469,
      "reward_std": 0.6939342021942139,
      "rewards/reward_function": 1.7947502136230469,
      "step": 440
    },
    {
      "completion_length": 109.00000762939453,
      "epoch": 0.2522883295194508,
      "grad_norm": 38.489349365234375,
      "kl": 2.4574570655822754,
      "learning_rate": 4.6554302449106396e-06,
      "loss": 0.0983,
      "reward": 1.7769073247909546,
      "reward_std": 0.7534574270248413,
      "rewards/reward_function": 1.7769073247909546,
      "step": 441
    },
    {
      "completion_length": 116.0714340209961,
      "epoch": 0.2528604118993135,
      "grad_norm": 0.4587455987930298,
      "kl": 0.22067458927631378,
      "learning_rate": 4.6528964189414715e-06,
      "loss": 0.0088,
      "reward": 2.329820394515991,
      "reward_std": 0.34202539920806885,
      "rewards/reward_function": 2.329820394515991,
      "step": 442
    },
    {
      "completion_length": 126.21429443359375,
      "epoch": 0.2534324942791762,
      "grad_norm": 0.7250040769577026,
      "kl": 0.27126628160476685,
      "learning_rate": 4.650354005501152e-06,
      "loss": 0.0109,
      "reward": 1.5522907972335815,
      "reward_std": 0.8349272012710571,
      "rewards/reward_function": 1.5522907972335815,
      "step": 443
    },
    {
      "completion_length": 122.78572082519531,
      "epoch": 0.2540045766590389,
      "grad_norm": 0.46303093433380127,
      "kl": 0.24022483825683594,
      "learning_rate": 4.647803014730856e-06,
      "loss": 0.0096,
      "reward": 1.8384201526641846,
      "reward_std": 0.6831220984458923,
      "rewards/reward_function": 1.8384201526641846,
      "step": 444
    },
    {
      "completion_length": 120.71429443359375,
      "epoch": 0.2545766590389016,
      "grad_norm": 0.4608452320098877,
      "kl": 0.26061952114105225,
      "learning_rate": 4.645243456805975e-06,
      "loss": 0.0104,
      "reward": 1.976240634918213,
      "reward_std": 1.005554437637329,
      "rewards/reward_function": 1.976240634918213,
      "step": 445
    },
    {
      "completion_length": 150.10714721679688,
      "epoch": 0.2551487414187643,
      "grad_norm": 0.8733582496643066,
      "kl": 0.5841924548149109,
      "learning_rate": 4.642675341936068e-06,
      "loss": 0.0234,
      "reward": 1.873429775238037,
      "reward_std": 0.7351866960525513,
      "rewards/reward_function": 1.873429775238037,
      "step": 446
    },
    {
      "completion_length": 103.92857360839844,
      "epoch": 0.255720823798627,
      "grad_norm": 6.882795810699463,
      "kl": 0.4667273163795471,
      "learning_rate": 4.640098680364832e-06,
      "loss": 0.0187,
      "reward": 1.6422063112258911,
      "reward_std": 0.8543702363967896,
      "rewards/reward_function": 1.6422063112258911,
      "step": 447
    },
    {
      "completion_length": 132.21429443359375,
      "epoch": 0.2562929061784897,
      "grad_norm": 10.528353691101074,
      "kl": 1.1862497329711914,
      "learning_rate": 4.6375134823700505e-06,
      "loss": 0.0475,
      "reward": 2.023416042327881,
      "reward_std": 0.712340235710144,
      "rewards/reward_function": 2.023416042327881,
      "step": 448
    },
    {
      "completion_length": 114.3214340209961,
      "epoch": 0.2568649885583524,
      "grad_norm": 0.9862861037254333,
      "kl": 0.39927423000335693,
      "learning_rate": 4.6349197582635596e-06,
      "loss": 0.016,
      "reward": 1.701601266860962,
      "reward_std": 0.47055506706237793,
      "rewards/reward_function": 1.701601266860962,
      "step": 449
    },
    {
      "completion_length": 112.53572082519531,
      "epoch": 0.2574370709382151,
      "grad_norm": 0.5830795764923096,
      "kl": 0.2673856317996979,
      "learning_rate": 4.632317518391203e-06,
      "loss": 0.0107,
      "reward": 1.6589293479919434,
      "reward_std": 1.1614196300506592,
      "rewards/reward_function": 1.6589293479919434,
      "step": 450
    },
    {
      "completion_length": 122.14286041259766,
      "epoch": 0.2580091533180778,
      "grad_norm": 0.5523030757904053,
      "kl": 0.28074777126312256,
      "learning_rate": 4.6297067731327925e-06,
      "loss": 0.0112,
      "reward": 1.9729747772216797,
      "reward_std": 0.7929600477218628,
      "rewards/reward_function": 1.9729747772216797,
      "step": 451
    },
    {
      "completion_length": 129.7857208251953,
      "epoch": 0.2585812356979405,
      "grad_norm": 1.3325494527816772,
      "kl": 0.4271882176399231,
      "learning_rate": 4.6270875329020674e-06,
      "loss": 0.0171,
      "reward": 1.5509028434753418,
      "reward_std": 1.1202255487442017,
      "rewards/reward_function": 1.5509028434753418,
      "step": 452
    },
    {
      "completion_length": 106.89286041259766,
      "epoch": 0.25915331807780323,
      "grad_norm": 1.1153916120529175,
      "kl": 0.24275143444538116,
      "learning_rate": 4.624459808146649e-06,
      "loss": 0.0097,
      "reward": 2.090061902999878,
      "reward_std": 0.5511574149131775,
      "rewards/reward_function": 2.090061902999878,
      "step": 453
    },
    {
      "completion_length": 152.6428680419922,
      "epoch": 0.2597254004576659,
      "grad_norm": 1.1669001579284668,
      "kl": 0.3943156898021698,
      "learning_rate": 4.621823609348004e-06,
      "loss": 0.0158,
      "reward": 1.9722771644592285,
      "reward_std": 0.7968889474868774,
      "rewards/reward_function": 1.9722771644592285,
      "step": 454
    },
    {
      "completion_length": 119.5714340209961,
      "epoch": 0.2602974828375286,
      "grad_norm": 6.193995475769043,
      "kl": 0.789469838142395,
      "learning_rate": 4.6191789470214e-06,
      "loss": 0.0316,
      "reward": 2.0734710693359375,
      "reward_std": 0.9088209867477417,
      "rewards/reward_function": 2.0734710693359375,
      "step": 455
    },
    {
      "completion_length": 154.96429443359375,
      "epoch": 0.2608695652173913,
      "grad_norm": 3511.654541015625,
      "kl": 23.923274993896484,
      "learning_rate": 4.616525831715863e-06,
      "loss": 0.9569,
      "reward": 1.8466905355453491,
      "reward_std": 1.1657880544662476,
      "rewards/reward_function": 1.8466905355453491,
      "step": 456
    },
    {
      "completion_length": 158.85714721679688,
      "epoch": 0.261441647597254,
      "grad_norm": 0.3162722885608673,
      "kl": 0.23984628915786743,
      "learning_rate": 4.613864274014137e-06,
      "loss": 0.0096,
      "reward": 2.313079595565796,
      "reward_std": 0.6497653722763062,
      "rewards/reward_function": 2.313079595565796,
      "step": 457
    },
    {
      "completion_length": 116.21429443359375,
      "epoch": 0.2620137299771167,
      "grad_norm": 159.79776000976562,
      "kl": 1.6998772621154785,
      "learning_rate": 4.611194284532641e-06,
      "loss": 0.068,
      "reward": 1.5771520137786865,
      "reward_std": 0.7561064958572388,
      "rewards/reward_function": 1.5771520137786865,
      "step": 458
    },
    {
      "completion_length": 121.10714721679688,
      "epoch": 0.2625858123569794,
      "grad_norm": 0.5057719945907593,
      "kl": 0.3444649875164032,
      "learning_rate": 4.608515873921426e-06,
      "loss": 0.0138,
      "reward": 1.9719767570495605,
      "reward_std": 0.47847500443458557,
      "rewards/reward_function": 1.9719767570495605,
      "step": 459
    },
    {
      "completion_length": 126.92857360839844,
      "epoch": 0.2631578947368421,
      "grad_norm": 5.04009485244751,
      "kl": 0.6261754035949707,
      "learning_rate": 4.605829052864134e-06,
      "loss": 0.025,
      "reward": 2.001967430114746,
      "reward_std": 0.838469386100769,
      "rewards/reward_function": 2.001967430114746,
      "step": 460
    },
    {
      "completion_length": 143.46429443359375,
      "epoch": 0.2637299771167048,
      "grad_norm": 0.9744802117347717,
      "kl": 0.33729034662246704,
      "learning_rate": 4.603133832077953e-06,
      "loss": 0.0135,
      "reward": 2.044074058532715,
      "reward_std": 1.0031421184539795,
      "rewards/reward_function": 2.044074058532715,
      "step": 461
    },
    {
      "completion_length": 113.17857360839844,
      "epoch": 0.2643020594965675,
      "grad_norm": 0.5741127729415894,
      "kl": 0.298587441444397,
      "learning_rate": 4.600430222313579e-06,
      "loss": 0.0119,
      "reward": 2.228283166885376,
      "reward_std": 0.8866510987281799,
      "rewards/reward_function": 2.228283166885376,
      "step": 462
    },
    {
      "completion_length": 136.46429443359375,
      "epoch": 0.2648741418764302,
      "grad_norm": 1.1860285997390747,
      "kl": 0.453858882188797,
      "learning_rate": 4.597718234355168e-06,
      "loss": 0.0182,
      "reward": 2.24509596824646,
      "reward_std": 0.9181378483772278,
      "rewards/reward_function": 2.24509596824646,
      "step": 463
    },
    {
      "completion_length": 148.42857360839844,
      "epoch": 0.2654462242562929,
      "grad_norm": 0.4714701771736145,
      "kl": 0.23086418211460114,
      "learning_rate": 4.594997879020292e-06,
      "loss": 0.0092,
      "reward": 1.4516441822052002,
      "reward_std": 0.4784846901893616,
      "rewards/reward_function": 1.4516441822052002,
      "step": 464
    },
    {
      "completion_length": 109.67857360839844,
      "epoch": 0.2660183066361556,
      "grad_norm": 0.4545504152774811,
      "kl": 0.2220166027545929,
      "learning_rate": 4.592269167159905e-06,
      "loss": 0.0089,
      "reward": 1.811591386795044,
      "reward_std": 0.5857572555541992,
      "rewards/reward_function": 1.811591386795044,
      "step": 465
    },
    {
      "completion_length": 105.53572082519531,
      "epoch": 0.2665903890160183,
      "grad_norm": 0.5675729513168335,
      "kl": 0.2744710147380829,
      "learning_rate": 4.589532109658289e-06,
      "loss": 0.011,
      "reward": 1.9185268878936768,
      "reward_std": 0.518285870552063,
      "rewards/reward_function": 1.9185268878936768,
      "step": 466
    },
    {
      "completion_length": 138.57144165039062,
      "epoch": 0.267162471395881,
      "grad_norm": 0.6765762567520142,
      "kl": 0.27847856283187866,
      "learning_rate": 4.586786717433016e-06,
      "loss": 0.0111,
      "reward": 2.0175282955169678,
      "reward_std": 0.8812178373336792,
      "rewards/reward_function": 2.0175282955169678,
      "step": 467
    },
    {
      "completion_length": 149.8928680419922,
      "epoch": 0.26773455377574373,
      "grad_norm": 0.6200852990150452,
      "kl": 0.3078249394893646,
      "learning_rate": 4.584033001434904e-06,
      "loss": 0.0123,
      "reward": 2.2025060653686523,
      "reward_std": 0.6469034552574158,
      "rewards/reward_function": 2.2025060653686523,
      "step": 468
    },
    {
      "completion_length": 148.32144165039062,
      "epoch": 0.2683066361556064,
      "grad_norm": 0.9397121071815491,
      "kl": 0.29061123728752136,
      "learning_rate": 4.581270972647974e-06,
      "loss": 0.0116,
      "reward": 0.9079384803771973,
      "reward_std": 0.5094661116600037,
      "rewards/reward_function": 0.9079384803771973,
      "step": 469
    },
    {
      "completion_length": 139.21429443359375,
      "epoch": 0.26887871853546913,
      "grad_norm": 0.47812917828559875,
      "kl": 0.3399879038333893,
      "learning_rate": 4.578500642089402e-06,
      "loss": 0.0136,
      "reward": 2.1049036979675293,
      "reward_std": 0.6791430711746216,
      "rewards/reward_function": 2.1049036979675293,
      "step": 470
    },
    {
      "completion_length": 119.8214340209961,
      "epoch": 0.2694508009153318,
      "grad_norm": 0.4316757023334503,
      "kl": 0.28149276971817017,
      "learning_rate": 4.57572202080948e-06,
      "loss": 0.0113,
      "reward": 1.2240657806396484,
      "reward_std": 0.8058105707168579,
      "rewards/reward_function": 1.2240657806396484,
      "step": 471
    },
    {
      "completion_length": 134.82144165039062,
      "epoch": 0.2700228832951945,
      "grad_norm": 0.40518397092819214,
      "kl": 0.21027973294258118,
      "learning_rate": 4.5729351198915715e-06,
      "loss": 0.0084,
      "reward": 1.1420345306396484,
      "reward_std": 0.6366257071495056,
      "rewards/reward_function": 1.1420345306396484,
      "step": 472
    },
    {
      "completion_length": 126.14286041259766,
      "epoch": 0.2705949656750572,
      "grad_norm": 0.5612007975578308,
      "kl": 0.2517709732055664,
      "learning_rate": 4.5701399504520614e-06,
      "loss": 0.0101,
      "reward": 1.3623584508895874,
      "reward_std": 0.5349483489990234,
      "rewards/reward_function": 1.3623584508895874,
      "step": 473
    },
    {
      "completion_length": 134.5357208251953,
      "epoch": 0.2711670480549199,
      "grad_norm": 0.453670859336853,
      "kl": 0.21429789066314697,
      "learning_rate": 4.567336523640322e-06,
      "loss": 0.0086,
      "reward": 1.7337956428527832,
      "reward_std": 0.8322926759719849,
      "rewards/reward_function": 1.7337956428527832,
      "step": 474
    },
    {
      "completion_length": 162.5357208251953,
      "epoch": 0.2717391304347826,
      "grad_norm": 0.3770849406719208,
      "kl": 0.29119619727134705,
      "learning_rate": 4.564524850638657e-06,
      "loss": 0.0116,
      "reward": 1.3070448637008667,
      "reward_std": 1.0406376123428345,
      "rewards/reward_function": 1.3070448637008667,
      "step": 475
    },
    {
      "completion_length": 131.85714721679688,
      "epoch": 0.2723112128146453,
      "grad_norm": 0.3998708426952362,
      "kl": 0.3102273941040039,
      "learning_rate": 4.561704942662267e-06,
      "loss": 0.0124,
      "reward": 1.8984161615371704,
      "reward_std": 0.7959803342819214,
      "rewards/reward_function": 1.8984161615371704,
      "step": 476
    },
    {
      "completion_length": 149.10714721679688,
      "epoch": 0.272883295194508,
      "grad_norm": 0.32895782589912415,
      "kl": 0.2083321511745453,
      "learning_rate": 4.558876810959196e-06,
      "loss": 0.0083,
      "reward": 1.5216991901397705,
      "reward_std": 0.5879191756248474,
      "rewards/reward_function": 1.5216991901397705,
      "step": 477
    },
    {
      "completion_length": 130.82144165039062,
      "epoch": 0.2734553775743707,
      "grad_norm": 0.4074607491493225,
      "kl": 0.21760544180870056,
      "learning_rate": 4.5560404668102956e-06,
      "loss": 0.0087,
      "reward": 1.5847071409225464,
      "reward_std": 0.39057719707489014,
      "rewards/reward_function": 1.5847071409225464,
      "step": 478
    },
    {
      "completion_length": 131.21429443359375,
      "epoch": 0.2740274599542334,
      "grad_norm": 0.44541865587234497,
      "kl": 0.2450018674135208,
      "learning_rate": 4.55319592152917e-06,
      "loss": 0.0098,
      "reward": 2.306325912475586,
      "reward_std": 0.7508701086044312,
      "rewards/reward_function": 2.306325912475586,
      "step": 479
    },
    {
      "completion_length": 192.92857360839844,
      "epoch": 0.2745995423340961,
      "grad_norm": 0.4072016477584839,
      "kl": 0.2732917070388794,
      "learning_rate": 4.5503431864621395e-06,
      "loss": 0.0109,
      "reward": 0.9873977899551392,
      "reward_std": 0.5991553664207458,
      "rewards/reward_function": 0.9873977899551392,
      "step": 480
    },
    {
      "completion_length": 156.60714721679688,
      "epoch": 0.2751716247139588,
      "grad_norm": 0.42143872380256653,
      "kl": 0.2530211806297302,
      "learning_rate": 4.547482272988192e-06,
      "loss": 0.0101,
      "reward": 1.637827754020691,
      "reward_std": 0.8168033957481384,
      "rewards/reward_function": 1.637827754020691,
      "step": 481
    },
    {
      "completion_length": 166.46429443359375,
      "epoch": 0.2757437070938215,
      "grad_norm": 0.3319096267223358,
      "kl": 0.2190791517496109,
      "learning_rate": 4.544613192518935e-06,
      "loss": 0.0088,
      "reward": 1.4849402904510498,
      "reward_std": 0.8733363151550293,
      "rewards/reward_function": 1.4849402904510498,
      "step": 482
    },
    {
      "completion_length": 130.57144165039062,
      "epoch": 0.27631578947368424,
      "grad_norm": 0.32811981439590454,
      "kl": 0.209286630153656,
      "learning_rate": 4.541735956498555e-06,
      "loss": 0.0084,
      "reward": 0.8455315232276917,
      "reward_std": 0.44075602293014526,
      "rewards/reward_function": 0.8455315232276917,
      "step": 483
    },
    {
      "completion_length": 132.0357208251953,
      "epoch": 0.2768878718535469,
      "grad_norm": 0.41199445724487305,
      "kl": 0.36435577273368835,
      "learning_rate": 4.538850576403767e-06,
      "loss": 0.0146,
      "reward": 1.2930189371109009,
      "reward_std": 0.3975881040096283,
      "rewards/reward_function": 1.2930189371109009,
      "step": 484
    },
    {
      "completion_length": 133.82144165039062,
      "epoch": 0.27745995423340963,
      "grad_norm": 0.39215347170829773,
      "kl": 0.19682319462299347,
      "learning_rate": 4.535957063743775e-06,
      "loss": 0.0079,
      "reward": 2.07631516456604,
      "reward_std": 0.5592927932739258,
      "rewards/reward_function": 2.07631516456604,
      "step": 485
    },
    {
      "completion_length": 143.92857360839844,
      "epoch": 0.2780320366132723,
      "grad_norm": 0.43241143226623535,
      "kl": 0.2111026495695114,
      "learning_rate": 4.533055430060216e-06,
      "loss": 0.0084,
      "reward": 1.3952072858810425,
      "reward_std": 0.7626815438270569,
      "rewards/reward_function": 1.3952072858810425,
      "step": 486
    },
    {
      "completion_length": 142.60714721679688,
      "epoch": 0.27860411899313503,
      "grad_norm": 0.340329647064209,
      "kl": 0.3026789128780365,
      "learning_rate": 4.530145686927126e-06,
      "loss": 0.0121,
      "reward": 1.7433359622955322,
      "reward_std": 0.800469160079956,
      "rewards/reward_function": 1.7433359622955322,
      "step": 487
    },
    {
      "completion_length": 162.67857360839844,
      "epoch": 0.2791762013729977,
      "grad_norm": 0.3960564434528351,
      "kl": 0.26710590720176697,
      "learning_rate": 4.527227845950884e-06,
      "loss": 0.0107,
      "reward": 1.643479585647583,
      "reward_std": 1.0279161930084229,
      "rewards/reward_function": 1.643479585647583,
      "step": 488
    },
    {
      "completion_length": 149.21429443359375,
      "epoch": 0.2797482837528604,
      "grad_norm": 0.257808655500412,
      "kl": 0.15491244196891785,
      "learning_rate": 4.524301918770173e-06,
      "loss": 0.0062,
      "reward": 1.963416576385498,
      "reward_std": 0.9698876142501831,
      "rewards/reward_function": 1.963416576385498,
      "step": 489
    },
    {
      "completion_length": 146.75,
      "epoch": 0.2803203661327231,
      "grad_norm": 0.39049699902534485,
      "kl": 0.25186610221862793,
      "learning_rate": 4.521367917055926e-06,
      "loss": 0.0101,
      "reward": 1.7200628519058228,
      "reward_std": 0.5543420314788818,
      "rewards/reward_function": 1.7200628519058228,
      "step": 490
    },
    {
      "completion_length": 140.82144165039062,
      "epoch": 0.2808924485125858,
      "grad_norm": 0.795330286026001,
      "kl": 0.27166110277175903,
      "learning_rate": 4.51842585251129e-06,
      "loss": 0.0109,
      "reward": 0.971901535987854,
      "reward_std": 0.5884334444999695,
      "rewards/reward_function": 0.971901535987854,
      "step": 491
    },
    {
      "completion_length": 133.96429443359375,
      "epoch": 0.2814645308924485,
      "grad_norm": 0.3234439194202423,
      "kl": 0.18059344589710236,
      "learning_rate": 4.515475736871566e-06,
      "loss": 0.0072,
      "reward": 1.786487102508545,
      "reward_std": 0.6225761771202087,
      "rewards/reward_function": 1.786487102508545,
      "step": 492
    },
    {
      "completion_length": 163.32144165039062,
      "epoch": 0.2820366132723112,
      "grad_norm": 1.0382165908813477,
      "kl": 0.7980419993400574,
      "learning_rate": 4.5125175819041735e-06,
      "loss": 0.0319,
      "reward": 1.1104196310043335,
      "reward_std": 0.43268853425979614,
      "rewards/reward_function": 1.1104196310043335,
      "step": 493
    },
    {
      "completion_length": 139.57144165039062,
      "epoch": 0.2826086956521739,
      "grad_norm": 0.4256740212440491,
      "kl": 0.22271700203418732,
      "learning_rate": 4.509551399408598e-06,
      "loss": 0.0089,
      "reward": 1.2183958292007446,
      "reward_std": 0.7855218648910522,
      "rewards/reward_function": 1.2183958292007446,
      "step": 494
    },
    {
      "completion_length": 170.71429443359375,
      "epoch": 0.2831807780320366,
      "grad_norm": 0.33020299673080444,
      "kl": 0.23153334856033325,
      "learning_rate": 4.506577201216346e-06,
      "loss": 0.0093,
      "reward": 1.4302849769592285,
      "reward_std": 0.7724705934524536,
      "rewards/reward_function": 1.4302849769592285,
      "step": 495
    },
    {
      "completion_length": 171.7857208251953,
      "epoch": 0.2837528604118993,
      "grad_norm": 28.784656524658203,
      "kl": 1.0991169214248657,
      "learning_rate": 4.5035949991908955e-06,
      "loss": 0.044,
      "reward": 1.6449105739593506,
      "reward_std": 0.850498616695404,
      "rewards/reward_function": 1.6449105739593506,
      "step": 496
    },
    {
      "completion_length": 153.10714721679688,
      "epoch": 0.284324942791762,
      "grad_norm": 0.25954508781433105,
      "kl": 0.16062794625759125,
      "learning_rate": 4.50060480522765e-06,
      "loss": 0.0064,
      "reward": 1.337046504020691,
      "reward_std": 0.5881532430648804,
      "rewards/reward_function": 1.337046504020691,
      "step": 497
    },
    {
      "completion_length": 172.07144165039062,
      "epoch": 0.28489702517162474,
      "grad_norm": 1.1429195404052734,
      "kl": 0.6584891676902771,
      "learning_rate": 4.497606631253896e-06,
      "loss": 0.0263,
      "reward": 1.54092276096344,
      "reward_std": 0.9960590600967407,
      "rewards/reward_function": 1.54092276096344,
      "step": 498
    },
    {
      "completion_length": 182.3928680419922,
      "epoch": 0.2854691075514874,
      "grad_norm": 0.23832781612873077,
      "kl": 0.18733768165111542,
      "learning_rate": 4.494600489228744e-06,
      "loss": 0.0075,
      "reward": 1.672544002532959,
      "reward_std": 0.7880839109420776,
      "rewards/reward_function": 1.672544002532959,
      "step": 499
    },
    {
      "completion_length": 155.25,
      "epoch": 0.28604118993135014,
      "grad_norm": 0.3216526508331299,
      "kl": 0.19687272608280182,
      "learning_rate": 4.49158639114309e-06,
      "loss": 0.0079,
      "reward": 1.8210067749023438,
      "reward_std": 0.8018940687179565,
      "rewards/reward_function": 1.8210067749023438,
      "step": 500
    },
    {
      "completion_length": 163.7857208251953,
      "epoch": 0.2866132723112128,
      "grad_norm": 0.3482306897640228,
      "kl": 0.20922218263149261,
      "learning_rate": 4.488564349019567e-06,
      "loss": 0.0084,
      "reward": 1.7350260019302368,
      "reward_std": 0.9087448716163635,
      "rewards/reward_function": 1.7350260019302368,
      "step": 501
    },
    {
      "completion_length": 174.8928680419922,
      "epoch": 0.28718535469107553,
      "grad_norm": 0.29870858788490295,
      "kl": 0.19968834519386292,
      "learning_rate": 4.485534374912494e-06,
      "loss": 0.008,
      "reward": 1.2248133420944214,
      "reward_std": 0.5122812986373901,
      "rewards/reward_function": 1.2248133420944214,
      "step": 502
    },
    {
      "completion_length": 159.2857208251953,
      "epoch": 0.2877574370709382,
      "grad_norm": 0.26699596643447876,
      "kl": 0.22807738184928894,
      "learning_rate": 4.482496480907828e-06,
      "loss": 0.0091,
      "reward": 1.6710809469223022,
      "reward_std": 0.8279319405555725,
      "rewards/reward_function": 1.6710809469223022,
      "step": 503
    },
    {
      "completion_length": 140.46429443359375,
      "epoch": 0.28832951945080093,
      "grad_norm": 0.30161723494529724,
      "kl": 0.2179427295923233,
      "learning_rate": 4.4794506791231175e-06,
      "loss": 0.0087,
      "reward": 1.3638322353363037,
      "reward_std": 0.4015527665615082,
      "rewards/reward_function": 1.3638322353363037,
      "step": 504
    },
    {
      "completion_length": 173.42857360839844,
      "epoch": 0.2889016018306636,
      "grad_norm": 0.30333390831947327,
      "kl": 0.23498573899269104,
      "learning_rate": 4.476396981707454e-06,
      "loss": 0.0094,
      "reward": 2.3101072311401367,
      "reward_std": 0.7363930344581604,
      "rewards/reward_function": 2.3101072311401367,
      "step": 505
    },
    {
      "completion_length": 150.2857208251953,
      "epoch": 0.2894736842105263,
      "grad_norm": 0.6662407517433167,
      "kl": 0.35804927349090576,
      "learning_rate": 4.473335400841422e-06,
      "loss": 0.0143,
      "reward": 1.6867488622665405,
      "reward_std": 0.7744774222373962,
      "rewards/reward_function": 1.6867488622665405,
      "step": 506
    },
    {
      "completion_length": 163.85714721679688,
      "epoch": 0.290045766590389,
      "grad_norm": 3747.46337890625,
      "kl": 13.44710636138916,
      "learning_rate": 4.470265948737053e-06,
      "loss": 0.5379,
      "reward": 1.8456603288650513,
      "reward_std": 0.9866312742233276,
      "rewards/reward_function": 1.8456603288650513,
      "step": 507
    },
    {
      "completion_length": 166.5357208251953,
      "epoch": 0.2906178489702517,
      "grad_norm": 0.2500198185443878,
      "kl": 0.18421582877635956,
      "learning_rate": 4.467188637637775e-06,
      "loss": 0.0074,
      "reward": 1.364561915397644,
      "reward_std": 0.6478800177574158,
      "rewards/reward_function": 1.364561915397644,
      "step": 508
    },
    {
      "completion_length": 147.82144165039062,
      "epoch": 0.2911899313501144,
      "grad_norm": 0.30894777178764343,
      "kl": 0.22879278659820557,
      "learning_rate": 4.464103479818363e-06,
      "loss": 0.0092,
      "reward": 1.0292110443115234,
      "reward_std": 0.41830670833587646,
      "rewards/reward_function": 1.0292110443115234,
      "step": 509
    },
    {
      "completion_length": 120.25000762939453,
      "epoch": 0.2917620137299771,
      "grad_norm": 0.30429884791374207,
      "kl": 0.1691063493490219,
      "learning_rate": 4.461010487584892e-06,
      "loss": 0.0068,
      "reward": 1.7997546195983887,
      "reward_std": 0.40408822894096375,
      "rewards/reward_function": 1.7997546195983887,
      "step": 510
    },
    {
      "completion_length": 162.96429443359375,
      "epoch": 0.2923340961098398,
      "grad_norm": 0.2874217629432678,
      "kl": 0.1745148003101349,
      "learning_rate": 4.457909673274686e-06,
      "loss": 0.007,
      "reward": 1.113578200340271,
      "reward_std": 0.5771594047546387,
      "rewards/reward_function": 1.113578200340271,
      "step": 511
    },
    {
      "completion_length": 161.96429443359375,
      "epoch": 0.2929061784897025,
      "grad_norm": 0.36285361647605896,
      "kl": 0.2346002161502838,
      "learning_rate": 4.45480104925627e-06,
      "loss": 0.0094,
      "reward": 2.321317672729492,
      "reward_std": 0.6548759341239929,
      "rewards/reward_function": 2.321317672729492,
      "step": 512
    },
    {
      "completion_length": 164.0357208251953,
      "epoch": 0.29347826086956524,
      "grad_norm": 235.92820739746094,
      "kl": 1.8405425548553467,
      "learning_rate": 4.45168462792932e-06,
      "loss": 0.0736,
      "reward": 2.1454615592956543,
      "reward_std": 0.9428712725639343,
      "rewards/reward_function": 2.1454615592956543,
      "step": 513
    },
    {
      "completion_length": 166.42857360839844,
      "epoch": 0.2940503432494279,
      "grad_norm": 0.2664567232131958,
      "kl": 0.2215391993522644,
      "learning_rate": 4.448560421724616e-06,
      "loss": 0.0089,
      "reward": 1.0907094478607178,
      "reward_std": 0.6727427244186401,
      "rewards/reward_function": 1.0907094478607178,
      "step": 514
    },
    {
      "completion_length": 170.96429443359375,
      "epoch": 0.29462242562929064,
      "grad_norm": 247.8568115234375,
      "kl": 1.3497196435928345,
      "learning_rate": 4.4454284431039875e-06,
      "loss": 0.054,
      "reward": 2.0259273052215576,
      "reward_std": 0.8155786395072937,
      "rewards/reward_function": 2.0259273052215576,
      "step": 515
    },
    {
      "completion_length": 165.25,
      "epoch": 0.2951945080091533,
      "grad_norm": 0.43828821182250977,
      "kl": 0.2669973373413086,
      "learning_rate": 4.442288704560268e-06,
      "loss": 0.0107,
      "reward": 1.589293122291565,
      "reward_std": 0.5872241258621216,
      "rewards/reward_function": 1.589293122291565,
      "step": 516
    },
    {
      "completion_length": 172.42857360839844,
      "epoch": 0.29576659038901604,
      "grad_norm": 0.5143162608146667,
      "kl": 0.252464234828949,
      "learning_rate": 4.439141218617244e-06,
      "loss": 0.0101,
      "reward": 2.100668430328369,
      "reward_std": 0.9035283923149109,
      "rewards/reward_function": 2.100668430328369,
      "step": 517
    },
    {
      "completion_length": 161.17857360839844,
      "epoch": 0.2963386727688787,
      "grad_norm": 0.3679174482822418,
      "kl": 0.255586713552475,
      "learning_rate": 4.435985997829603e-06,
      "loss": 0.0102,
      "reward": 1.5752097368240356,
      "reward_std": 0.7873923778533936,
      "rewards/reward_function": 1.5752097368240356,
      "step": 518
    },
    {
      "completion_length": 147.35714721679688,
      "epoch": 0.29691075514874143,
      "grad_norm": 0.32651352882385254,
      "kl": 0.18201051652431488,
      "learning_rate": 4.432823054782889e-06,
      "loss": 0.0073,
      "reward": 1.0584114789962769,
      "reward_std": 0.6168755888938904,
      "rewards/reward_function": 1.0584114789962769,
      "step": 519
    },
    {
      "completion_length": 171.21429443359375,
      "epoch": 0.2974828375286041,
      "grad_norm": 1.1260119676589966,
      "kl": 0.5506652593612671,
      "learning_rate": 4.429652402093443e-06,
      "loss": 0.022,
      "reward": 1.8280678987503052,
      "reward_std": 0.8363029360771179,
      "rewards/reward_function": 1.8280678987503052,
      "step": 520
    },
    {
      "completion_length": 149.07144165039062,
      "epoch": 0.29805491990846683,
      "grad_norm": 0.28136903047561646,
      "kl": 0.18667267262935638,
      "learning_rate": 4.426474052408364e-06,
      "loss": 0.0075,
      "reward": 1.634071707725525,
      "reward_std": 0.3315223157405853,
      "rewards/reward_function": 1.634071707725525,
      "step": 521
    },
    {
      "completion_length": 174.71429443359375,
      "epoch": 0.2986270022883295,
      "grad_norm": 0.26725783944129944,
      "kl": 0.22262533009052277,
      "learning_rate": 4.423288018405449e-06,
      "loss": 0.0089,
      "reward": 1.9422218799591064,
      "reward_std": 1.0097899436950684,
      "rewards/reward_function": 1.9422218799591064,
      "step": 522
    },
    {
      "completion_length": 183.92857360839844,
      "epoch": 0.29919908466819223,
      "grad_norm": 7.397769927978516,
      "kl": 0.6622462272644043,
      "learning_rate": 4.420094312793148e-06,
      "loss": 0.0265,
      "reward": 1.561123013496399,
      "reward_std": 1.0609776973724365,
      "rewards/reward_function": 1.561123013496399,
      "step": 523
    },
    {
      "completion_length": 202.96429443359375,
      "epoch": 0.2997711670480549,
      "grad_norm": 0.2608702778816223,
      "kl": 0.19361503422260284,
      "learning_rate": 4.416892948310511e-06,
      "loss": 0.0077,
      "reward": 1.5102163553237915,
      "reward_std": 0.8623517751693726,
      "rewards/reward_function": 1.5102163553237915,
      "step": 524
    },
    {
      "completion_length": 161.92857360839844,
      "epoch": 0.3003432494279176,
      "grad_norm": 12.407898902893066,
      "kl": 0.6012916564941406,
      "learning_rate": 4.4136839377271365e-06,
      "loss": 0.0241,
      "reward": 1.20400869846344,
      "reward_std": 0.6343669891357422,
      "rewards/reward_function": 1.20400869846344,
      "step": 525
    },
    {
      "completion_length": 193.42857360839844,
      "epoch": 0.3009153318077803,
      "grad_norm": 0.3476836383342743,
      "kl": 0.3443866968154907,
      "learning_rate": 4.410467293843123e-06,
      "loss": 0.0138,
      "reward": 1.639494776725769,
      "reward_std": 1.0363049507141113,
      "rewards/reward_function": 1.639494776725769,
      "step": 526
    },
    {
      "completion_length": 166.32144165039062,
      "epoch": 0.301487414187643,
      "grad_norm": 3.913153886795044,
      "kl": 0.6835634112358093,
      "learning_rate": 4.407243029489018e-06,
      "loss": 0.0273,
      "reward": 1.8795716762542725,
      "reward_std": 0.9814324378967285,
      "rewards/reward_function": 1.8795716762542725,
      "step": 527
    },
    {
      "completion_length": 214.6428680419922,
      "epoch": 0.30205949656750575,
      "grad_norm": 0.29172709584236145,
      "kl": 0.22586731612682343,
      "learning_rate": 4.404011157525761e-06,
      "loss": 0.009,
      "reward": 1.517635464668274,
      "reward_std": 0.779082715511322,
      "rewards/reward_function": 1.517635464668274,
      "step": 528
    },
    {
      "completion_length": 196.82144165039062,
      "epoch": 0.3026315789473684,
      "grad_norm": 0.33362409472465515,
      "kl": 0.2036721259355545,
      "learning_rate": 4.400771690844643e-06,
      "loss": 0.0081,
      "reward": 2.0596418380737305,
      "reward_std": 0.7829192280769348,
      "rewards/reward_function": 2.0596418380737305,
      "step": 529
    },
    {
      "completion_length": 189.00001525878906,
      "epoch": 0.30320366132723114,
      "grad_norm": 0.2522986829280853,
      "kl": 0.23559924960136414,
      "learning_rate": 4.397524642367244e-06,
      "loss": 0.0094,
      "reward": 1.194425344467163,
      "reward_std": 0.5229185819625854,
      "rewards/reward_function": 1.194425344467163,
      "step": 530
    },
    {
      "completion_length": 151.75,
      "epoch": 0.3037757437070938,
      "grad_norm": 0.6237693428993225,
      "kl": 0.44375601410865784,
      "learning_rate": 4.394270025045388e-06,
      "loss": 0.0178,
      "reward": 2.0679659843444824,
      "reward_std": 0.7726342678070068,
      "rewards/reward_function": 2.0679659843444824,
      "step": 531
    },
    {
      "completion_length": 165.92857360839844,
      "epoch": 0.30434782608695654,
      "grad_norm": 0.3459445536136627,
      "kl": 0.1785653978586197,
      "learning_rate": 4.391007851861089e-06,
      "loss": 0.0071,
      "reward": 1.0744621753692627,
      "reward_std": 0.5338218212127686,
      "rewards/reward_function": 1.0744621753692627,
      "step": 532
    },
    {
      "completion_length": 157.60714721679688,
      "epoch": 0.3049199084668192,
      "grad_norm": 0.2863959074020386,
      "kl": 0.18569734692573547,
      "learning_rate": 4.387738135826501e-06,
      "loss": 0.0074,
      "reward": 0.7431676983833313,
      "reward_std": 0.2450118064880371,
      "rewards/reward_function": 0.7431676983833313,
      "step": 533
    },
    {
      "completion_length": 184.10714721679688,
      "epoch": 0.30549199084668194,
      "grad_norm": 0.25961095094680786,
      "kl": 0.18141235411167145,
      "learning_rate": 4.384460889983864e-06,
      "loss": 0.0073,
      "reward": 1.7764030694961548,
      "reward_std": 0.9601207971572876,
      "rewards/reward_function": 1.7764030694961548,
      "step": 534
    },
    {
      "completion_length": 159.35714721679688,
      "epoch": 0.3060640732265446,
      "grad_norm": 1.746524453163147,
      "kl": 0.845272421836853,
      "learning_rate": 4.3811761274054535e-06,
      "loss": 0.0338,
      "reward": 1.476712703704834,
      "reward_std": 0.7681740522384644,
      "rewards/reward_function": 1.476712703704834,
      "step": 535
    },
    {
      "completion_length": 156.92857360839844,
      "epoch": 0.30663615560640733,
      "grad_norm": 3.327455520629883,
      "kl": 0.8283001780509949,
      "learning_rate": 4.377883861193526e-06,
      "loss": 0.0331,
      "reward": 1.1935455799102783,
      "reward_std": 0.7638345956802368,
      "rewards/reward_function": 1.1935455799102783,
      "step": 536
    },
    {
      "completion_length": 162.96429443359375,
      "epoch": 0.30720823798627,
      "grad_norm": 0.31985703110694885,
      "kl": 0.22370433807373047,
      "learning_rate": 4.374584104480271e-06,
      "loss": 0.0089,
      "reward": 2.2072603702545166,
      "reward_std": 0.8702509999275208,
      "rewards/reward_function": 2.2072603702545166,
      "step": 537
    },
    {
      "completion_length": 155.25,
      "epoch": 0.30778032036613273,
      "grad_norm": 0.35214537382125854,
      "kl": 0.19426093995571136,
      "learning_rate": 4.3712768704277535e-06,
      "loss": 0.0078,
      "reward": 1.7004172801971436,
      "reward_std": 0.8058720231056213,
      "rewards/reward_function": 1.7004172801971436,
      "step": 538
    },
    {
      "completion_length": 187.07144165039062,
      "epoch": 0.3083524027459954,
      "grad_norm": 0.32907402515411377,
      "kl": 0.278493732213974,
      "learning_rate": 4.367962172227866e-06,
      "loss": 0.0111,
      "reward": 2.142878770828247,
      "reward_std": 0.6116499900817871,
      "rewards/reward_function": 2.142878770828247,
      "step": 539
    },
    {
      "completion_length": 155.10714721679688,
      "epoch": 0.30892448512585813,
      "grad_norm": 0.2527097761631012,
      "kl": 0.16127876937389374,
      "learning_rate": 4.364640023102275e-06,
      "loss": 0.0065,
      "reward": 1.2749650478363037,
      "reward_std": 0.6877351999282837,
      "rewards/reward_function": 1.2749650478363037,
      "step": 540
    },
    {
      "completion_length": 176.46429443359375,
      "epoch": 0.3094965675057208,
      "grad_norm": 1.1961147785186768,
      "kl": 0.583297848701477,
      "learning_rate": 4.361310436302364e-06,
      "loss": 0.0233,
      "reward": 1.716264009475708,
      "reward_std": 0.9600126147270203,
      "rewards/reward_function": 1.716264009475708,
      "step": 541
    },
    {
      "completion_length": 176.0357208251953,
      "epoch": 0.3100686498855835,
      "grad_norm": 0.33236414194107056,
      "kl": 0.16040414571762085,
      "learning_rate": 4.357973425109187e-06,
      "loss": 0.0064,
      "reward": 1.8081538677215576,
      "reward_std": 0.5220481753349304,
      "rewards/reward_function": 1.8081538677215576,
      "step": 542
    },
    {
      "completion_length": 198.46429443359375,
      "epoch": 0.31064073226544625,
      "grad_norm": 0.3281603753566742,
      "kl": 0.22552408277988434,
      "learning_rate": 4.354629002833408e-06,
      "loss": 0.009,
      "reward": 1.135749340057373,
      "reward_std": 0.4299235939979553,
      "rewards/reward_function": 1.135749340057373,
      "step": 543
    },
    {
      "completion_length": 179.67857360839844,
      "epoch": 0.3112128146453089,
      "grad_norm": 0.3235718309879303,
      "kl": 0.2224029302597046,
      "learning_rate": 4.351277182815259e-06,
      "loss": 0.0089,
      "reward": 1.44709050655365,
      "reward_std": 0.566405713558197,
      "rewards/reward_function": 1.44709050655365,
      "step": 544
    },
    {
      "completion_length": 180.21429443359375,
      "epoch": 0.31178489702517165,
      "grad_norm": 0.3678370416164398,
      "kl": 0.33764538168907166,
      "learning_rate": 4.3479179784244754e-06,
      "loss": 0.0135,
      "reward": 2.2020413875579834,
      "reward_std": 0.7093104124069214,
      "rewards/reward_function": 2.2020413875579834,
      "step": 545
    },
    {
      "completion_length": 191.46429443359375,
      "epoch": 0.3123569794050343,
      "grad_norm": 0.41091397404670715,
      "kl": 0.36596620082855225,
      "learning_rate": 4.344551403060247e-06,
      "loss": 0.0146,
      "reward": 1.4783011674880981,
      "reward_std": 0.7843088507652283,
      "rewards/reward_function": 1.4783011674880981,
      "step": 546
    },
    {
      "completion_length": 181.96429443359375,
      "epoch": 0.31292906178489704,
      "grad_norm": 0.296642005443573,
      "kl": 0.2399391084909439,
      "learning_rate": 4.341177470151168e-06,
      "loss": 0.0096,
      "reward": 2.1102943420410156,
      "reward_std": 1.1402618885040283,
      "rewards/reward_function": 2.1102943420410156,
      "step": 547
    },
    {
      "completion_length": 202.96429443359375,
      "epoch": 0.3135011441647597,
      "grad_norm": 0.49794548749923706,
      "kl": 0.3468903601169586,
      "learning_rate": 4.337796193155177e-06,
      "loss": 0.0139,
      "reward": 2.53847599029541,
      "reward_std": 0.9375091791152954,
      "rewards/reward_function": 2.53847599029541,
      "step": 548
    },
    {
      "completion_length": 203.7857208251953,
      "epoch": 0.31407322654462244,
      "grad_norm": 0.31221845746040344,
      "kl": 0.22393770515918732,
      "learning_rate": 4.33440758555951e-06,
      "loss": 0.009,
      "reward": 1.3776577711105347,
      "reward_std": 1.053886890411377,
      "rewards/reward_function": 1.3776577711105347,
      "step": 549
    },
    {
      "completion_length": 187.2857208251953,
      "epoch": 0.3146453089244851,
      "grad_norm": 0.3054976463317871,
      "kl": 0.20163311064243317,
      "learning_rate": 4.331011660880642e-06,
      "loss": 0.0081,
      "reward": 1.8747998476028442,
      "reward_std": 0.9617244005203247,
      "rewards/reward_function": 1.8747998476028442,
      "step": 550
    },
    {
      "completion_length": 188.3928680419922,
      "epoch": 0.31521739130434784,
      "grad_norm": 0.2845914363861084,
      "kl": 0.21168704330921173,
      "learning_rate": 4.327608432664232e-06,
      "loss": 0.0085,
      "reward": 1.7652709484100342,
      "reward_std": 0.5103840827941895,
      "rewards/reward_function": 1.7652709484100342,
      "step": 551
    },
    {
      "completion_length": 195.32144165039062,
      "epoch": 0.3157894736842105,
      "grad_norm": 0.41424500942230225,
      "kl": 0.32905495166778564,
      "learning_rate": 4.324197914485075e-06,
      "loss": 0.0132,
      "reward": 1.1698718070983887,
      "reward_std": 0.6045928001403809,
      "rewards/reward_function": 1.1698718070983887,
      "step": 552
    },
    {
      "completion_length": 217.10714721679688,
      "epoch": 0.31636155606407323,
      "grad_norm": 0.6983904242515564,
      "kl": 0.31256434321403503,
      "learning_rate": 4.320780119947042e-06,
      "loss": 0.0125,
      "reward": 1.3578120470046997,
      "reward_std": 0.44349271059036255,
      "rewards/reward_function": 1.3578120470046997,
      "step": 553
    },
    {
      "completion_length": 174.5357208251953,
      "epoch": 0.3169336384439359,
      "grad_norm": 0.3152120113372803,
      "kl": 0.26250967383384705,
      "learning_rate": 4.31735506268303e-06,
      "loss": 0.0105,
      "reward": 1.79605233669281,
      "reward_std": 0.9627654552459717,
      "rewards/reward_function": 1.79605233669281,
      "step": 554
    },
    {
      "completion_length": 196.5357208251953,
      "epoch": 0.31750572082379863,
      "grad_norm": 0.2456715703010559,
      "kl": 0.19430384039878845,
      "learning_rate": 4.313922756354902e-06,
      "loss": 0.0078,
      "reward": 1.8962770700454712,
      "reward_std": 1.1194885969161987,
      "rewards/reward_function": 1.8962770700454712,
      "step": 555
    },
    {
      "completion_length": 162.42857360839844,
      "epoch": 0.3180778032036613,
      "grad_norm": 0.32084789872169495,
      "kl": 0.22721755504608154,
      "learning_rate": 4.310483214653439e-06,
      "loss": 0.0091,
      "reward": 1.4062535762786865,
      "reward_std": 0.6449627876281738,
      "rewards/reward_function": 1.4062535762786865,
      "step": 556
    },
    {
      "completion_length": 189.71429443359375,
      "epoch": 0.31864988558352403,
      "grad_norm": 0.2805863916873932,
      "kl": 0.16517221927642822,
      "learning_rate": 4.307036451298282e-06,
      "loss": 0.0066,
      "reward": 1.8296490907669067,
      "reward_std": 0.9636064767837524,
      "rewards/reward_function": 1.8296490907669067,
      "step": 557
    },
    {
      "completion_length": 210.92857360839844,
      "epoch": 0.31922196796338675,
      "grad_norm": 1.0820069313049316,
      "kl": 0.2339024692773819,
      "learning_rate": 4.3035824800378765e-06,
      "loss": 0.0094,
      "reward": 1.1028753519058228,
      "reward_std": 0.6142609119415283,
      "rewards/reward_function": 1.1028753519058228,
      "step": 558
    },
    {
      "completion_length": 168.07144165039062,
      "epoch": 0.3197940503432494,
      "grad_norm": 0.24397020041942596,
      "kl": 0.17751647531986237,
      "learning_rate": 4.30012131464942e-06,
      "loss": 0.0071,
      "reward": 1.1539857387542725,
      "reward_std": 0.5515402555465698,
      "rewards/reward_function": 1.1539857387542725,
      "step": 559
    },
    {
      "completion_length": 218.1428680419922,
      "epoch": 0.32036613272311215,
      "grad_norm": 0.3258097171783447,
      "kl": 0.253081351518631,
      "learning_rate": 4.296652968938807e-06,
      "loss": 0.0101,
      "reward": 2.0412662029266357,
      "reward_std": 1.0736823081970215,
      "rewards/reward_function": 2.0412662029266357,
      "step": 560
    },
    {
      "completion_length": 178.0357208251953,
      "epoch": 0.3209382151029748,
      "grad_norm": 0.26388317346572876,
      "kl": 0.17069274187088013,
      "learning_rate": 4.29317745674057e-06,
      "loss": 0.0068,
      "reward": 2.0467679500579834,
      "reward_std": 1.0732760429382324,
      "rewards/reward_function": 2.0467679500579834,
      "step": 561
    },
    {
      "completion_length": 178.35714721679688,
      "epoch": 0.32151029748283755,
      "grad_norm": 0.23071327805519104,
      "kl": 0.15518805384635925,
      "learning_rate": 4.289694791917828e-06,
      "loss": 0.0062,
      "reward": 1.5068360567092896,
      "reward_std": 0.6527971029281616,
      "rewards/reward_function": 1.5068360567092896,
      "step": 562
    },
    {
      "completion_length": 190.57144165039062,
      "epoch": 0.3220823798627002,
      "grad_norm": 0.2825309634208679,
      "kl": 0.21008749306201935,
      "learning_rate": 4.286204988362233e-06,
      "loss": 0.0084,
      "reward": 1.6776701211929321,
      "reward_std": 0.42292752861976624,
      "rewards/reward_function": 1.6776701211929321,
      "step": 563
    },
    {
      "completion_length": 182.46429443359375,
      "epoch": 0.32265446224256294,
      "grad_norm": 1.1122941970825195,
      "kl": 0.361390620470047,
      "learning_rate": 4.2827080599939095e-06,
      "loss": 0.0145,
      "reward": 1.7013400793075562,
      "reward_std": 0.6561534404754639,
      "rewards/reward_function": 1.7013400793075562,
      "step": 564
    },
    {
      "completion_length": 246.6785888671875,
      "epoch": 0.3232265446224256,
      "grad_norm": 0.6099883317947388,
      "kl": 0.2942289710044861,
      "learning_rate": 4.279204020761401e-06,
      "loss": 0.0118,
      "reward": 1.6106127500534058,
      "reward_std": 0.4963645040988922,
      "rewards/reward_function": 1.6106127500534058,
      "step": 565
    },
    {
      "completion_length": 191.60714721679688,
      "epoch": 0.32379862700228834,
      "grad_norm": 0.2986891269683838,
      "kl": 0.2018136829137802,
      "learning_rate": 4.275692884641617e-06,
      "loss": 0.0081,
      "reward": 1.6833183765411377,
      "reward_std": 0.7101523876190186,
      "rewards/reward_function": 1.6833183765411377,
      "step": 566
    },
    {
      "completion_length": 194.50001525878906,
      "epoch": 0.324370709382151,
      "grad_norm": 0.36337772011756897,
      "kl": 0.17345406115055084,
      "learning_rate": 4.272174665639773e-06,
      "loss": 0.0069,
      "reward": 1.8500674962997437,
      "reward_std": 1.1582505702972412,
      "rewards/reward_function": 1.8500674962997437,
      "step": 567
    },
    {
      "completion_length": 193.71429443359375,
      "epoch": 0.32494279176201374,
      "grad_norm": 0.2737533152103424,
      "kl": 0.19255374372005463,
      "learning_rate": 4.268649377789338e-06,
      "loss": 0.0077,
      "reward": 1.2419264316558838,
      "reward_std": 0.8130329847335815,
      "rewards/reward_function": 1.2419264316558838,
      "step": 568
    },
    {
      "completion_length": 218.0357208251953,
      "epoch": 0.3255148741418764,
      "grad_norm": 0.2923491895198822,
      "kl": 0.3034772574901581,
      "learning_rate": 4.2651170351519765e-06,
      "loss": 0.0121,
      "reward": 1.2334129810333252,
      "reward_std": 0.9033427238464355,
      "rewards/reward_function": 1.2334129810333252,
      "step": 569
    },
    {
      "completion_length": 199.82144165039062,
      "epoch": 0.32608695652173914,
      "grad_norm": 0.24816812574863434,
      "kl": 0.15188007056713104,
      "learning_rate": 4.261577651817493e-06,
      "loss": 0.0061,
      "reward": 1.5130923986434937,
      "reward_std": 0.5094484090805054,
      "rewards/reward_function": 1.5130923986434937,
      "step": 570
    },
    {
      "completion_length": 198.75001525878906,
      "epoch": 0.3266590389016018,
      "grad_norm": 0.2805885970592499,
      "kl": 0.2013147473335266,
      "learning_rate": 4.258031241903778e-06,
      "loss": 0.0081,
      "reward": 1.7822909355163574,
      "reward_std": 1.0701515674591064,
      "rewards/reward_function": 1.7822909355163574,
      "step": 571
    },
    {
      "completion_length": 205.92857360839844,
      "epoch": 0.32723112128146453,
      "grad_norm": 0.2870146930217743,
      "kl": 0.19177141785621643,
      "learning_rate": 4.254477819556746e-06,
      "loss": 0.0077,
      "reward": 2.19626784324646,
      "reward_std": 0.9277130365371704,
      "rewards/reward_function": 2.19626784324646,
      "step": 572
    },
    {
      "completion_length": 192.3928680419922,
      "epoch": 0.32780320366132726,
      "grad_norm": 0.3391956686973572,
      "kl": 0.22700752317905426,
      "learning_rate": 4.250917398950286e-06,
      "loss": 0.0091,
      "reward": 1.5779963731765747,
      "reward_std": 0.7131573557853699,
      "rewards/reward_function": 1.5779963731765747,
      "step": 573
    },
    {
      "completion_length": 169.85714721679688,
      "epoch": 0.32837528604118993,
      "grad_norm": 0.3558005094528198,
      "kl": 0.33715400099754333,
      "learning_rate": 4.2473499942862e-06,
      "loss": 0.0135,
      "reward": 1.771531105041504,
      "reward_std": 0.9006665945053101,
      "rewards/reward_function": 1.771531105041504,
      "step": 574
    },
    {
      "completion_length": 185.35714721679688,
      "epoch": 0.32894736842105265,
      "grad_norm": 0.2744128108024597,
      "kl": 0.24745598435401917,
      "learning_rate": 4.243775619794147e-06,
      "loss": 0.0099,
      "reward": 1.6605247259140015,
      "reward_std": 0.8384236693382263,
      "rewards/reward_function": 1.6605247259140015,
      "step": 575
    },
    {
      "completion_length": 188.3928680419922,
      "epoch": 0.3295194508009153,
      "grad_norm": 0.30438563227653503,
      "kl": 0.20558947324752808,
      "learning_rate": 4.24019428973159e-06,
      "loss": 0.0082,
      "reward": 1.55459463596344,
      "reward_std": 1.006606936454773,
      "rewards/reward_function": 1.55459463596344,
      "step": 576
    },
    {
      "completion_length": 198.71429443359375,
      "epoch": 0.33009153318077805,
      "grad_norm": 0.28016987442970276,
      "kl": 0.19538766145706177,
      "learning_rate": 4.2366060183837346e-06,
      "loss": 0.0078,
      "reward": 2.200742483139038,
      "reward_std": 0.8554632663726807,
      "rewards/reward_function": 2.200742483139038,
      "step": 577
    },
    {
      "completion_length": 190.92857360839844,
      "epoch": 0.3306636155606407,
      "grad_norm": 0.8124709725379944,
      "kl": 0.8857913017272949,
      "learning_rate": 4.233010820063473e-06,
      "loss": 0.0354,
      "reward": 1.8792247772216797,
      "reward_std": 0.8827163577079773,
      "rewards/reward_function": 1.8792247772216797,
      "step": 578
    },
    {
      "completion_length": 175.0357208251953,
      "epoch": 0.33123569794050345,
      "grad_norm": 1.5960856676101685,
      "kl": 0.3719918131828308,
      "learning_rate": 4.2294087091113305e-06,
      "loss": 0.0149,
      "reward": 1.6882226467132568,
      "reward_std": 0.9780197143554688,
      "rewards/reward_function": 1.6882226467132568,
      "step": 579
    },
    {
      "completion_length": 185.50001525878906,
      "epoch": 0.3318077803203661,
      "grad_norm": 0.27441850304603577,
      "kl": 0.3236770033836365,
      "learning_rate": 4.225799699895402e-06,
      "loss": 0.0129,
      "reward": 1.5982180833816528,
      "reward_std": 0.5485815405845642,
      "rewards/reward_function": 1.5982180833816528,
      "step": 580
    },
    {
      "completion_length": 197.46429443359375,
      "epoch": 0.33237986270022885,
      "grad_norm": 0.35700538754463196,
      "kl": 0.2819240987300873,
      "learning_rate": 4.222183806811302e-06,
      "loss": 0.0113,
      "reward": 1.8875523805618286,
      "reward_std": 0.6303904056549072,
      "rewards/reward_function": 1.8875523805618286,
      "step": 581
    },
    {
      "completion_length": 196.67857360839844,
      "epoch": 0.3329519450800915,
      "grad_norm": 0.2776491940021515,
      "kl": 0.17391730844974518,
      "learning_rate": 4.218561044282099e-06,
      "loss": 0.007,
      "reward": 2.177502155303955,
      "reward_std": 0.745019793510437,
      "rewards/reward_function": 2.177502155303955,
      "step": 582
    },
    {
      "completion_length": 158.46429443359375,
      "epoch": 0.33352402745995424,
      "grad_norm": 0.27022600173950195,
      "kl": 0.1805935800075531,
      "learning_rate": 4.214931426758267e-06,
      "loss": 0.0072,
      "reward": 1.5716899633407593,
      "reward_std": 0.5642725825309753,
      "rewards/reward_function": 1.5716899633407593,
      "step": 583
    },
    {
      "completion_length": 171.35714721679688,
      "epoch": 0.3340961098398169,
      "grad_norm": 0.26056787371635437,
      "kl": 0.16571705043315887,
      "learning_rate": 4.211294968717618e-06,
      "loss": 0.0066,
      "reward": 1.5187908411026,
      "reward_std": 0.8410926461219788,
      "rewards/reward_function": 1.5187908411026,
      "step": 584
    },
    {
      "completion_length": 210.92857360839844,
      "epoch": 0.33466819221967964,
      "grad_norm": 4.089920520782471,
      "kl": 1.0158218145370483,
      "learning_rate": 4.207651684665256e-06,
      "loss": 0.0406,
      "reward": 1.3566601276397705,
      "reward_std": 0.791976273059845,
      "rewards/reward_function": 1.3566601276397705,
      "step": 585
    },
    {
      "completion_length": 177.7857208251953,
      "epoch": 0.3352402745995423,
      "grad_norm": 0.20849338173866272,
      "kl": 0.1832217574119568,
      "learning_rate": 4.2040015891335056e-06,
      "loss": 0.0073,
      "reward": 1.3375474214553833,
      "reward_std": 0.1537579596042633,
      "rewards/reward_function": 1.3375474214553833,
      "step": 586
    },
    {
      "completion_length": 193.10714721679688,
      "epoch": 0.33581235697940504,
      "grad_norm": 0.46408316493034363,
      "kl": 0.3071887195110321,
      "learning_rate": 4.200344696681866e-06,
      "loss": 0.0123,
      "reward": 1.5787081718444824,
      "reward_std": 0.5134558081626892,
      "rewards/reward_function": 1.5787081718444824,
      "step": 587
    },
    {
      "completion_length": 218.8928680419922,
      "epoch": 0.33638443935926776,
      "grad_norm": 0.30167487263679504,
      "kl": 0.19169345498085022,
      "learning_rate": 4.1966810218969465e-06,
      "loss": 0.0077,
      "reward": 2.4364306926727295,
      "reward_std": 0.8219931721687317,
      "rewards/reward_function": 2.4364306926727295,
      "step": 588
    },
    {
      "completion_length": 215.42857360839844,
      "epoch": 0.33695652173913043,
      "grad_norm": 0.7845038175582886,
      "kl": 0.7202792763710022,
      "learning_rate": 4.193010579392408e-06,
      "loss": 0.0288,
      "reward": 1.6498111486434937,
      "reward_std": 1.115336298942566,
      "rewards/reward_function": 1.6498111486434937,
      "step": 589
    },
    {
      "completion_length": 247.50001525878906,
      "epoch": 0.33752860411899316,
      "grad_norm": 0.22792667150497437,
      "kl": 0.15924035012722015,
      "learning_rate": 4.18933338380891e-06,
      "loss": 0.0064,
      "reward": 1.6592656373977661,
      "reward_std": 0.7952954173088074,
      "rewards/reward_function": 1.6592656373977661,
      "step": 590
    },
    {
      "completion_length": 187.00001525878906,
      "epoch": 0.33810068649885583,
      "grad_norm": 0.48453593254089355,
      "kl": 0.41090357303619385,
      "learning_rate": 4.185649449814046e-06,
      "loss": 0.0164,
      "reward": 1.770067811012268,
      "reward_std": 0.7428638339042664,
      "rewards/reward_function": 1.770067811012268,
      "step": 591
    },
    {
      "completion_length": 201.32144165039062,
      "epoch": 0.33867276887871856,
      "grad_norm": 0.2603423595428467,
      "kl": 0.1748545616865158,
      "learning_rate": 4.181958792102286e-06,
      "loss": 0.007,
      "reward": 1.9438817501068115,
      "reward_std": 0.9289571642875671,
      "rewards/reward_function": 1.9438817501068115,
      "step": 592
    },
    {
      "completion_length": 184.60714721679688,
      "epoch": 0.3392448512585812,
      "grad_norm": 0.2568473815917969,
      "kl": 0.16925367712974548,
      "learning_rate": 4.178261425394926e-06,
      "loss": 0.0068,
      "reward": 2.095613956451416,
      "reward_std": 1.247373104095459,
      "rewards/reward_function": 2.095613956451416,
      "step": 593
    },
    {
      "completion_length": 223.10714721679688,
      "epoch": 0.33981693363844395,
      "grad_norm": 0.3720371425151825,
      "kl": 0.33519360423088074,
      "learning_rate": 4.174557364440016e-06,
      "loss": 0.0134,
      "reward": 1.702874779701233,
      "reward_std": 1.181618332862854,
      "rewards/reward_function": 1.702874779701233,
      "step": 594
    },
    {
      "completion_length": 203.50001525878906,
      "epoch": 0.3403890160183066,
      "grad_norm": 0.27401578426361084,
      "kl": 0.1973544955253601,
      "learning_rate": 4.170846624012311e-06,
      "loss": 0.0079,
      "reward": 1.1145262718200684,
      "reward_std": 0.7578732371330261,
      "rewards/reward_function": 1.1145262718200684,
      "step": 595
    },
    {
      "completion_length": 202.35714721679688,
      "epoch": 0.34096109839816935,
      "grad_norm": 0.2664780020713806,
      "kl": 0.19350631535053253,
      "learning_rate": 4.16712921891321e-06,
      "loss": 0.0077,
      "reward": 1.474770426750183,
      "reward_std": 0.5854412317276001,
      "rewards/reward_function": 1.474770426750183,
      "step": 596
    },
    {
      "completion_length": 227.35714721679688,
      "epoch": 0.341533180778032,
      "grad_norm": 0.2905801236629486,
      "kl": 0.32028836011886597,
      "learning_rate": 4.163405163970694e-06,
      "loss": 0.0128,
      "reward": 1.668544888496399,
      "reward_std": 1.1340383291244507,
      "rewards/reward_function": 1.668544888496399,
      "step": 597
    },
    {
      "completion_length": 201.25001525878906,
      "epoch": 0.34210526315789475,
      "grad_norm": 0.22834530472755432,
      "kl": 0.14589455723762512,
      "learning_rate": 4.159674474039269e-06,
      "loss": 0.0058,
      "reward": 1.8055425882339478,
      "reward_std": 0.690442681312561,
      "rewards/reward_function": 1.8055425882339478,
      "step": 598
    },
    {
      "completion_length": 220.25001525878906,
      "epoch": 0.3426773455377574,
      "grad_norm": 3.076378345489502,
      "kl": 0.9852467179298401,
      "learning_rate": 4.155937163999909e-06,
      "loss": 0.0394,
      "reward": 1.9638853073120117,
      "reward_std": 0.8216890692710876,
      "rewards/reward_function": 1.9638853073120117,
      "step": 599
    },
    {
      "completion_length": 204.0357208251953,
      "epoch": 0.34324942791762014,
      "grad_norm": 0.8164689540863037,
      "kl": 0.5556936264038086,
      "learning_rate": 4.152193248759993e-06,
      "loss": 0.0222,
      "reward": 2.231685161590576,
      "reward_std": 0.7327955961227417,
      "rewards/reward_function": 2.231685161590576,
      "step": 600
    },
    {
      "completion_length": 205.07144165039062,
      "epoch": 0.3438215102974828,
      "grad_norm": 0.2612718641757965,
      "kl": 0.328238844871521,
      "learning_rate": 4.148442743253244e-06,
      "loss": 0.0131,
      "reward": 1.6911845207214355,
      "reward_std": 0.499217689037323,
      "rewards/reward_function": 1.6911845207214355,
      "step": 601
    },
    {
      "completion_length": 219.96429443359375,
      "epoch": 0.34439359267734554,
      "grad_norm": 0.26409220695495605,
      "kl": 0.41145840287208557,
      "learning_rate": 4.1446856624396755e-06,
      "loss": 0.0165,
      "reward": 2.5704450607299805,
      "reward_std": 0.7239485383033752,
      "rewards/reward_function": 2.5704450607299805,
      "step": 602
    },
    {
      "completion_length": 191.21429443359375,
      "epoch": 0.34496567505720827,
      "grad_norm": 0.3284524977207184,
      "kl": 0.21505212783813477,
      "learning_rate": 4.1409220213055305e-06,
      "loss": 0.0086,
      "reward": 2.0786616802215576,
      "reward_std": 0.6159040331840515,
      "rewards/reward_function": 2.0786616802215576,
      "step": 603
    },
    {
      "completion_length": 237.32144165039062,
      "epoch": 0.34553775743707094,
      "grad_norm": 0.2789163291454315,
      "kl": 0.3244093954563141,
      "learning_rate": 4.137151834863213e-06,
      "loss": 0.013,
      "reward": 1.456344485282898,
      "reward_std": 0.9812706708908081,
      "rewards/reward_function": 1.456344485282898,
      "step": 604
    },
    {
      "completion_length": 189.8928680419922,
      "epoch": 0.34610983981693366,
      "grad_norm": 0.40599554777145386,
      "kl": 0.24549545347690582,
      "learning_rate": 4.133375118151242e-06,
      "loss": 0.0098,
      "reward": 1.8265905380249023,
      "reward_std": 1.122491717338562,
      "rewards/reward_function": 1.8265905380249023,
      "step": 605
    },
    {
      "completion_length": 187.8928680419922,
      "epoch": 0.34668192219679633,
      "grad_norm": 0.25575581192970276,
      "kl": 0.22286048531532288,
      "learning_rate": 4.129591886234181e-06,
      "loss": 0.0089,
      "reward": 1.9546918869018555,
      "reward_std": 0.7357393503189087,
      "rewards/reward_function": 1.9546918869018555,
      "step": 606
    },
    {
      "completion_length": 195.1428680419922,
      "epoch": 0.34725400457665906,
      "grad_norm": 0.25075671076774597,
      "kl": 0.2029861956834793,
      "learning_rate": 4.125802154202581e-06,
      "loss": 0.0081,
      "reward": 1.773512840270996,
      "reward_std": 1.1283472776412964,
      "rewards/reward_function": 1.773512840270996,
      "step": 607
    },
    {
      "completion_length": 177.00001525878906,
      "epoch": 0.34782608695652173,
      "grad_norm": 12.859251022338867,
      "kl": 0.7368335127830505,
      "learning_rate": 4.122005937172921e-06,
      "loss": 0.0295,
      "reward": 1.3016576766967773,
      "reward_std": 0.7257692813873291,
      "rewards/reward_function": 1.3016576766967773,
      "step": 608
    },
    {
      "completion_length": 198.85714721679688,
      "epoch": 0.34839816933638446,
      "grad_norm": 0.44834500551223755,
      "kl": 0.17561782896518707,
      "learning_rate": 4.11820325028755e-06,
      "loss": 0.007,
      "reward": 2.119173049926758,
      "reward_std": 0.9489991068840027,
      "rewards/reward_function": 2.119173049926758,
      "step": 609
    },
    {
      "completion_length": 194.57144165039062,
      "epoch": 0.3489702517162471,
      "grad_norm": 0.3266267776489258,
      "kl": 0.2239949256181717,
      "learning_rate": 4.11439410871462e-06,
      "loss": 0.009,
      "reward": 2.4826650619506836,
      "reward_std": 0.3428305983543396,
      "rewards/reward_function": 2.4826650619506836,
      "step": 610
    },
    {
      "completion_length": 191.2857208251953,
      "epoch": 0.34954233409610985,
      "grad_norm": 2.448491096496582,
      "kl": 0.6281548738479614,
      "learning_rate": 4.110578527648032e-06,
      "loss": 0.0251,
      "reward": 1.8479533195495605,
      "reward_std": 0.9500001668930054,
      "rewards/reward_function": 1.8479533195495605,
      "step": 611
    },
    {
      "completion_length": 194.67857360839844,
      "epoch": 0.3501144164759725,
      "grad_norm": 0.8140442371368408,
      "kl": 0.4954872727394104,
      "learning_rate": 4.106756522307371e-06,
      "loss": 0.0198,
      "reward": 1.893096685409546,
      "reward_std": 0.6918936967849731,
      "rewards/reward_function": 1.893096685409546,
      "step": 612
    },
    {
      "completion_length": 214.25001525878906,
      "epoch": 0.35068649885583525,
      "grad_norm": 0.2937369644641876,
      "kl": 0.19559665024280548,
      "learning_rate": 4.102928107937849e-06,
      "loss": 0.0078,
      "reward": 2.0402753353118896,
      "reward_std": 0.7381238341331482,
      "rewards/reward_function": 2.0402753353118896,
      "step": 613
    },
    {
      "completion_length": 208.25001525878906,
      "epoch": 0.3512585812356979,
      "grad_norm": 2.542306661605835,
      "kl": 0.45693933963775635,
      "learning_rate": 4.099093299810241e-06,
      "loss": 0.0183,
      "reward": 1.523569941520691,
      "reward_std": 0.8933788537979126,
      "rewards/reward_function": 1.523569941520691,
      "step": 614
    },
    {
      "completion_length": 197.42857360839844,
      "epoch": 0.35183066361556065,
      "grad_norm": 0.26279181241989136,
      "kl": 0.15862885117530823,
      "learning_rate": 4.095252113220827e-06,
      "loss": 0.0063,
      "reward": 1.832460641860962,
      "reward_std": 0.5998427867889404,
      "rewards/reward_function": 1.832460641860962,
      "step": 615
    },
    {
      "completion_length": 207.85714721679688,
      "epoch": 0.3524027459954233,
      "grad_norm": 0.2624010145664215,
      "kl": 0.18448983132839203,
      "learning_rate": 4.091404563491328e-06,
      "loss": 0.0074,
      "reward": 2.0088858604431152,
      "reward_std": 0.6426089406013489,
      "rewards/reward_function": 2.0088858604431152,
      "step": 616
    },
    {
      "completion_length": 226.0357208251953,
      "epoch": 0.35297482837528604,
      "grad_norm": 0.3694618344306946,
      "kl": 0.48400843143463135,
      "learning_rate": 4.087550665968846e-06,
      "loss": 0.0194,
      "reward": 1.2914985418319702,
      "reward_std": 0.58961421251297,
      "rewards/reward_function": 1.2914985418319702,
      "step": 617
    },
    {
      "completion_length": 185.42857360839844,
      "epoch": 0.35354691075514877,
      "grad_norm": 0.26571714878082275,
      "kl": 0.19280599057674408,
      "learning_rate": 4.083690436025805e-06,
      "loss": 0.0077,
      "reward": 1.7674458026885986,
      "reward_std": 0.7399384379386902,
      "rewards/reward_function": 1.7674458026885986,
      "step": 618
    },
    {
      "completion_length": 193.25001525878906,
      "epoch": 0.35411899313501144,
      "grad_norm": 0.319059818983078,
      "kl": 0.24422836303710938,
      "learning_rate": 4.079823889059884e-06,
      "loss": 0.0098,
      "reward": 1.9831159114837646,
      "reward_std": 0.6494472026824951,
      "rewards/reward_function": 1.9831159114837646,
      "step": 619
    },
    {
      "completion_length": 195.21429443359375,
      "epoch": 0.35469107551487417,
      "grad_norm": 6.096250534057617,
      "kl": 1.1326661109924316,
      "learning_rate": 4.075951040493965e-06,
      "loss": 0.0453,
      "reward": 1.7142287492752075,
      "reward_std": 0.809895396232605,
      "rewards/reward_function": 1.7142287492752075,
      "step": 620
    },
    {
      "completion_length": 191.32144165039062,
      "epoch": 0.35526315789473684,
      "grad_norm": 0.2512000799179077,
      "kl": 0.23059304058551788,
      "learning_rate": 4.07207190577606e-06,
      "loss": 0.0092,
      "reward": 2.1531453132629395,
      "reward_std": 0.518380880355835,
      "rewards/reward_function": 2.1531453132629395,
      "step": 621
    },
    {
      "completion_length": 213.8928680419922,
      "epoch": 0.35583524027459956,
      "grad_norm": 0.2778318226337433,
      "kl": 0.23870569467544556,
      "learning_rate": 4.068186500379259e-06,
      "loss": 0.0095,
      "reward": 2.469891309738159,
      "reward_std": 0.817742645740509,
      "rewards/reward_function": 2.469891309738159,
      "step": 622
    },
    {
      "completion_length": 206.67857360839844,
      "epoch": 0.35640732265446223,
      "grad_norm": 0.25437307357788086,
      "kl": 0.17951247096061707,
      "learning_rate": 4.0642948398016614e-06,
      "loss": 0.0072,
      "reward": 2.0408010482788086,
      "reward_std": 0.729115903377533,
      "rewards/reward_function": 2.0408010482788086,
      "step": 623
    },
    {
      "completion_length": 189.0357208251953,
      "epoch": 0.35697940503432496,
      "grad_norm": 0.47880351543426514,
      "kl": 0.2051168829202652,
      "learning_rate": 4.060396939566323e-06,
      "loss": 0.0082,
      "reward": 2.0360615253448486,
      "reward_std": 0.8716009855270386,
      "rewards/reward_function": 2.0360615253448486,
      "step": 624
    },
    {
      "completion_length": 196.67857360839844,
      "epoch": 0.35755148741418763,
      "grad_norm": 0.29003071784973145,
      "kl": 0.25710418820381165,
      "learning_rate": 4.056492815221181e-06,
      "loss": 0.0103,
      "reward": 2.0614593029022217,
      "reward_std": 0.4008675515651703,
      "rewards/reward_function": 2.0614593029022217,
      "step": 625
    },
    {
      "completion_length": 233.57144165039062,
      "epoch": 0.35812356979405036,
      "grad_norm": 0.24783723056316376,
      "kl": 0.17851608991622925,
      "learning_rate": 4.052582482339004e-06,
      "loss": 0.0071,
      "reward": 2.2535486221313477,
      "reward_std": 0.9281532168388367,
      "rewards/reward_function": 2.2535486221313477,
      "step": 626
    },
    {
      "completion_length": 255.10714721679688,
      "epoch": 0.358695652173913,
      "grad_norm": 0.49713099002838135,
      "kl": 0.45880746841430664,
      "learning_rate": 4.048665956517326e-06,
      "loss": 0.0184,
      "reward": 1.5785937309265137,
      "reward_std": 0.7202542424201965,
      "rewards/reward_function": 1.5785937309265137,
      "step": 627
    },
    {
      "completion_length": 198.17857360839844,
      "epoch": 0.35926773455377575,
      "grad_norm": 0.28046709299087524,
      "kl": 0.23946906626224518,
      "learning_rate": 4.044743253378378e-06,
      "loss": 0.0096,
      "reward": 1.762255311012268,
      "reward_std": 0.8446600437164307,
      "rewards/reward_function": 1.762255311012268,
      "step": 628
    },
    {
      "completion_length": 187.0357208251953,
      "epoch": 0.3598398169336384,
      "grad_norm": 0.24804049730300903,
      "kl": 0.17490297555923462,
      "learning_rate": 4.040814388569036e-06,
      "loss": 0.007,
      "reward": 1.7905184030532837,
      "reward_std": 0.8591800928115845,
      "rewards/reward_function": 1.7905184030532837,
      "step": 629
    },
    {
      "completion_length": 204.35714721679688,
      "epoch": 0.36041189931350115,
      "grad_norm": 0.2850884795188904,
      "kl": 0.20996741950511932,
      "learning_rate": 4.036879377760753e-06,
      "loss": 0.0084,
      "reward": 2.490939140319824,
      "reward_std": 0.7816380262374878,
      "rewards/reward_function": 2.490939140319824,
      "step": 630
    },
    {
      "completion_length": 187.6428680419922,
      "epoch": 0.3609839816933638,
      "grad_norm": 0.2541449964046478,
      "kl": 0.18554897606372833,
      "learning_rate": 4.032938236649495e-06,
      "loss": 0.0074,
      "reward": 2.129103183746338,
      "reward_std": 0.6847583651542664,
      "rewards/reward_function": 2.129103183746338,
      "step": 631
    },
    {
      "completion_length": 217.7857208251953,
      "epoch": 0.36155606407322655,
      "grad_norm": 0.6070631146430969,
      "kl": 1.0106768608093262,
      "learning_rate": 4.028990980955682e-06,
      "loss": 0.0404,
      "reward": 1.7437222003936768,
      "reward_std": 0.8904696106910706,
      "rewards/reward_function": 1.7437222003936768,
      "step": 632
    },
    {
      "completion_length": 197.85714721679688,
      "epoch": 0.3621281464530893,
      "grad_norm": 0.3074342608451843,
      "kl": 0.20284965634346008,
      "learning_rate": 4.025037626424123e-06,
      "loss": 0.0081,
      "reward": 1.7059260606765747,
      "reward_std": 0.7736976146697998,
      "rewards/reward_function": 1.7059260606765747,
      "step": 633
    },
    {
      "completion_length": 224.75001525878906,
      "epoch": 0.36270022883295194,
      "grad_norm": 0.25279363989830017,
      "kl": 0.2062186300754547,
      "learning_rate": 4.021078188823958e-06,
      "loss": 0.0082,
      "reward": 1.8956331014633179,
      "reward_std": 0.9756341576576233,
      "rewards/reward_function": 1.8956331014633179,
      "step": 634
    },
    {
      "completion_length": 224.6785888671875,
      "epoch": 0.36327231121281467,
      "grad_norm": 0.26435866951942444,
      "kl": 0.188995823264122,
      "learning_rate": 4.017112683948584e-06,
      "loss": 0.0076,
      "reward": 1.7795724868774414,
      "reward_std": 0.7143328189849854,
      "rewards/reward_function": 1.7795724868774414,
      "step": 635
    },
    {
      "completion_length": 228.7857208251953,
      "epoch": 0.36384439359267734,
      "grad_norm": 4.344752788543701,
      "kl": 0.6097906231880188,
      "learning_rate": 4.013141127615605e-06,
      "loss": 0.0244,
      "reward": 1.6102728843688965,
      "reward_std": 0.6683823466300964,
      "rewards/reward_function": 1.6102728843688965,
      "step": 636
    },
    {
      "completion_length": 219.3928680419922,
      "epoch": 0.36441647597254007,
      "grad_norm": 0.26563701033592224,
      "kl": 0.24151982367038727,
      "learning_rate": 4.009163535666761e-06,
      "loss": 0.0097,
      "reward": 2.005204916000366,
      "reward_std": 1.0896812677383423,
      "rewards/reward_function": 2.005204916000366,
      "step": 637
    },
    {
      "completion_length": 212.92857360839844,
      "epoch": 0.36498855835240274,
      "grad_norm": 0.4837154448032379,
      "kl": 0.4118029475212097,
      "learning_rate": 4.005179923967866e-06,
      "loss": 0.0165,
      "reward": 2.0830867290496826,
      "reward_std": 1.1566301584243774,
      "rewards/reward_function": 2.0830867290496826,
      "step": 638
    },
    {
      "completion_length": 212.21429443359375,
      "epoch": 0.36556064073226546,
      "grad_norm": 0.2707799971103668,
      "kl": 0.29163846373558044,
      "learning_rate": 4.0011903084087475e-06,
      "loss": 0.0117,
      "reward": 1.478429913520813,
      "reward_std": 0.7829305529594421,
      "rewards/reward_function": 1.478429913520813,
      "step": 639
    },
    {
      "completion_length": 204.17857360839844,
      "epoch": 0.36613272311212813,
      "grad_norm": 0.4379064738750458,
      "kl": 0.3311264216899872,
      "learning_rate": 3.997194704903179e-06,
      "loss": 0.0132,
      "reward": 2.4971847534179688,
      "reward_std": 0.8100430965423584,
      "rewards/reward_function": 2.4971847534179688,
      "step": 640
    },
    {
      "completion_length": 211.8928680419922,
      "epoch": 0.36670480549199086,
      "grad_norm": 0.43897172808647156,
      "kl": 0.2286294847726822,
      "learning_rate": 3.993193129388821e-06,
      "loss": 0.0091,
      "reward": 1.7022594213485718,
      "reward_std": 1.0088887214660645,
      "rewards/reward_function": 1.7022594213485718,
      "step": 641
    },
    {
      "completion_length": 207.7857208251953,
      "epoch": 0.36727688787185353,
      "grad_norm": 0.2787800133228302,
      "kl": 0.4307352602481842,
      "learning_rate": 3.989185597827154e-06,
      "loss": 0.0172,
      "reward": 1.5721333026885986,
      "reward_std": 0.7269337773323059,
      "rewards/reward_function": 1.5721333026885986,
      "step": 642
    },
    {
      "completion_length": 233.6428680419922,
      "epoch": 0.36784897025171626,
      "grad_norm": 0.26246148347854614,
      "kl": 0.17854127287864685,
      "learning_rate": 3.985172126203416e-06,
      "loss": 0.0071,
      "reward": 2.04939341545105,
      "reward_std": 1.0742099285125732,
      "rewards/reward_function": 2.04939341545105,
      "step": 643
    },
    {
      "completion_length": 208.60714721679688,
      "epoch": 0.3684210526315789,
      "grad_norm": 0.24642549455165863,
      "kl": 0.18351276218891144,
      "learning_rate": 3.981152730526538e-06,
      "loss": 0.0073,
      "reward": 1.7066986560821533,
      "reward_std": 0.7294153571128845,
      "rewards/reward_function": 1.7066986560821533,
      "step": 644
    },
    {
      "completion_length": 249.9285888671875,
      "epoch": 0.36899313501144165,
      "grad_norm": 6.433060646057129,
      "kl": 0.8321535587310791,
      "learning_rate": 3.9771274268290815e-06,
      "loss": 0.0333,
      "reward": 1.8751037120819092,
      "reward_std": 0.9464235305786133,
      "rewards/reward_function": 1.8751037120819092,
      "step": 645
    },
    {
      "completion_length": 210.46429443359375,
      "epoch": 0.3695652173913043,
      "grad_norm": 0.3784260153770447,
      "kl": 0.24818110466003418,
      "learning_rate": 3.973096231167175e-06,
      "loss": 0.0099,
      "reward": 1.5845032930374146,
      "reward_std": 0.7141943573951721,
      "rewards/reward_function": 1.5845032930374146,
      "step": 646
    },
    {
      "completion_length": 251.71429443359375,
      "epoch": 0.37013729977116705,
      "grad_norm": 0.23284713923931122,
      "kl": 0.16456693410873413,
      "learning_rate": 3.969059159620448e-06,
      "loss": 0.0066,
      "reward": 1.384604811668396,
      "reward_std": 1.0588010549545288,
      "rewards/reward_function": 1.384604811668396,
      "step": 647
    },
    {
      "completion_length": 217.10714721679688,
      "epoch": 0.3707093821510298,
      "grad_norm": 0.38059890270233154,
      "kl": 0.29684731364250183,
      "learning_rate": 3.965016228291966e-06,
      "loss": 0.0119,
      "reward": 1.4654841423034668,
      "reward_std": 0.9374172687530518,
      "rewards/reward_function": 1.4654841423034668,
      "step": 648
    },
    {
      "completion_length": 213.17857360839844,
      "epoch": 0.37128146453089245,
      "grad_norm": 0.33628755807876587,
      "kl": 0.20166298747062683,
      "learning_rate": 3.96096745330817e-06,
      "loss": 0.0081,
      "reward": 1.982947826385498,
      "reward_std": 0.8843981027603149,
      "rewards/reward_function": 1.982947826385498,
      "step": 649
    },
    {
      "completion_length": 237.1785888671875,
      "epoch": 0.3718535469107552,
      "grad_norm": 0.8167922496795654,
      "kl": 1.0880992412567139,
      "learning_rate": 3.9569128508188115e-06,
      "loss": 0.0435,
      "reward": 1.784766435623169,
      "reward_std": 1.2590712308883667,
      "rewards/reward_function": 1.784766435623169,
      "step": 650
    },
    {
      "completion_length": 210.60714721679688,
      "epoch": 0.37242562929061784,
      "grad_norm": 0.21996662020683289,
      "kl": 0.13114240765571594,
      "learning_rate": 3.952852436996883e-06,
      "loss": 0.0052,
      "reward": 1.195534348487854,
      "reward_std": 0.5913504362106323,
      "rewards/reward_function": 1.195534348487854,
      "step": 651
    },
    {
      "completion_length": 193.60714721679688,
      "epoch": 0.37299771167048057,
      "grad_norm": 0.2940770387649536,
      "kl": 0.18451687693595886,
      "learning_rate": 3.94878622803856e-06,
      "loss": 0.0074,
      "reward": 1.8553900718688965,
      "reward_std": 0.897017776966095,
      "rewards/reward_function": 1.8553900718688965,
      "step": 652
    },
    {
      "completion_length": 244.1428680419922,
      "epoch": 0.37356979405034324,
      "grad_norm": 0.2713499665260315,
      "kl": 0.21694737672805786,
      "learning_rate": 3.944714240163133e-06,
      "loss": 0.0087,
      "reward": 2.025129556655884,
      "reward_std": 0.9075071215629578,
      "rewards/reward_function": 2.025129556655884,
      "step": 653
    },
    {
      "completion_length": 226.10714721679688,
      "epoch": 0.37414187643020597,
      "grad_norm": 3.3872852325439453,
      "kl": 1.100857138633728,
      "learning_rate": 3.940636489612943e-06,
      "loss": 0.044,
      "reward": 1.6748155355453491,
      "reward_std": 0.6643547415733337,
      "rewards/reward_function": 1.6748155355453491,
      "step": 654
    },
    {
      "completion_length": 200.60714721679688,
      "epoch": 0.37471395881006864,
      "grad_norm": 0.24069449305534363,
      "kl": 0.16249100863933563,
      "learning_rate": 3.936552992653318e-06,
      "loss": 0.0065,
      "reward": 1.3329577445983887,
      "reward_std": 0.9509798884391785,
      "rewards/reward_function": 1.3329577445983887,
      "step": 655
    },
    {
      "completion_length": 242.07144165039062,
      "epoch": 0.37528604118993136,
      "grad_norm": 0.28645363450050354,
      "kl": 0.20977945625782013,
      "learning_rate": 3.932463765572506e-06,
      "loss": 0.0084,
      "reward": 1.6209614276885986,
      "reward_std": 0.9286724925041199,
      "rewards/reward_function": 1.6209614276885986,
      "step": 656
    },
    {
      "completion_length": 220.60714721679688,
      "epoch": 0.37585812356979403,
      "grad_norm": 0.21058471500873566,
      "kl": 0.17216728627681732,
      "learning_rate": 3.9283688246816124e-06,
      "loss": 0.0069,
      "reward": 2.0253407955169678,
      "reward_std": 0.7581916451454163,
      "rewards/reward_function": 2.0253407955169678,
      "step": 657
    },
    {
      "completion_length": 234.50001525878906,
      "epoch": 0.37643020594965676,
      "grad_norm": 2.281554937362671,
      "kl": 0.6017587780952454,
      "learning_rate": 3.924268186314532e-06,
      "loss": 0.0241,
      "reward": 1.820309042930603,
      "reward_std": 1.1138577461242676,
      "rewards/reward_function": 1.820309042930603,
      "step": 658
    },
    {
      "completion_length": 217.60714721679688,
      "epoch": 0.37700228832951943,
      "grad_norm": 0.2770625948905945,
      "kl": 0.22586087882518768,
      "learning_rate": 3.92016186682789e-06,
      "loss": 0.009,
      "reward": 1.5649434328079224,
      "reward_std": 0.5673580169677734,
      "rewards/reward_function": 1.5649434328079224,
      "step": 659
    },
    {
      "completion_length": 197.57144165039062,
      "epoch": 0.37757437070938216,
      "grad_norm": 0.3725230395793915,
      "kl": 0.25901246070861816,
      "learning_rate": 3.916049882600967e-06,
      "loss": 0.0104,
      "reward": 1.8410601615905762,
      "reward_std": 0.9864839315414429,
      "rewards/reward_function": 1.8410601615905762,
      "step": 660
    },
    {
      "completion_length": 200.1428680419922,
      "epoch": 0.3781464530892448,
      "grad_norm": 0.3027307093143463,
      "kl": 0.24781212210655212,
      "learning_rate": 3.911932250035644e-06,
      "loss": 0.0099,
      "reward": 1.9625581502914429,
      "reward_std": 0.9949765801429749,
      "rewards/reward_function": 1.9625581502914429,
      "step": 661
    },
    {
      "completion_length": 225.4285888671875,
      "epoch": 0.37871853546910755,
      "grad_norm": 0.35079142451286316,
      "kl": 0.20507346093654633,
      "learning_rate": 3.907808985556329e-06,
      "loss": 0.0082,
      "reward": 1.5609333515167236,
      "reward_std": 0.7410569787025452,
      "rewards/reward_function": 1.5609333515167236,
      "step": 662
    },
    {
      "completion_length": 237.21429443359375,
      "epoch": 0.3792906178489702,
      "grad_norm": 0.23816630244255066,
      "kl": 0.2557298243045807,
      "learning_rate": 3.903680105609893e-06,
      "loss": 0.0102,
      "reward": 2.3530149459838867,
      "reward_std": 0.893742561340332,
      "rewards/reward_function": 2.3530149459838867,
      "step": 663
    },
    {
      "completion_length": 236.71429443359375,
      "epoch": 0.37986270022883295,
      "grad_norm": 0.2719256281852722,
      "kl": 0.1671796292066574,
      "learning_rate": 3.899545626665612e-06,
      "loss": 0.0067,
      "reward": 1.826676368713379,
      "reward_std": 1.282509207725525,
      "rewards/reward_function": 1.826676368713379,
      "step": 664
    },
    {
      "completion_length": 209.42857360839844,
      "epoch": 0.3804347826086957,
      "grad_norm": 0.27346548438072205,
      "kl": 0.17766325175762177,
      "learning_rate": 3.895405565215089e-06,
      "loss": 0.0071,
      "reward": 1.8165245056152344,
      "reward_std": 0.6853259205818176,
      "rewards/reward_function": 1.8165245056152344,
      "step": 665
    },
    {
      "completion_length": 224.6428680419922,
      "epoch": 0.38100686498855835,
      "grad_norm": 0.33367854356765747,
      "kl": 0.6444622278213501,
      "learning_rate": 3.891259937772199e-06,
      "loss": 0.0258,
      "reward": 1.909562587738037,
      "reward_std": 0.9218664765357971,
      "rewards/reward_function": 1.909562587738037,
      "step": 666
    },
    {
      "completion_length": 249.9285888671875,
      "epoch": 0.3815789473684211,
      "grad_norm": 0.2553740441799164,
      "kl": 0.1816992610692978,
      "learning_rate": 3.887108760873013e-06,
      "loss": 0.0073,
      "reward": 2.0966906547546387,
      "reward_std": 0.7900927662849426,
      "rewards/reward_function": 2.0966906547546387,
      "step": 667
    },
    {
      "completion_length": 252.9285888671875,
      "epoch": 0.38215102974828374,
      "grad_norm": 0.2975911498069763,
      "kl": 0.2690715789794922,
      "learning_rate": 3.8829520510757465e-06,
      "loss": 0.0108,
      "reward": 1.9737955331802368,
      "reward_std": 1.0747712850570679,
      "rewards/reward_function": 1.9737955331802368,
      "step": 668
    },
    {
      "completion_length": 231.07144165039062,
      "epoch": 0.38272311212814647,
      "grad_norm": 0.28424936532974243,
      "kl": 0.21339987218379974,
      "learning_rate": 3.878789824960677e-06,
      "loss": 0.0085,
      "reward": 1.4159835577011108,
      "reward_std": 0.9279583096504211,
      "rewards/reward_function": 1.4159835577011108,
      "step": 669
    },
    {
      "completion_length": 235.9285888671875,
      "epoch": 0.38329519450800914,
      "grad_norm": 0.25584158301353455,
      "kl": 0.20358474552631378,
      "learning_rate": 3.874622099130087e-06,
      "loss": 0.0081,
      "reward": 2.5012948513031006,
      "reward_std": 0.6131973266601562,
      "rewards/reward_function": 2.5012948513031006,
      "step": 670
    },
    {
      "completion_length": 231.3928680419922,
      "epoch": 0.38386727688787187,
      "grad_norm": 0.3486650288105011,
      "kl": 0.2898634374141693,
      "learning_rate": 3.8704488902082e-06,
      "loss": 0.0116,
      "reward": 2.188333511352539,
      "reward_std": 0.9149548411369324,
      "rewards/reward_function": 2.188333511352539,
      "step": 671
    },
    {
      "completion_length": 185.17857360839844,
      "epoch": 0.38443935926773454,
      "grad_norm": 0.24133208394050598,
      "kl": 0.17889371514320374,
      "learning_rate": 3.866270214841106e-06,
      "loss": 0.0072,
      "reward": 2.0507311820983887,
      "reward_std": 0.9484045505523682,
      "rewards/reward_function": 2.0507311820983887,
      "step": 672
    },
    {
      "completion_length": 233.50001525878906,
      "epoch": 0.38501144164759726,
      "grad_norm": 0.24835412204265594,
      "kl": 0.17944028973579407,
      "learning_rate": 3.8620860896967e-06,
      "loss": 0.0072,
      "reward": 2.0637056827545166,
      "reward_std": 0.9194315671920776,
      "rewards/reward_function": 2.0637056827545166,
      "step": 673
    },
    {
      "completion_length": 235.82144165039062,
      "epoch": 0.38558352402745993,
      "grad_norm": 0.21759861707687378,
      "kl": 0.18124668300151825,
      "learning_rate": 3.857896531464619e-06,
      "loss": 0.0073,
      "reward": 1.6971584558486938,
      "reward_std": 0.4159383475780487,
      "rewards/reward_function": 1.6971584558486938,
      "step": 674
    },
    {
      "completion_length": 225.50001525878906,
      "epoch": 0.38615560640732266,
      "grad_norm": 0.2733363211154938,
      "kl": 0.23518654704093933,
      "learning_rate": 3.853701556856167e-06,
      "loss": 0.0094,
      "reward": 1.719801902770996,
      "reward_std": 0.8237119913101196,
      "rewards/reward_function": 1.719801902770996,
      "step": 675
    },
    {
      "completion_length": 242.60714721679688,
      "epoch": 0.38672768878718533,
      "grad_norm": 0.22073641419410706,
      "kl": 0.3161342144012451,
      "learning_rate": 3.849501182604255e-06,
      "loss": 0.0126,
      "reward": 1.1738389730453491,
      "reward_std": 0.8403375148773193,
      "rewards/reward_function": 1.1738389730453491,
      "step": 676
    },
    {
      "completion_length": 218.00001525878906,
      "epoch": 0.38729977116704806,
      "grad_norm": 0.22647130489349365,
      "kl": 0.17702455818653107,
      "learning_rate": 3.845295425463332e-06,
      "loss": 0.0071,
      "reward": 1.5450472831726074,
      "reward_std": 0.5170073509216309,
      "rewards/reward_function": 1.5450472831726074,
      "step": 677
    },
    {
      "completion_length": 222.85714721679688,
      "epoch": 0.3878718535469107,
      "grad_norm": 0.34489625692367554,
      "kl": 0.1912798285484314,
      "learning_rate": 3.841084302209318e-06,
      "loss": 0.0077,
      "reward": 1.6208434104919434,
      "reward_std": 0.834653913974762,
      "rewards/reward_function": 1.6208434104919434,
      "step": 678
    },
    {
      "completion_length": 225.2857208251953,
      "epoch": 0.38844393592677345,
      "grad_norm": 0.20181837677955627,
      "kl": 0.21175700426101685,
      "learning_rate": 3.8368678296395375e-06,
      "loss": 0.0085,
      "reward": 1.9691327810287476,
      "reward_std": 0.6879289150238037,
      "rewards/reward_function": 1.9691327810287476,
      "step": 679
    },
    {
      "completion_length": 207.71429443359375,
      "epoch": 0.3890160183066362,
      "grad_norm": 0.2513899505138397,
      "kl": 0.19650425016880035,
      "learning_rate": 3.832646024572652e-06,
      "loss": 0.0079,
      "reward": 2.550473690032959,
      "reward_std": 0.7235797047615051,
      "rewards/reward_function": 2.550473690032959,
      "step": 680
    },
    {
      "completion_length": 239.00001525878906,
      "epoch": 0.38958810068649885,
      "grad_norm": 0.4119032919406891,
      "kl": 0.9075524806976318,
      "learning_rate": 3.828418903848593e-06,
      "loss": 0.0363,
      "reward": 1.2984919548034668,
      "reward_std": 0.8703590631484985,
      "rewards/reward_function": 1.2984919548034668,
      "step": 681
    },
    {
      "completion_length": 217.5357208251953,
      "epoch": 0.3901601830663616,
      "grad_norm": 0.2579589784145355,
      "kl": 0.20498459041118622,
      "learning_rate": 3.8241864843284974e-06,
      "loss": 0.0082,
      "reward": 2.5392773151397705,
      "reward_std": 0.8492514491081238,
      "rewards/reward_function": 2.5392773151397705,
      "step": 682
    },
    {
      "completion_length": 243.00001525878906,
      "epoch": 0.39073226544622425,
      "grad_norm": 0.3180287778377533,
      "kl": 0.5650938153266907,
      "learning_rate": 3.8199487828946334e-06,
      "loss": 0.0226,
      "reward": 2.4467828273773193,
      "reward_std": 0.9577003121376038,
      "rewards/reward_function": 2.4467828273773193,
      "step": 683
    },
    {
      "completion_length": 215.96429443359375,
      "epoch": 0.391304347826087,
      "grad_norm": 0.3490123450756073,
      "kl": 0.1776987463235855,
      "learning_rate": 3.815705816450342e-06,
      "loss": 0.0071,
      "reward": 1.6838477849960327,
      "reward_std": 1.0532121658325195,
      "rewards/reward_function": 1.6838477849960327,
      "step": 684
    },
    {
      "completion_length": 217.42857360839844,
      "epoch": 0.39187643020594964,
      "grad_norm": 0.23919817805290222,
      "kl": 0.1704656481742859,
      "learning_rate": 3.811457601919963e-06,
      "loss": 0.0068,
      "reward": 1.5036202669143677,
      "reward_std": 0.6946762800216675,
      "rewards/reward_function": 1.5036202669143677,
      "step": 685
    },
    {
      "completion_length": 188.00001525878906,
      "epoch": 0.39244851258581237,
      "grad_norm": 0.2311408519744873,
      "kl": 0.15116237103939056,
      "learning_rate": 3.8072041562487714e-06,
      "loss": 0.006,
      "reward": 1.3801332712173462,
      "reward_std": 0.7446973323822021,
      "rewards/reward_function": 1.3801332712173462,
      "step": 686
    },
    {
      "completion_length": 213.25001525878906,
      "epoch": 0.39302059496567504,
      "grad_norm": 0.2608413100242615,
      "kl": 0.19326217472553253,
      "learning_rate": 3.802945496402907e-06,
      "loss": 0.0077,
      "reward": 1.7265125513076782,
      "reward_std": 0.7703563570976257,
      "rewards/reward_function": 1.7265125513076782,
      "step": 687
    },
    {
      "completion_length": 217.10714721679688,
      "epoch": 0.39359267734553777,
      "grad_norm": 0.24922412633895874,
      "kl": 0.22238166630268097,
      "learning_rate": 3.798681639369306e-06,
      "loss": 0.0089,
      "reward": 1.6177706718444824,
      "reward_std": 0.8311880826950073,
      "rewards/reward_function": 1.6177706718444824,
      "step": 688
    },
    {
      "completion_length": 188.46429443359375,
      "epoch": 0.39416475972540044,
      "grad_norm": 0.24554111063480377,
      "kl": 0.22674120962619781,
      "learning_rate": 3.794412602155639e-06,
      "loss": 0.0091,
      "reward": 1.380237102508545,
      "reward_std": 0.8726444840431213,
      "rewards/reward_function": 1.380237102508545,
      "step": 689
    },
    {
      "completion_length": 233.5357208251953,
      "epoch": 0.39473684210526316,
      "grad_norm": 0.3065892457962036,
      "kl": 0.3116830289363861,
      "learning_rate": 3.7901384017902388e-06,
      "loss": 0.0125,
      "reward": 1.751401424407959,
      "reward_std": 0.8988329172134399,
      "rewards/reward_function": 1.751401424407959,
      "step": 690
    },
    {
      "completion_length": 243.5357208251953,
      "epoch": 0.39530892448512583,
      "grad_norm": 0.23195384442806244,
      "kl": 0.1900961846113205,
      "learning_rate": 3.785859055322029e-06,
      "loss": 0.0076,
      "reward": 1.508020043373108,
      "reward_std": 0.8325338959693909,
      "rewards/reward_function": 1.508020043373108,
      "step": 691
    },
    {
      "completion_length": 208.42857360839844,
      "epoch": 0.39588100686498856,
      "grad_norm": 0.2951774001121521,
      "kl": 0.21095408499240875,
      "learning_rate": 3.7815745798204646e-06,
      "loss": 0.0084,
      "reward": 2.012877941131592,
      "reward_std": 0.892473578453064,
      "rewards/reward_function": 2.012877941131592,
      "step": 692
    },
    {
      "completion_length": 241.85714721679688,
      "epoch": 0.39645308924485123,
      "grad_norm": 0.23710289597511292,
      "kl": 0.18371036648750305,
      "learning_rate": 3.7772849923754563e-06,
      "loss": 0.0073,
      "reward": 2.1580281257629395,
      "reward_std": 1.0072904825210571,
      "rewards/reward_function": 2.1580281257629395,
      "step": 693
    },
    {
      "completion_length": 234.1785888671875,
      "epoch": 0.39702517162471396,
      "grad_norm": 0.2680186331272125,
      "kl": 0.170673668384552,
      "learning_rate": 3.772990310097306e-06,
      "loss": 0.0068,
      "reward": 1.9807549715042114,
      "reward_std": 0.8380691409111023,
      "rewards/reward_function": 1.9807549715042114,
      "step": 694
    },
    {
      "completion_length": 196.35714721679688,
      "epoch": 0.3975972540045767,
      "grad_norm": 0.6164718866348267,
      "kl": 0.6658717393875122,
      "learning_rate": 3.7686905501166392e-06,
      "loss": 0.0266,
      "reward": 1.885048270225525,
      "reward_std": 0.8863494396209717,
      "rewards/reward_function": 1.885048270225525,
      "step": 695
    },
    {
      "completion_length": 219.82144165039062,
      "epoch": 0.39816933638443935,
      "grad_norm": 0.24270479381084442,
      "kl": 0.21832159161567688,
      "learning_rate": 3.7643857295843343e-06,
      "loss": 0.0087,
      "reward": 1.8954507112503052,
      "reward_std": 0.5669800043106079,
      "rewards/reward_function": 1.8954507112503052,
      "step": 696
    },
    {
      "completion_length": 213.57144165039062,
      "epoch": 0.3987414187643021,
      "grad_norm": 0.2640223801136017,
      "kl": 0.19495411217212677,
      "learning_rate": 3.7600758656714547e-06,
      "loss": 0.0078,
      "reward": 1.9033384323120117,
      "reward_std": 1.0556011199951172,
      "rewards/reward_function": 1.9033384323120117,
      "step": 697
    },
    {
      "completion_length": 212.42857360839844,
      "epoch": 0.39931350114416475,
      "grad_norm": 0.2715308666229248,
      "kl": 0.19159910082817078,
      "learning_rate": 3.755760975569182e-06,
      "loss": 0.0077,
      "reward": 1.8104861974716187,
      "reward_std": 0.7346107363700867,
      "rewards/reward_function": 1.8104861974716187,
      "step": 698
    },
    {
      "completion_length": 175.35714721679688,
      "epoch": 0.3998855835240275,
      "grad_norm": 0.2949763238430023,
      "kl": 0.16581428050994873,
      "learning_rate": 3.7514410764887453e-06,
      "loss": 0.0066,
      "reward": 1.5021071434020996,
      "reward_std": 0.8160464763641357,
      "rewards/reward_function": 1.5021071434020996,
      "step": 699
    },
    {
      "completion_length": 216.96429443359375,
      "epoch": 0.40045766590389015,
      "grad_norm": 0.23378093540668488,
      "kl": 0.18395628035068512,
      "learning_rate": 3.7471161856613534e-06,
      "loss": 0.0074,
      "reward": 1.5295617580413818,
      "reward_std": 0.6706359386444092,
      "rewards/reward_function": 1.5295617580413818,
      "step": 700
    },
    {
      "completion_length": 249.50001525878906,
      "epoch": 0.4010297482837529,
      "grad_norm": 0.28911879658699036,
      "kl": 0.3021906912326813,
      "learning_rate": 3.7427863203381274e-06,
      "loss": 0.0121,
      "reward": 1.9504244327545166,
      "reward_std": 0.5253058671951294,
      "rewards/reward_function": 1.9504244327545166,
      "step": 701
    },
    {
      "completion_length": 210.1428680419922,
      "epoch": 0.40160183066361554,
      "grad_norm": 0.28643539547920227,
      "kl": 0.21439887583255768,
      "learning_rate": 3.7384514977900297e-06,
      "loss": 0.0086,
      "reward": 1.453407645225525,
      "reward_std": 0.9375320672988892,
      "rewards/reward_function": 1.453407645225525,
      "step": 702
    },
    {
      "completion_length": 195.60714721679688,
      "epoch": 0.40217391304347827,
      "grad_norm": 0.2312176674604416,
      "kl": 0.15952228009700775,
      "learning_rate": 3.7341117353077964e-06,
      "loss": 0.0064,
      "reward": 2.542217969894409,
      "reward_std": 0.7949197292327881,
      "rewards/reward_function": 2.542217969894409,
      "step": 703
    },
    {
      "completion_length": 272.6785888671875,
      "epoch": 0.40274599542334094,
      "grad_norm": 0.20254774391651154,
      "kl": 0.1784723848104477,
      "learning_rate": 3.729767050201868e-06,
      "loss": 0.0071,
      "reward": 2.1988110542297363,
      "reward_std": 0.8086631298065186,
      "rewards/reward_function": 2.1988110542297363,
      "step": 704
    },
    {
      "completion_length": 213.42857360839844,
      "epoch": 0.40331807780320367,
      "grad_norm": 0.2732858657836914,
      "kl": 0.1894611418247223,
      "learning_rate": 3.7254174598023196e-06,
      "loss": 0.0076,
      "reward": 1.6749049425125122,
      "reward_std": 0.9921029806137085,
      "rewards/reward_function": 1.6749049425125122,
      "step": 705
    },
    {
      "completion_length": 197.46429443359375,
      "epoch": 0.40389016018306634,
      "grad_norm": 0.3766961991786957,
      "kl": 0.6461892127990723,
      "learning_rate": 3.721062981458793e-06,
      "loss": 0.0258,
      "reward": 1.8674451112747192,
      "reward_std": 0.790052056312561,
      "rewards/reward_function": 1.8674451112747192,
      "step": 706
    },
    {
      "completion_length": 203.07144165039062,
      "epoch": 0.40446224256292906,
      "grad_norm": 0.4205032289028168,
      "kl": 0.4418928623199463,
      "learning_rate": 3.7167036325404286e-06,
      "loss": 0.0177,
      "reward": 1.9171854257583618,
      "reward_std": 1.1381884813308716,
      "rewards/reward_function": 1.9171854257583618,
      "step": 707
    },
    {
      "completion_length": 240.0357208251953,
      "epoch": 0.40503432494279173,
      "grad_norm": 0.28804492950439453,
      "kl": 0.17648904025554657,
      "learning_rate": 3.712339430435792e-06,
      "loss": 0.0071,
      "reward": 2.798112630844116,
      "reward_std": 0.5961655378341675,
      "rewards/reward_function": 2.798112630844116,
      "step": 708
    },
    {
      "completion_length": 242.8928680419922,
      "epoch": 0.40560640732265446,
      "grad_norm": 0.22408168017864227,
      "kl": 0.19271139800548553,
      "learning_rate": 3.707970392552809e-06,
      "loss": 0.0077,
      "reward": 2.154794216156006,
      "reward_std": 0.9589378833770752,
      "rewards/reward_function": 2.154794216156006,
      "step": 709
    },
    {
      "completion_length": 242.57144165039062,
      "epoch": 0.4061784897025172,
      "grad_norm": 6.743587970733643,
      "kl": 1.0274932384490967,
      "learning_rate": 3.703596536318692e-06,
      "loss": 0.0411,
      "reward": 2.505065441131592,
      "reward_std": 0.9046006798744202,
      "rewards/reward_function": 2.505065441131592,
      "step": 710
    },
    {
      "completion_length": 239.71429443359375,
      "epoch": 0.40675057208237986,
      "grad_norm": 0.23847252130508423,
      "kl": 0.16827194392681122,
      "learning_rate": 3.699217879179877e-06,
      "loss": 0.0067,
      "reward": 2.1518895626068115,
      "reward_std": 0.9817197322845459,
      "rewards/reward_function": 2.1518895626068115,
      "step": 711
    },
    {
      "completion_length": 259.39288330078125,
      "epoch": 0.4073226544622426,
      "grad_norm": 0.2611777186393738,
      "kl": 0.5514611601829529,
      "learning_rate": 3.6948344386019465e-06,
      "loss": 0.0221,
      "reward": 1.9643394947052002,
      "reward_std": 1.1149523258209229,
      "rewards/reward_function": 1.9643394947052002,
      "step": 712
    },
    {
      "completion_length": 246.75001525878906,
      "epoch": 0.40789473684210525,
      "grad_norm": 0.22801145911216736,
      "kl": 0.151764377951622,
      "learning_rate": 3.690446232069563e-06,
      "loss": 0.0061,
      "reward": 1.5196565389633179,
      "reward_std": 0.9226846694946289,
      "rewards/reward_function": 1.5196565389633179,
      "step": 713
    },
    {
      "completion_length": 222.60714721679688,
      "epoch": 0.408466819221968,
      "grad_norm": 1.8622249364852905,
      "kl": 1.0616745948791504,
      "learning_rate": 3.686053277086401e-06,
      "loss": 0.0425,
      "reward": 2.0095224380493164,
      "reward_std": 0.5474222898483276,
      "rewards/reward_function": 2.0095224380493164,
      "step": 714
    },
    {
      "completion_length": 233.35714721679688,
      "epoch": 0.40903890160183065,
      "grad_norm": 0.25133779644966125,
      "kl": 0.20631535351276398,
      "learning_rate": 3.6816555911750756e-06,
      "loss": 0.0083,
      "reward": 1.8979724645614624,
      "reward_std": 0.7549960613250732,
      "rewards/reward_function": 1.8979724645614624,
      "step": 715
    },
    {
      "completion_length": 251.9285888671875,
      "epoch": 0.4096109839816934,
      "grad_norm": 0.2203996181488037,
      "kl": 0.623556911945343,
      "learning_rate": 3.677253191877072e-06,
      "loss": 0.0249,
      "reward": 1.3243978023529053,
      "reward_std": 0.8617923259735107,
      "rewards/reward_function": 1.3243978023529053,
      "step": 716
    },
    {
      "completion_length": 252.1428680419922,
      "epoch": 0.41018306636155605,
      "grad_norm": 0.6164810657501221,
      "kl": 0.7069993615150452,
      "learning_rate": 3.672846096752676e-06,
      "loss": 0.0283,
      "reward": 1.7415151596069336,
      "reward_std": 0.9561216235160828,
      "rewards/reward_function": 1.7415151596069336,
      "step": 717
    },
    {
      "completion_length": 243.4285888671875,
      "epoch": 0.4107551487414188,
      "grad_norm": 0.2239142805337906,
      "kl": 0.17558960616588593,
      "learning_rate": 3.668434323380904e-06,
      "loss": 0.007,
      "reward": 2.6094679832458496,
      "reward_std": 0.7835744619369507,
      "rewards/reward_function": 2.6094679832458496,
      "step": 718
    },
    {
      "completion_length": 274.96429443359375,
      "epoch": 0.41132723112128144,
      "grad_norm": 0.2517474293708801,
      "kl": 0.4084773659706116,
      "learning_rate": 3.6640178893594337e-06,
      "loss": 0.0163,
      "reward": 1.4399574995040894,
      "reward_std": 0.8014026880264282,
      "rewards/reward_function": 1.4399574995040894,
      "step": 719
    },
    {
      "completion_length": 226.6428680419922,
      "epoch": 0.41189931350114417,
      "grad_norm": 0.2986207902431488,
      "kl": 0.22943812608718872,
      "learning_rate": 3.6595968123045333e-06,
      "loss": 0.0092,
      "reward": 1.669092059135437,
      "reward_std": 0.8825855851173401,
      "rewards/reward_function": 1.669092059135437,
      "step": 720
    },
    {
      "completion_length": 286.1785888671875,
      "epoch": 0.41247139588100684,
      "grad_norm": 0.20768776535987854,
      "kl": 0.19462212920188904,
      "learning_rate": 3.6551711098509906e-06,
      "loss": 0.0078,
      "reward": 1.5533854961395264,
      "reward_std": 0.6019328236579895,
      "rewards/reward_function": 1.5533854961395264,
      "step": 721
    },
    {
      "completion_length": 249.00001525878906,
      "epoch": 0.41304347826086957,
      "grad_norm": 0.2585422396659851,
      "kl": 0.17925463616847992,
      "learning_rate": 3.650740799652043e-06,
      "loss": 0.0072,
      "reward": 1.4231020212173462,
      "reward_std": 0.858225405216217,
      "rewards/reward_function": 1.4231020212173462,
      "step": 722
    },
    {
      "completion_length": 256.4285888671875,
      "epoch": 0.41361556064073224,
      "grad_norm": 12.304181098937988,
      "kl": 0.7923781871795654,
      "learning_rate": 3.6463058993793065e-06,
      "loss": 0.0317,
      "reward": 1.9056493043899536,
      "reward_std": 1.0190696716308594,
      "rewards/reward_function": 1.9056493043899536,
      "step": 723
    },
    {
      "completion_length": 254.5357208251953,
      "epoch": 0.41418764302059496,
      "grad_norm": 0.2639178931713104,
      "kl": 0.16692809760570526,
      "learning_rate": 3.641866426722709e-06,
      "loss": 0.0067,
      "reward": 1.382798433303833,
      "reward_std": 0.7465202212333679,
      "rewards/reward_function": 1.382798433303833,
      "step": 724
    },
    {
      "completion_length": 252.46429443359375,
      "epoch": 0.4147597254004577,
      "grad_norm": 0.250370591878891,
      "kl": 0.1730031818151474,
      "learning_rate": 3.637422399390413e-06,
      "loss": 0.0069,
      "reward": 1.6910128593444824,
      "reward_std": 0.9119789600372314,
      "rewards/reward_function": 1.6910128593444824,
      "step": 725
    },
    {
      "completion_length": 263.4285888671875,
      "epoch": 0.41533180778032036,
      "grad_norm": 0.2511601746082306,
      "kl": 0.21901148557662964,
      "learning_rate": 3.6329738351087517e-06,
      "loss": 0.0088,
      "reward": 2.136321783065796,
      "reward_std": 0.7587451338768005,
      "rewards/reward_function": 2.136321783065796,
      "step": 726
    },
    {
      "completion_length": 205.17857360839844,
      "epoch": 0.4159038901601831,
      "grad_norm": 0.2705441117286682,
      "kl": 0.18482579290866852,
      "learning_rate": 3.6285207516221534e-06,
      "loss": 0.0074,
      "reward": 1.604120135307312,
      "reward_std": 0.6820914149284363,
      "rewards/reward_function": 1.604120135307312,
      "step": 727
    },
    {
      "completion_length": 208.6428680419922,
      "epoch": 0.41647597254004576,
      "grad_norm": 0.2791569232940674,
      "kl": 0.17424613237380981,
      "learning_rate": 3.6240631666930726e-06,
      "loss": 0.007,
      "reward": 2.1577811241149902,
      "reward_std": 0.9717882871627808,
      "rewards/reward_function": 2.1577811241149902,
      "step": 728
    },
    {
      "completion_length": 231.2857208251953,
      "epoch": 0.4170480549199085,
      "grad_norm": 0.2080627828836441,
      "kl": 0.16361817717552185,
      "learning_rate": 3.619601098101921e-06,
      "loss": 0.0065,
      "reward": 2.2275819778442383,
      "reward_std": 0.8628280162811279,
      "rewards/reward_function": 2.2275819778442383,
      "step": 729
    },
    {
      "completion_length": 242.0357208251953,
      "epoch": 0.41762013729977115,
      "grad_norm": 0.30850911140441895,
      "kl": 0.24177658557891846,
      "learning_rate": 3.615134563646993e-06,
      "loss": 0.0097,
      "reward": 1.7015833854675293,
      "reward_std": 0.9119086265563965,
      "rewards/reward_function": 1.7015833854675293,
      "step": 730
    },
    {
      "completion_length": 248.57144165039062,
      "epoch": 0.4181922196796339,
      "grad_norm": 0.23755484819412231,
      "kl": 0.2503703236579895,
      "learning_rate": 3.610663581144396e-06,
      "loss": 0.01,
      "reward": 1.9362624883651733,
      "reward_std": 0.9020224213600159,
      "rewards/reward_function": 1.9362624883651733,
      "step": 731
    },
    {
      "completion_length": 257.96429443359375,
      "epoch": 0.41876430205949655,
      "grad_norm": 0.23843024671077728,
      "kl": 0.1530599147081375,
      "learning_rate": 3.6061881684279826e-06,
      "loss": 0.0061,
      "reward": 1.7184710502624512,
      "reward_std": 0.7058342099189758,
      "rewards/reward_function": 1.7184710502624512,
      "step": 732
    },
    {
      "completion_length": 259.3214416503906,
      "epoch": 0.4193363844393593,
      "grad_norm": 0.5380492806434631,
      "kl": 0.4224580228328705,
      "learning_rate": 3.601708343349273e-06,
      "loss": 0.0169,
      "reward": 1.7882471084594727,
      "reward_std": 0.9536249041557312,
      "rewards/reward_function": 1.7882471084594727,
      "step": 733
    },
    {
      "completion_length": 227.71429443359375,
      "epoch": 0.41990846681922195,
      "grad_norm": 0.439045250415802,
      "kl": 0.22927553951740265,
      "learning_rate": 3.59722412377739e-06,
      "loss": 0.0092,
      "reward": 2.2232396602630615,
      "reward_std": 0.6732690334320068,
      "rewards/reward_function": 2.2232396602630615,
      "step": 734
    },
    {
      "completion_length": 269.0714416503906,
      "epoch": 0.4204805491990847,
      "grad_norm": 0.4050872027873993,
      "kl": 0.36509889364242554,
      "learning_rate": 3.5927355275989837e-06,
      "loss": 0.0146,
      "reward": 1.9206374883651733,
      "reward_std": 0.6745644807815552,
      "rewards/reward_function": 1.9206374883651733,
      "step": 735
    },
    {
      "completion_length": 224.07144165039062,
      "epoch": 0.42105263157894735,
      "grad_norm": 0.2305774688720703,
      "kl": 0.17375749349594116,
      "learning_rate": 3.5882425727181625e-06,
      "loss": 0.007,
      "reward": 1.8068482875823975,
      "reward_std": 0.5412890315055847,
      "rewards/reward_function": 1.8068482875823975,
      "step": 736
    },
    {
      "completion_length": 215.82144165039062,
      "epoch": 0.42162471395881007,
      "grad_norm": 0.26255613565444946,
      "kl": 0.1870434433221817,
      "learning_rate": 3.5837452770564203e-06,
      "loss": 0.0075,
      "reward": 1.8318597078323364,
      "reward_std": 0.7981951832771301,
      "rewards/reward_function": 1.8318597078323364,
      "step": 737
    },
    {
      "completion_length": 260.14288330078125,
      "epoch": 0.42219679633867274,
      "grad_norm": 8.623380661010742,
      "kl": 1.5449050664901733,
      "learning_rate": 3.5792436585525663e-06,
      "loss": 0.0618,
      "reward": 2.842902421951294,
      "reward_std": 0.6372544169425964,
      "rewards/reward_function": 2.842902421951294,
      "step": 738
    },
    {
      "completion_length": 271.71429443359375,
      "epoch": 0.42276887871853547,
      "grad_norm": 0.25286486744880676,
      "kl": 0.35261842608451843,
      "learning_rate": 3.5747377351626515e-06,
      "loss": 0.0141,
      "reward": 2.1360678672790527,
      "reward_std": 1.1726957559585571,
      "rewards/reward_function": 2.1360678672790527,
      "step": 739
    },
    {
      "completion_length": 214.07144165039062,
      "epoch": 0.4233409610983982,
      "grad_norm": 0.636835515499115,
      "kl": 0.6138141751289368,
      "learning_rate": 3.570227524859899e-06,
      "loss": 0.0246,
      "reward": 2.2570507526397705,
      "reward_std": 1.0229190587997437,
      "rewards/reward_function": 2.2570507526397705,
      "step": 740
    },
    {
      "completion_length": 232.71429443359375,
      "epoch": 0.42391304347826086,
      "grad_norm": 0.3716529905796051,
      "kl": 0.20880919694900513,
      "learning_rate": 3.565713045634631e-06,
      "loss": 0.0084,
      "reward": 1.5825893878936768,
      "reward_std": 0.8698139786720276,
      "rewards/reward_function": 1.5825893878936768,
      "step": 741
    },
    {
      "completion_length": 241.35714721679688,
      "epoch": 0.4244851258581236,
      "grad_norm": 0.21321436762809753,
      "kl": 0.17649389803409576,
      "learning_rate": 3.5611943154941985e-06,
      "loss": 0.0071,
      "reward": 1.6801096200942993,
      "reward_std": 0.7181106805801392,
      "rewards/reward_function": 1.6801096200942993,
      "step": 742
    },
    {
      "completion_length": 250.00001525878906,
      "epoch": 0.42505720823798626,
      "grad_norm": 0.2170165330171585,
      "kl": 0.14468471705913544,
      "learning_rate": 3.5566713524629065e-06,
      "loss": 0.0058,
      "reward": 1.260148525238037,
      "reward_std": 0.7236912846565247,
      "rewards/reward_function": 1.260148525238037,
      "step": 743
    },
    {
      "completion_length": 230.85714721679688,
      "epoch": 0.425629290617849,
      "grad_norm": 0.23744268715381622,
      "kl": 0.13680163025856018,
      "learning_rate": 3.5521441745819474e-06,
      "loss": 0.0055,
      "reward": 1.772740364074707,
      "reward_std": 0.8227633237838745,
      "rewards/reward_function": 1.772740364074707,
      "step": 744
    },
    {
      "completion_length": 222.96429443359375,
      "epoch": 0.42620137299771166,
      "grad_norm": 0.470373272895813,
      "kl": 0.7580512762069702,
      "learning_rate": 3.5476127999093226e-06,
      "loss": 0.0303,
      "reward": 1.6550624370574951,
      "reward_std": 0.6917431950569153,
      "rewards/reward_function": 1.6550624370574951,
      "step": 745
    },
    {
      "completion_length": 244.4285888671875,
      "epoch": 0.4267734553775744,
      "grad_norm": 0.2764291763305664,
      "kl": 0.19647721946239471,
      "learning_rate": 3.543077246519775e-06,
      "loss": 0.0079,
      "reward": 1.3831202983856201,
      "reward_std": 0.6179050803184509,
      "rewards/reward_function": 1.3831202983856201,
      "step": 746
    },
    {
      "completion_length": 259.89288330078125,
      "epoch": 0.42734553775743706,
      "grad_norm": 0.2579757571220398,
      "kl": 0.3673584759235382,
      "learning_rate": 3.5385375325047167e-06,
      "loss": 0.0147,
      "reward": 1.6873749494552612,
      "reward_std": 1.0640045404434204,
      "rewards/reward_function": 1.6873749494552612,
      "step": 747
    },
    {
      "completion_length": 229.75001525878906,
      "epoch": 0.4279176201372998,
      "grad_norm": 42.88764953613281,
      "kl": 2.6234946250915527,
      "learning_rate": 3.5339936759721537e-06,
      "loss": 0.1049,
      "reward": 2.420093536376953,
      "reward_std": 0.6893568634986877,
      "rewards/reward_function": 2.420093536376953,
      "step": 748
    },
    {
      "completion_length": 278.1071472167969,
      "epoch": 0.42848970251716245,
      "grad_norm": 0.22933483123779297,
      "kl": 0.1674114316701889,
      "learning_rate": 3.529445695046616e-06,
      "loss": 0.0067,
      "reward": 1.5707955360412598,
      "reward_std": 0.4141789376735687,
      "rewards/reward_function": 1.5707955360412598,
      "step": 749
    },
    {
      "completion_length": 239.25001525878906,
      "epoch": 0.4290617848970252,
      "grad_norm": 0.20849673449993134,
      "kl": 0.1520286202430725,
      "learning_rate": 3.5248936078690877e-06,
      "loss": 0.0061,
      "reward": 1.9608805179595947,
      "reward_std": 0.615058183670044,
      "rewards/reward_function": 1.9608805179595947,
      "step": 750
    },
    {
      "completion_length": 225.5357208251953,
      "epoch": 0.42963386727688785,
      "grad_norm": 0.3014717698097229,
      "kl": 0.3480134904384613,
      "learning_rate": 3.5203374325969277e-06,
      "loss": 0.0139,
      "reward": 1.5274405479431152,
      "reward_std": 0.7215219736099243,
      "rewards/reward_function": 1.5274405479431152,
      "step": 751
    },
    {
      "completion_length": 262.0714416503906,
      "epoch": 0.4302059496567506,
      "grad_norm": 0.269380658864975,
      "kl": 0.2372574359178543,
      "learning_rate": 3.5157771874038033e-06,
      "loss": 0.0095,
      "reward": 2.4595696926116943,
      "reward_std": 1.1049879789352417,
      "rewards/reward_function": 2.4595696926116943,
      "step": 752
    },
    {
      "completion_length": 228.7857208251953,
      "epoch": 0.43077803203661325,
      "grad_norm": 0.4329148530960083,
      "kl": 0.25554031133651733,
      "learning_rate": 3.511212890479617e-06,
      "loss": 0.0102,
      "reward": 1.0399068593978882,
      "reward_std": 0.777509331703186,
      "rewards/reward_function": 1.0399068593978882,
      "step": 753
    },
    {
      "completion_length": 223.9285888671875,
      "epoch": 0.43135011441647597,
      "grad_norm": 0.29212796688079834,
      "kl": 0.3754730522632599,
      "learning_rate": 3.506644560030433e-06,
      "loss": 0.015,
      "reward": 1.6361501216888428,
      "reward_std": 0.6424623727798462,
      "rewards/reward_function": 1.6361501216888428,
      "step": 754
    },
    {
      "completion_length": 256.64288330078125,
      "epoch": 0.4319221967963387,
      "grad_norm": 0.3015454113483429,
      "kl": 0.15838973224163055,
      "learning_rate": 3.502072214278401e-06,
      "loss": 0.0063,
      "reward": 2.103397846221924,
      "reward_std": 0.7161799669265747,
      "rewards/reward_function": 2.103397846221924,
      "step": 755
    },
    {
      "completion_length": 234.07144165039062,
      "epoch": 0.43249427917620137,
      "grad_norm": 0.2600444555282593,
      "kl": 0.17976507544517517,
      "learning_rate": 3.4974958714616913e-06,
      "loss": 0.0072,
      "reward": 2.1186509132385254,
      "reward_std": 0.9203479290008545,
      "rewards/reward_function": 2.1186509132385254,
      "step": 756
    },
    {
      "completion_length": 222.07144165039062,
      "epoch": 0.4330663615560641,
      "grad_norm": 0.23599769175052643,
      "kl": 0.16798970103263855,
      "learning_rate": 3.4929155498344147e-06,
      "loss": 0.0067,
      "reward": 2.371927261352539,
      "reward_std": 0.34410974383354187,
      "rewards/reward_function": 2.371927261352539,
      "step": 757
    },
    {
      "completion_length": 280.6071472167969,
      "epoch": 0.43363844393592677,
      "grad_norm": 0.296040415763855,
      "kl": 0.22603096067905426,
      "learning_rate": 3.4883312676665537e-06,
      "loss": 0.009,
      "reward": 1.827903389930725,
      "reward_std": 0.7916952967643738,
      "rewards/reward_function": 1.827903389930725,
      "step": 758
    },
    {
      "completion_length": 231.46429443359375,
      "epoch": 0.4342105263157895,
      "grad_norm": 0.5787628293037415,
      "kl": 0.8546956181526184,
      "learning_rate": 3.4837430432438898e-06,
      "loss": 0.0342,
      "reward": 1.8754651546478271,
      "reward_std": 1.053328514099121,
      "rewards/reward_function": 1.8754651546478271,
      "step": 759
    },
    {
      "completion_length": 206.35714721679688,
      "epoch": 0.43478260869565216,
      "grad_norm": 0.22358635067939758,
      "kl": 0.15060560405254364,
      "learning_rate": 3.4791508948679263e-06,
      "loss": 0.006,
      "reward": 1.460934042930603,
      "reward_std": 0.948421061038971,
      "rewards/reward_function": 1.460934042930603,
      "step": 760
    },
    {
      "completion_length": 272.0714416503906,
      "epoch": 0.4353546910755149,
      "grad_norm": 0.2045428603887558,
      "kl": 0.15791933238506317,
      "learning_rate": 3.4745548408558203e-06,
      "loss": 0.0063,
      "reward": 1.7546720504760742,
      "reward_std": 0.9782888293266296,
      "rewards/reward_function": 1.7546720504760742,
      "step": 761
    },
    {
      "completion_length": 282.5357360839844,
      "epoch": 0.43592677345537756,
      "grad_norm": 0.2469659447669983,
      "kl": 0.25163060426712036,
      "learning_rate": 3.4699548995403094e-06,
      "loss": 0.0101,
      "reward": 1.950549602508545,
      "reward_std": 0.8838918209075928,
      "rewards/reward_function": 1.950549602508545,
      "step": 762
    },
    {
      "completion_length": 277.5357360839844,
      "epoch": 0.4364988558352403,
      "grad_norm": 0.582806408405304,
      "kl": 0.6309739947319031,
      "learning_rate": 3.4653510892696337e-06,
      "loss": 0.0252,
      "reward": 1.9506425857543945,
      "reward_std": 1.0710747241973877,
      "rewards/reward_function": 1.9506425857543945,
      "step": 763
    },
    {
      "completion_length": 231.32144165039062,
      "epoch": 0.43707093821510296,
      "grad_norm": 0.2456943541765213,
      "kl": 0.15794429183006287,
      "learning_rate": 3.460743428407467e-06,
      "loss": 0.0063,
      "reward": 1.9180760383605957,
      "reward_std": 0.525676429271698,
      "rewards/reward_function": 1.9180760383605957,
      "step": 764
    },
    {
      "completion_length": 257.46429443359375,
      "epoch": 0.4376430205949657,
      "grad_norm": 0.368333637714386,
      "kl": 0.2234993875026703,
      "learning_rate": 3.456131935332844e-06,
      "loss": 0.0089,
      "reward": 2.3220367431640625,
      "reward_std": 0.669535756111145,
      "rewards/reward_function": 2.3220367431640625,
      "step": 765
    },
    {
      "completion_length": 239.9285888671875,
      "epoch": 0.43821510297482835,
      "grad_norm": 0.23295268416404724,
      "kl": 0.14906297624111176,
      "learning_rate": 3.451516628440082e-06,
      "loss": 0.006,
      "reward": 1.9432916641235352,
      "reward_std": 0.5700173377990723,
      "rewards/reward_function": 1.9432916641235352,
      "step": 766
    },
    {
      "completion_length": 239.46429443359375,
      "epoch": 0.4387871853546911,
      "grad_norm": 0.3387638032436371,
      "kl": 0.20781998336315155,
      "learning_rate": 3.4468975261387128e-06,
      "loss": 0.0083,
      "reward": 1.7415151596069336,
      "reward_std": 0.8489940166473389,
      "rewards/reward_function": 1.7415151596069336,
      "step": 767
    },
    {
      "completion_length": 241.46429443359375,
      "epoch": 0.43935926773455375,
      "grad_norm": 0.31539860367774963,
      "kl": 0.1881791055202484,
      "learning_rate": 3.4422746468534083e-06,
      "loss": 0.0075,
      "reward": 1.7132985591888428,
      "reward_std": 0.9999175667762756,
      "rewards/reward_function": 1.7132985591888428,
      "step": 768
    },
    {
      "completion_length": 236.2857208251953,
      "epoch": 0.4399313501144165,
      "grad_norm": 0.30903926491737366,
      "kl": 0.3386319577693939,
      "learning_rate": 3.437648009023905e-06,
      "loss": 0.0135,
      "reward": 1.6746222972869873,
      "reward_std": 1.0754754543304443,
      "rewards/reward_function": 1.6746222972869873,
      "step": 769
    },
    {
      "completion_length": 239.75001525878906,
      "epoch": 0.4405034324942792,
      "grad_norm": 0.25569331645965576,
      "kl": 0.2175375521183014,
      "learning_rate": 3.43301763110493e-06,
      "loss": 0.0087,
      "reward": 1.6040056943893433,
      "reward_std": 0.9083433747291565,
      "rewards/reward_function": 1.6040056943893433,
      "step": 770
    },
    {
      "completion_length": 206.0357208251953,
      "epoch": 0.44107551487414187,
      "grad_norm": 0.22300416231155396,
      "kl": 0.19573737680912018,
      "learning_rate": 3.428383531566131e-06,
      "loss": 0.0078,
      "reward": 1.652726650238037,
      "reward_std": 0.8850477933883667,
      "rewards/reward_function": 1.652726650238037,
      "step": 771
    },
    {
      "completion_length": 297.6071472167969,
      "epoch": 0.4416475972540046,
      "grad_norm": 0.20561042428016663,
      "kl": 0.24422447383403778,
      "learning_rate": 3.4237457288920004e-06,
      "loss": 0.0098,
      "reward": 2.265882730484009,
      "reward_std": 0.7131860256195068,
      "rewards/reward_function": 2.265882730484009,
      "step": 772
    },
    {
      "completion_length": 246.00001525878906,
      "epoch": 0.44221967963386727,
      "grad_norm": 0.236515611410141,
      "kl": 0.26431792974472046,
      "learning_rate": 3.4191042415818003e-06,
      "loss": 0.0106,
      "reward": 1.7923214435577393,
      "reward_std": 1.0022025108337402,
      "rewards/reward_function": 1.7923214435577393,
      "step": 773
    },
    {
      "completion_length": 246.96429443359375,
      "epoch": 0.44279176201373,
      "grad_norm": 0.24523232877254486,
      "kl": 0.22163626551628113,
      "learning_rate": 3.414459088149491e-06,
      "loss": 0.0089,
      "reward": 1.5057199001312256,
      "reward_std": 0.8323166370391846,
      "rewards/reward_function": 1.5057199001312256,
      "step": 774
    },
    {
      "completion_length": 226.3928680419922,
      "epoch": 0.44336384439359267,
      "grad_norm": 0.2899928092956543,
      "kl": 0.1877797245979309,
      "learning_rate": 3.409810287123656e-06,
      "loss": 0.0075,
      "reward": 1.8672019243240356,
      "reward_std": 0.41747215390205383,
      "rewards/reward_function": 1.8672019243240356,
      "step": 775
    },
    {
      "completion_length": 305.1785888671875,
      "epoch": 0.4439359267734554,
      "grad_norm": 0.2537756860256195,
      "kl": 0.3523102104663849,
      "learning_rate": 3.4051578570474283e-06,
      "loss": 0.0141,
      "reward": 2.315948486328125,
      "reward_std": 0.9072524309158325,
      "rewards/reward_function": 2.315948486328125,
      "step": 776
    },
    {
      "completion_length": 253.1428680419922,
      "epoch": 0.44450800915331806,
      "grad_norm": 0.2657868564128876,
      "kl": 0.3079828917980194,
      "learning_rate": 3.4005018164784166e-06,
      "loss": 0.0123,
      "reward": 1.457331895828247,
      "reward_std": 0.6249231696128845,
      "rewards/reward_function": 1.457331895828247,
      "step": 777
    },
    {
      "completion_length": 217.3928680419922,
      "epoch": 0.4450800915331808,
      "grad_norm": 0.7627407312393188,
      "kl": 0.662860631942749,
      "learning_rate": 3.39584218398863e-06,
      "loss": 0.0265,
      "reward": 1.3036037683486938,
      "reward_std": 1.014625072479248,
      "rewards/reward_function": 1.3036037683486938,
      "step": 778
    },
    {
      "completion_length": 246.6428680419922,
      "epoch": 0.44565217391304346,
      "grad_norm": 0.25042200088500977,
      "kl": 0.3597237467765808,
      "learning_rate": 3.3911789781644074e-06,
      "loss": 0.0144,
      "reward": 2.3742024898529053,
      "reward_std": 0.4393809139728546,
      "rewards/reward_function": 2.3742024898529053,
      "step": 779
    },
    {
      "completion_length": 238.10714721679688,
      "epoch": 0.4462242562929062,
      "grad_norm": 0.2804965376853943,
      "kl": 0.18150803446769714,
      "learning_rate": 3.386512217606339e-06,
      "loss": 0.0073,
      "reward": 1.3925102949142456,
      "reward_std": 0.6683364510536194,
      "rewards/reward_function": 1.3925102949142456,
      "step": 780
    },
    {
      "completion_length": 237.00001525878906,
      "epoch": 0.44679633867276886,
      "grad_norm": 0.24438481032848358,
      "kl": 0.17404507100582123,
      "learning_rate": 3.381841920929196e-06,
      "loss": 0.007,
      "reward": 1.9805941581726074,
      "reward_std": 1.3235666751861572,
      "rewards/reward_function": 1.9805941581726074,
      "step": 781
    },
    {
      "completion_length": 231.1785888671875,
      "epoch": 0.4473684210526316,
      "grad_norm": 0.6300503611564636,
      "kl": 0.25597965717315674,
      "learning_rate": 3.3771681067618513e-06,
      "loss": 0.0102,
      "reward": 2.7295174598693848,
      "reward_std": 0.7064225673675537,
      "rewards/reward_function": 2.7295174598693848,
      "step": 782
    },
    {
      "completion_length": 288.3571472167969,
      "epoch": 0.44794050343249425,
      "grad_norm": 0.22880308330059052,
      "kl": 0.15202468633651733,
      "learning_rate": 3.3724907937472117e-06,
      "loss": 0.0061,
      "reward": 1.7311164140701294,
      "reward_std": 0.9075415730476379,
      "rewards/reward_function": 1.7311164140701294,
      "step": 783
    },
    {
      "completion_length": 250.7857208251953,
      "epoch": 0.448512585812357,
      "grad_norm": 0.2325151413679123,
      "kl": 0.2799983024597168,
      "learning_rate": 3.367810000542139e-06,
      "loss": 0.0112,
      "reward": 2.2574477195739746,
      "reward_std": 0.6349935531616211,
      "rewards/reward_function": 2.2574477195739746,
      "step": 784
    },
    {
      "completion_length": 249.4285888671875,
      "epoch": 0.4490846681922197,
      "grad_norm": 0.21767474710941315,
      "kl": 0.17614811658859253,
      "learning_rate": 3.363125745817375e-06,
      "loss": 0.007,
      "reward": 1.954627513885498,
      "reward_std": 0.9443531632423401,
      "rewards/reward_function": 1.954627513885498,
      "step": 785
    },
    {
      "completion_length": 245.75001525878906,
      "epoch": 0.4496567505720824,
      "grad_norm": 0.2131534367799759,
      "kl": 0.1759442239999771,
      "learning_rate": 3.358438048257472e-06,
      "loss": 0.007,
      "reward": 2.0801892280578613,
      "reward_std": 0.5821663737297058,
      "rewards/reward_function": 2.0801892280578613,
      "step": 786
    },
    {
      "completion_length": 249.71429443359375,
      "epoch": 0.4502288329519451,
      "grad_norm": 0.3821372985839844,
      "kl": 1.3233096599578857,
      "learning_rate": 3.3537469265607125e-06,
      "loss": 0.0529,
      "reward": 2.53147554397583,
      "reward_std": 1.0092997550964355,
      "rewards/reward_function": 2.53147554397583,
      "step": 787
    },
    {
      "completion_length": 257.64288330078125,
      "epoch": 0.45080091533180777,
      "grad_norm": 0.18837009370326996,
      "kl": 0.1684047281742096,
      "learning_rate": 3.3490523994390384e-06,
      "loss": 0.0067,
      "reward": 1.9704135656356812,
      "reward_std": 0.9836868047714233,
      "rewards/reward_function": 1.9704135656356812,
      "step": 788
    },
    {
      "completion_length": 225.2857208251953,
      "epoch": 0.4513729977116705,
      "grad_norm": 0.2450709342956543,
      "kl": 0.15515534579753876,
      "learning_rate": 3.344354485617976e-06,
      "loss": 0.0062,
      "reward": 2.1061413288116455,
      "reward_std": 0.7006552815437317,
      "rewards/reward_function": 2.1061413288116455,
      "step": 789
    },
    {
      "completion_length": 226.5357208251953,
      "epoch": 0.45194508009153317,
      "grad_norm": 0.24310839176177979,
      "kl": 0.17853912711143494,
      "learning_rate": 3.3396532038365583e-06,
      "loss": 0.0071,
      "reward": 1.8912618160247803,
      "reward_std": 0.9267652034759521,
      "rewards/reward_function": 1.8912618160247803,
      "step": 790
    },
    {
      "completion_length": 296.2857360839844,
      "epoch": 0.4525171624713959,
      "grad_norm": 0.21738570928573608,
      "kl": 0.20601390302181244,
      "learning_rate": 3.3349485728472536e-06,
      "loss": 0.0082,
      "reward": 1.9456095695495605,
      "reward_std": 0.9353324174880981,
      "rewards/reward_function": 1.9456095695495605,
      "step": 791
    },
    {
      "completion_length": 256.89288330078125,
      "epoch": 0.45308924485125857,
      "grad_norm": 0.5012170672416687,
      "kl": 0.33837637305259705,
      "learning_rate": 3.33024061141589e-06,
      "loss": 0.0135,
      "reward": 1.4447294473648071,
      "reward_std": 0.7609240412712097,
      "rewards/reward_function": 1.4447294473648071,
      "step": 792
    },
    {
      "completion_length": 276.9285888671875,
      "epoch": 0.4536613272311213,
      "grad_norm": 0.25675657391548157,
      "kl": 0.2475465089082718,
      "learning_rate": 3.325529338321579e-06,
      "loss": 0.0099,
      "reward": 2.1885087490081787,
      "reward_std": 1.0888780355453491,
      "rewards/reward_function": 2.1885087490081787,
      "step": 793
    },
    {
      "completion_length": 271.21429443359375,
      "epoch": 0.45423340961098396,
      "grad_norm": 0.3419303894042969,
      "kl": 0.45711880922317505,
      "learning_rate": 3.3208147723566418e-06,
      "loss": 0.0183,
      "reward": 1.3767386674880981,
      "reward_std": 1.026566505432129,
      "rewards/reward_function": 1.3767386674880981,
      "step": 794
    },
    {
      "completion_length": 304.4285888671875,
      "epoch": 0.4548054919908467,
      "grad_norm": 0.37090834975242615,
      "kl": 0.17295874655246735,
      "learning_rate": 3.3160969323265356e-06,
      "loss": 0.0069,
      "reward": 1.932070016860962,
      "reward_std": 1.2881062030792236,
      "rewards/reward_function": 1.932070016860962,
      "step": 795
    },
    {
      "completion_length": 284.96429443359375,
      "epoch": 0.45537757437070936,
      "grad_norm": 0.2137794941663742,
      "kl": 0.18886978924274445,
      "learning_rate": 3.311375837049776e-06,
      "loss": 0.0076,
      "reward": 1.5926625728607178,
      "reward_std": 0.9213885068893433,
      "rewards/reward_function": 1.5926625728607178,
      "step": 796
    },
    {
      "completion_length": 269.5357360839844,
      "epoch": 0.4559496567505721,
      "grad_norm": 0.27604544162750244,
      "kl": 0.7924100756645203,
      "learning_rate": 3.3066515053578615e-06,
      "loss": 0.0317,
      "reward": 1.7489840984344482,
      "reward_std": 1.1862704753875732,
      "rewards/reward_function": 1.7489840984344482,
      "step": 797
    },
    {
      "completion_length": 236.8928680419922,
      "epoch": 0.45652173913043476,
      "grad_norm": 0.22248385846614838,
      "kl": 0.13545842468738556,
      "learning_rate": 3.301923956095204e-06,
      "loss": 0.0054,
      "reward": 1.8071452379226685,
      "reward_std": 0.7201143503189087,
      "rewards/reward_function": 1.8071452379226685,
      "step": 798
    },
    {
      "completion_length": 291.46429443359375,
      "epoch": 0.4570938215102975,
      "grad_norm": 0.24195687472820282,
      "kl": 0.18039976060390472,
      "learning_rate": 3.2971932081190474e-06,
      "loss": 0.0072,
      "reward": 1.6841126680374146,
      "reward_std": 1.0960444211959839,
      "rewards/reward_function": 1.6841126680374146,
      "step": 799
    },
    {
      "completion_length": 228.82144165039062,
      "epoch": 0.4576659038901602,
      "grad_norm": 0.28356656432151794,
      "kl": 0.17124919593334198,
      "learning_rate": 3.292459280299392e-06,
      "loss": 0.0069,
      "reward": 2.5060348510742188,
      "reward_std": 0.8074661493301392,
      "rewards/reward_function": 2.5060348510742188,
      "step": 800
    },
    {
      "completion_length": 300.21429443359375,
      "epoch": 0.4582379862700229,
      "grad_norm": 0.2783462703227997,
      "kl": 0.28591763973236084,
      "learning_rate": 3.287722191518927e-06,
      "loss": 0.0114,
      "reward": 1.6557564735412598,
      "reward_std": 0.9753643274307251,
      "rewards/reward_function": 1.6557564735412598,
      "step": 801
    },
    {
      "completion_length": 240.6428680419922,
      "epoch": 0.4588100686498856,
      "grad_norm": 0.2718451917171478,
      "kl": 0.15873858332633972,
      "learning_rate": 3.282981960672948e-06,
      "loss": 0.0063,
      "reward": 1.4797676801681519,
      "reward_std": 0.644866943359375,
      "rewards/reward_function": 1.4797676801681519,
      "step": 802
    },
    {
      "completion_length": 228.46429443359375,
      "epoch": 0.4593821510297483,
      "grad_norm": 0.24765558540821075,
      "kl": 0.3072304427623749,
      "learning_rate": 3.2782386066692804e-06,
      "loss": 0.0123,
      "reward": 1.9548027515411377,
      "reward_std": 0.9259627461433411,
      "rewards/reward_function": 1.9548027515411377,
      "step": 803
    },
    {
      "completion_length": 322.71429443359375,
      "epoch": 0.459954233409611,
      "grad_norm": 0.5723429918289185,
      "kl": 1.1608566045761108,
      "learning_rate": 3.2734921484282132e-06,
      "loss": 0.0464,
      "reward": 1.8458929061889648,
      "reward_std": 0.791182279586792,
      "rewards/reward_function": 1.8458929061889648,
      "step": 804
    },
    {
      "completion_length": 278.7857360839844,
      "epoch": 0.4605263157894737,
      "grad_norm": 1.3441767692565918,
      "kl": 0.6126006841659546,
      "learning_rate": 3.2687426048824124e-06,
      "loss": 0.0245,
      "reward": 2.1484410762786865,
      "reward_std": 0.9273087382316589,
      "rewards/reward_function": 2.1484410762786865,
      "step": 805
    },
    {
      "completion_length": 240.00001525878906,
      "epoch": 0.4610983981693364,
      "grad_norm": 0.2575643062591553,
      "kl": 0.28571200370788574,
      "learning_rate": 3.263989994976852e-06,
      "loss": 0.0114,
      "reward": 1.728276252746582,
      "reward_std": 0.40742650628089905,
      "rewards/reward_function": 1.728276252746582,
      "step": 806
    },
    {
      "completion_length": 251.6428680419922,
      "epoch": 0.46167048054919907,
      "grad_norm": 1.1245108842849731,
      "kl": 1.3736846446990967,
      "learning_rate": 3.2592343376687406e-06,
      "loss": 0.0549,
      "reward": 2.3418290615081787,
      "reward_std": 0.819881021976471,
      "rewards/reward_function": 2.3418290615081787,
      "step": 807
    },
    {
      "completion_length": 251.46429443359375,
      "epoch": 0.4622425629290618,
      "grad_norm": 0.2718649208545685,
      "kl": 0.5123030543327332,
      "learning_rate": 3.2544756519274378e-06,
      "loss": 0.0205,
      "reward": 1.8558372259140015,
      "reward_std": 0.841848611831665,
      "rewards/reward_function": 1.8558372259140015,
      "step": 808
    },
    {
      "completion_length": 269.7857360839844,
      "epoch": 0.46281464530892447,
      "grad_norm": 0.2236190289258957,
      "kl": 0.19079594314098358,
      "learning_rate": 3.2497139567343846e-06,
      "loss": 0.0076,
      "reward": 2.323625087738037,
      "reward_std": 0.7320543527603149,
      "rewards/reward_function": 2.323625087738037,
      "step": 809
    },
    {
      "completion_length": 275.64288330078125,
      "epoch": 0.4633867276887872,
      "grad_norm": 0.23026445508003235,
      "kl": 0.24806751310825348,
      "learning_rate": 3.2449492710830268e-06,
      "loss": 0.0099,
      "reward": 1.980905294418335,
      "reward_std": 1.1082617044448853,
      "rewards/reward_function": 1.980905294418335,
      "step": 810
    },
    {
      "completion_length": 340.46429443359375,
      "epoch": 0.46395881006864986,
      "grad_norm": 0.20427018404006958,
      "kl": 0.4161158800125122,
      "learning_rate": 3.240181613978737e-06,
      "loss": 0.0166,
      "reward": 1.876552700996399,
      "reward_std": 1.2290633916854858,
      "rewards/reward_function": 1.876552700996399,
      "step": 811
    },
    {
      "completion_length": 306.71429443359375,
      "epoch": 0.4645308924485126,
      "grad_norm": 0.20877133309841156,
      "kl": 0.1490364819765091,
      "learning_rate": 3.235411004438741e-06,
      "loss": 0.006,
      "reward": 2.4642608165740967,
      "reward_std": 1.0602240562438965,
      "rewards/reward_function": 2.4642608165740967,
      "step": 812
    },
    {
      "completion_length": 276.3214416503906,
      "epoch": 0.46510297482837526,
      "grad_norm": 0.5306329727172852,
      "kl": 0.5143252015113831,
      "learning_rate": 3.2306374614920434e-06,
      "loss": 0.0206,
      "reward": 1.975346565246582,
      "reward_std": 1.0946815013885498,
      "rewards/reward_function": 1.975346565246582,
      "step": 813
    },
    {
      "completion_length": 265.1785888671875,
      "epoch": 0.465675057208238,
      "grad_norm": 0.2500179409980774,
      "kl": 0.149267315864563,
      "learning_rate": 3.225861004179346e-06,
      "loss": 0.006,
      "reward": 1.7854712009429932,
      "reward_std": 0.8651689291000366,
      "rewards/reward_function": 1.7854712009429932,
      "step": 814
    },
    {
      "completion_length": 269.5357360839844,
      "epoch": 0.4662471395881007,
      "grad_norm": 0.2299981266260147,
      "kl": 0.13918158411979675,
      "learning_rate": 3.221081651552978e-06,
      "loss": 0.0056,
      "reward": 1.6229039430618286,
      "reward_std": 0.4577193260192871,
      "rewards/reward_function": 1.6229039430618286,
      "step": 815
    },
    {
      "completion_length": 273.5357360839844,
      "epoch": 0.4668192219679634,
      "grad_norm": 0.20772907137870789,
      "kl": 0.13795064389705658,
      "learning_rate": 3.2162994226768156e-06,
      "loss": 0.0055,
      "reward": 2.189099073410034,
      "reward_std": 0.9367181658744812,
      "rewards/reward_function": 2.189099073410034,
      "step": 816
    },
    {
      "completion_length": 288.0,
      "epoch": 0.4673913043478261,
      "grad_norm": 0.20625516772270203,
      "kl": 0.13982386887073517,
      "learning_rate": 3.21151433662621e-06,
      "loss": 0.0056,
      "reward": 1.590290904045105,
      "reward_std": 0.8364889025688171,
      "rewards/reward_function": 1.590290904045105,
      "step": 817
    },
    {
      "completion_length": 294.4285888671875,
      "epoch": 0.4679633867276888,
      "grad_norm": 0.3055521547794342,
      "kl": 0.15735915303230286,
      "learning_rate": 3.2067264124879065e-06,
      "loss": 0.0063,
      "reward": 1.915257453918457,
      "reward_std": 0.7527197599411011,
      "rewards/reward_function": 1.915257453918457,
      "step": 818
    },
    {
      "completion_length": 294.6071472167969,
      "epoch": 0.4685354691075515,
      "grad_norm": 24.286834716796875,
      "kl": 2.8541791439056396,
      "learning_rate": 3.2019356693599737e-06,
      "loss": 0.1142,
      "reward": 1.8375974893569946,
      "reward_std": 0.9820327758789062,
      "rewards/reward_function": 1.8375974893569946,
      "step": 819
    },
    {
      "completion_length": 293.46429443359375,
      "epoch": 0.4691075514874142,
      "grad_norm": 0.21289852261543274,
      "kl": 0.16783057153224945,
      "learning_rate": 3.1971421263517224e-06,
      "loss": 0.0067,
      "reward": 2.8218328952789307,
      "reward_std": 0.7135964632034302,
      "rewards/reward_function": 2.8218328952789307,
      "step": 820
    },
    {
      "completion_length": 335.3571472167969,
      "epoch": 0.4696796338672769,
      "grad_norm": 0.36004990339279175,
      "kl": 0.5666213631629944,
      "learning_rate": 3.1923458025836305e-06,
      "loss": 0.0227,
      "reward": 2.395515203475952,
      "reward_std": 0.7552555799484253,
      "rewards/reward_function": 2.395515203475952,
      "step": 821
    },
    {
      "completion_length": 264.14288330078125,
      "epoch": 0.4702517162471396,
      "grad_norm": 0.2905164659023285,
      "kl": 0.19292515516281128,
      "learning_rate": 3.187546717187273e-06,
      "loss": 0.0077,
      "reward": 1.7493669986724854,
      "reward_std": 0.8702834248542786,
      "rewards/reward_function": 1.7493669986724854,
      "step": 822
    },
    {
      "completion_length": 235.2857208251953,
      "epoch": 0.4708237986270023,
      "grad_norm": 0.24656017124652863,
      "kl": 0.19526122510433197,
      "learning_rate": 3.182744889305235e-06,
      "loss": 0.0078,
      "reward": 2.6179745197296143,
      "reward_std": 0.7364758253097534,
      "rewards/reward_function": 2.6179745197296143,
      "step": 823
    },
    {
      "completion_length": 307.39288330078125,
      "epoch": 0.47139588100686497,
      "grad_norm": 0.28913962841033936,
      "kl": 0.6733137965202332,
      "learning_rate": 3.177940338091043e-06,
      "loss": 0.0269,
      "reward": 2.031243085861206,
      "reward_std": 0.944770872592926,
      "rewards/reward_function": 2.031243085861206,
      "step": 824
    },
    {
      "completion_length": 314.64288330078125,
      "epoch": 0.4719679633867277,
      "grad_norm": 6.767683029174805,
      "kl": 0.7346335053443909,
      "learning_rate": 3.1731330827090865e-06,
      "loss": 0.0294,
      "reward": 2.1180713176727295,
      "reward_std": 1.149366855621338,
      "rewards/reward_function": 2.1180713176727295,
      "step": 825
    },
    {
      "completion_length": 229.3928680419922,
      "epoch": 0.47254004576659037,
      "grad_norm": 0.21978360414505005,
      "kl": 0.13130255043506622,
      "learning_rate": 3.1683231423345417e-06,
      "loss": 0.0053,
      "reward": 2.1552414894104004,
      "reward_std": 1.093698263168335,
      "rewards/reward_function": 2.1552414894104004,
      "step": 826
    },
    {
      "completion_length": 312.0714416503906,
      "epoch": 0.4731121281464531,
      "grad_norm": 0.19777065515518188,
      "kl": 0.15314093232154846,
      "learning_rate": 3.163510536153293e-06,
      "loss": 0.0061,
      "reward": 2.360118865966797,
      "reward_std": 1.1822589635849,
      "rewards/reward_function": 2.360118865966797,
      "step": 827
    },
    {
      "completion_length": 285.25,
      "epoch": 0.47368421052631576,
      "grad_norm": 2.4991743564605713,
      "kl": 0.7376531362533569,
      "learning_rate": 3.158695283361861e-06,
      "loss": 0.0295,
      "reward": 1.3484755754470825,
      "reward_std": 0.7513796091079712,
      "rewards/reward_function": 1.3484755754470825,
      "step": 828
    },
    {
      "completion_length": 289.21429443359375,
      "epoch": 0.4742562929061785,
      "grad_norm": 0.19772149622440338,
      "kl": 0.23551949858665466,
      "learning_rate": 3.1538774031673194e-06,
      "loss": 0.0094,
      "reward": 2.3970963954925537,
      "reward_std": 0.4715765118598938,
      "rewards/reward_function": 2.3970963954925537,
      "step": 829
    },
    {
      "completion_length": 290.0,
      "epoch": 0.4748283752860412,
      "grad_norm": 0.2538474500179291,
      "kl": 0.22294393181800842,
      "learning_rate": 3.149056914787224e-06,
      "loss": 0.0089,
      "reward": 2.0359649658203125,
      "reward_std": 1.1935662031173706,
      "rewards/reward_function": 2.0359649658203125,
      "step": 830
    },
    {
      "completion_length": 286.64288330078125,
      "epoch": 0.4754004576659039,
      "grad_norm": 0.19749870896339417,
      "kl": 0.1567390263080597,
      "learning_rate": 3.1442338374495366e-06,
      "loss": 0.0063,
      "reward": 2.2104082107543945,
      "reward_std": 0.9072138071060181,
      "rewards/reward_function": 2.2104082107543945,
      "step": 831
    },
    {
      "completion_length": 318.6071472167969,
      "epoch": 0.4759725400457666,
      "grad_norm": 0.21242065727710724,
      "kl": 0.17098598182201385,
      "learning_rate": 3.139408190392541e-06,
      "loss": 0.0068,
      "reward": 2.628072738647461,
      "reward_std": 1.0010292530059814,
      "rewards/reward_function": 2.628072738647461,
      "step": 832
    },
    {
      "completion_length": 305.4285888671875,
      "epoch": 0.4765446224256293,
      "grad_norm": 0.19053882360458374,
      "kl": 0.3868914842605591,
      "learning_rate": 3.134579992864775e-06,
      "loss": 0.0155,
      "reward": 1.7525756359100342,
      "reward_std": 1.2146129608154297,
      "rewards/reward_function": 1.7525756359100342,
      "step": 833
    },
    {
      "completion_length": 303.5357360839844,
      "epoch": 0.477116704805492,
      "grad_norm": 0.18980638682842255,
      "kl": 0.1477152556180954,
      "learning_rate": 3.1297492641249474e-06,
      "loss": 0.0059,
      "reward": 2.0559325218200684,
      "reward_std": 0.9635282754898071,
      "rewards/reward_function": 2.0559325218200684,
      "step": 834
    },
    {
      "completion_length": 267.3214416503906,
      "epoch": 0.4776887871853547,
      "grad_norm": 0.219776451587677,
      "kl": 0.15838785469532013,
      "learning_rate": 3.1249160234418646e-06,
      "loss": 0.0063,
      "reward": 2.374739170074463,
      "reward_std": 0.9685264229774475,
      "rewards/reward_function": 2.374739170074463,
      "step": 835
    },
    {
      "completion_length": 258.1071472167969,
      "epoch": 0.4782608695652174,
      "grad_norm": 0.2018679976463318,
      "kl": 0.14000555872917175,
      "learning_rate": 3.1200802900943517e-06,
      "loss": 0.0056,
      "reward": 2.450563907623291,
      "reward_std": 0.9457153081893921,
      "rewards/reward_function": 2.450563907623291,
      "step": 836
    },
    {
      "completion_length": 336.5357360839844,
      "epoch": 0.4788329519450801,
      "grad_norm": 0.21638357639312744,
      "kl": 0.16397127509117126,
      "learning_rate": 3.115242083371179e-06,
      "loss": 0.0066,
      "reward": 1.6818840503692627,
      "reward_std": 0.5663355588912964,
      "rewards/reward_function": 1.6818840503692627,
      "step": 837
    },
    {
      "completion_length": 252.96429443359375,
      "epoch": 0.4794050343249428,
      "grad_norm": 0.1996784210205078,
      "kl": 0.1821206510066986,
      "learning_rate": 3.1104014225709787e-06,
      "loss": 0.0073,
      "reward": 1.6710810661315918,
      "reward_std": 0.9717928767204285,
      "rewards/reward_function": 1.6710810661315918,
      "step": 838
    },
    {
      "completion_length": 278.1071472167969,
      "epoch": 0.4799771167048055,
      "grad_norm": 0.2115052342414856,
      "kl": 0.170805424451828,
      "learning_rate": 3.105558327002175e-06,
      "loss": 0.0068,
      "reward": 1.8885575532913208,
      "reward_std": 0.751812756061554,
      "rewards/reward_function": 1.8885575532913208,
      "step": 839
    },
    {
      "completion_length": 296.8571472167969,
      "epoch": 0.4805491990846682,
      "grad_norm": 0.19157111644744873,
      "kl": 0.2562447488307953,
      "learning_rate": 3.1007128159829023e-06,
      "loss": 0.0103,
      "reward": 1.9992918968200684,
      "reward_std": 0.7328622341156006,
      "rewards/reward_function": 1.9992918968200684,
      "step": 840
    },
    {
      "completion_length": 263.9285888671875,
      "epoch": 0.48112128146453087,
      "grad_norm": 0.2226649522781372,
      "kl": 0.148444265127182,
      "learning_rate": 3.0958649088409314e-06,
      "loss": 0.0059,
      "reward": 2.2175588607788086,
      "reward_std": 0.8429732918739319,
      "rewards/reward_function": 2.2175588607788086,
      "step": 841
    },
    {
      "completion_length": 249.21429443359375,
      "epoch": 0.4816933638443936,
      "grad_norm": 0.27627044916152954,
      "kl": 0.2209891676902771,
      "learning_rate": 3.091014624913589e-06,
      "loss": 0.0088,
      "reward": 1.5759145021438599,
      "reward_std": 0.8256129026412964,
      "rewards/reward_function": 1.5759145021438599,
      "step": 842
    },
    {
      "completion_length": 297.7857360839844,
      "epoch": 0.48226544622425627,
      "grad_norm": 1.007222294807434,
      "kl": 0.8415696620941162,
      "learning_rate": 3.086161983547682e-06,
      "loss": 0.0337,
      "reward": 2.244699001312256,
      "reward_std": 0.6597459316253662,
      "rewards/reward_function": 2.244699001312256,
      "step": 843
    },
    {
      "completion_length": 251.9285888671875,
      "epoch": 0.482837528604119,
      "grad_norm": 0.594884991645813,
      "kl": 0.495612233877182,
      "learning_rate": 3.0813070040994226e-06,
      "loss": 0.0198,
      "reward": 1.3451416492462158,
      "reward_std": 0.9002770185470581,
      "rewards/reward_function": 1.3451416492462158,
      "step": 844
    },
    {
      "completion_length": 325.8214416503906,
      "epoch": 0.4834096109839817,
      "grad_norm": 0.2705564498901367,
      "kl": 0.22078707814216614,
      "learning_rate": 3.0764497059343473e-06,
      "loss": 0.0088,
      "reward": 2.04384183883667,
      "reward_std": 0.8160290122032166,
      "rewards/reward_function": 2.04384183883667,
      "step": 845
    },
    {
      "completion_length": 287.0,
      "epoch": 0.4839816933638444,
      "grad_norm": 0.18490514159202576,
      "kl": 0.1815681904554367,
      "learning_rate": 3.071590108427244e-06,
      "loss": 0.0073,
      "reward": 2.02823805809021,
      "reward_std": 0.6788418292999268,
      "rewards/reward_function": 2.02823805809021,
      "step": 846
    },
    {
      "completion_length": 292.1071472167969,
      "epoch": 0.4845537757437071,
      "grad_norm": 0.29064613580703735,
      "kl": 0.21178866922855377,
      "learning_rate": 3.066728230962069e-06,
      "loss": 0.0085,
      "reward": 1.5043107271194458,
      "reward_std": 0.8790081143379211,
      "rewards/reward_function": 1.5043107271194458,
      "step": 847
    },
    {
      "completion_length": 295.46429443359375,
      "epoch": 0.4851258581235698,
      "grad_norm": 0.20505762100219727,
      "kl": 0.2502254843711853,
      "learning_rate": 3.0618640929318744e-06,
      "loss": 0.01,
      "reward": 2.1831541061401367,
      "reward_std": 0.9767600297927856,
      "rewards/reward_function": 2.1831541061401367,
      "step": 848
    },
    {
      "completion_length": 302.46429443359375,
      "epoch": 0.4856979405034325,
      "grad_norm": 0.17127054929733276,
      "kl": 0.1349862813949585,
      "learning_rate": 3.05699771373873e-06,
      "loss": 0.0054,
      "reward": 1.3964667320251465,
      "reward_std": 0.6524763107299805,
      "rewards/reward_function": 1.3964667320251465,
      "step": 849
    },
    {
      "completion_length": 285.1071472167969,
      "epoch": 0.4862700228832952,
      "grad_norm": 0.20644019544124603,
      "kl": 0.1689797341823578,
      "learning_rate": 3.0521291127936437e-06,
      "loss": 0.0068,
      "reward": 2.524242401123047,
      "reward_std": 0.9072580933570862,
      "rewards/reward_function": 2.524242401123047,
      "step": 850
    },
    {
      "completion_length": 272.39288330078125,
      "epoch": 0.4868421052631579,
      "grad_norm": 0.39432933926582336,
      "kl": 0.4272276759147644,
      "learning_rate": 3.0472583095164875e-06,
      "loss": 0.0171,
      "reward": 2.607804775238037,
      "reward_std": 1.0663436651229858,
      "rewards/reward_function": 2.607804775238037,
      "step": 851
    },
    {
      "completion_length": 285.71429443359375,
      "epoch": 0.4874141876430206,
      "grad_norm": 0.42206352949142456,
      "kl": 0.6941148638725281,
      "learning_rate": 3.042385323335915e-06,
      "loss": 0.0278,
      "reward": 2.565422773361206,
      "reward_std": 0.9980140328407288,
      "rewards/reward_function": 2.565422773361206,
      "step": 852
    },
    {
      "completion_length": 300.1071472167969,
      "epoch": 0.4879862700228833,
      "grad_norm": 0.1876029074192047,
      "kl": 0.1412210464477539,
      "learning_rate": 3.037510173689291e-06,
      "loss": 0.0056,
      "reward": 1.7357559204101562,
      "reward_std": 0.6747377514839172,
      "rewards/reward_function": 1.7357559204101562,
      "step": 853
    },
    {
      "completion_length": 324.71429443359375,
      "epoch": 0.488558352402746,
      "grad_norm": 0.21100077033042908,
      "kl": 0.1898089498281479,
      "learning_rate": 3.032632880022606e-06,
      "loss": 0.0076,
      "reward": 1.6573625802993774,
      "reward_std": 0.8641158938407898,
      "rewards/reward_function": 1.6573625802993774,
      "step": 854
    },
    {
      "completion_length": 296.1785888671875,
      "epoch": 0.4891304347826087,
      "grad_norm": 1.4272122383117676,
      "kl": 0.931486189365387,
      "learning_rate": 3.027753461790404e-06,
      "loss": 0.0373,
      "reward": 2.146742343902588,
      "reward_std": 1.0110100507736206,
      "rewards/reward_function": 2.146742343902588,
      "step": 855
    },
    {
      "completion_length": 329.14288330078125,
      "epoch": 0.4897025171624714,
      "grad_norm": 0.3881724774837494,
      "kl": 1.0081419944763184,
      "learning_rate": 3.022871938455705e-06,
      "loss": 0.0403,
      "reward": 1.4329644441604614,
      "reward_std": 1.004045009613037,
      "rewards/reward_function": 1.4329644441604614,
      "step": 856
    },
    {
      "completion_length": 319.71429443359375,
      "epoch": 0.4902745995423341,
      "grad_norm": 0.2616538107395172,
      "kl": 0.20552638173103333,
      "learning_rate": 3.017988329489923e-06,
      "loss": 0.0082,
      "reward": 2.2502684593200684,
      "reward_std": 0.5366341471672058,
      "rewards/reward_function": 2.2502684593200684,
      "step": 857
    },
    {
      "completion_length": 265.2857360839844,
      "epoch": 0.49084668192219677,
      "grad_norm": 0.2502187490463257,
      "kl": 0.20525439083576202,
      "learning_rate": 3.0131026543727937e-06,
      "loss": 0.0082,
      "reward": 1.467465877532959,
      "reward_std": 0.8647984266281128,
      "rewards/reward_function": 1.467465877532959,
      "step": 858
    },
    {
      "completion_length": 303.0,
      "epoch": 0.4914187643020595,
      "grad_norm": 0.2270292490720749,
      "kl": 0.2051761895418167,
      "learning_rate": 3.0082149325922944e-06,
      "loss": 0.0082,
      "reward": 2.3335659503936768,
      "reward_std": 0.9184761643409729,
      "rewards/reward_function": 2.3335659503936768,
      "step": 859
    },
    {
      "completion_length": 229.21429443359375,
      "epoch": 0.4919908466819222,
      "grad_norm": 0.23549973964691162,
      "kl": 0.16534368693828583,
      "learning_rate": 3.0033251836445638e-06,
      "loss": 0.0066,
      "reward": 2.1697323322296143,
      "reward_std": 0.9825607538223267,
      "rewards/reward_function": 2.1697323322296143,
      "step": 860
    },
    {
      "completion_length": 333.5357360839844,
      "epoch": 0.4925629290617849,
      "grad_norm": 0.27585169672966003,
      "kl": 0.2492418885231018,
      "learning_rate": 2.998433427033827e-06,
      "loss": 0.01,
      "reward": 2.1021573543548584,
      "reward_std": 0.735519289970398,
      "rewards/reward_function": 2.1021573543548584,
      "step": 861
    },
    {
      "completion_length": 290.8571472167969,
      "epoch": 0.4931350114416476,
      "grad_norm": 0.4370723366737366,
      "kl": 0.2935715913772583,
      "learning_rate": 2.9935396822723194e-06,
      "loss": 0.0117,
      "reward": 2.223468542098999,
      "reward_std": 0.6501122713088989,
      "rewards/reward_function": 2.223468542098999,
      "step": 862
    },
    {
      "completion_length": 264.1785888671875,
      "epoch": 0.4937070938215103,
      "grad_norm": 0.18670226633548737,
      "kl": 0.13589338958263397,
      "learning_rate": 2.9886439688802048e-06,
      "loss": 0.0054,
      "reward": 2.227066993713379,
      "reward_std": 1.1597334146499634,
      "rewards/reward_function": 2.227066993713379,
      "step": 863
    },
    {
      "completion_length": 242.0357208251953,
      "epoch": 0.494279176201373,
      "grad_norm": 0.21806500852108002,
      "kl": 0.1660381257534027,
      "learning_rate": 2.9837463063854995e-06,
      "loss": 0.0066,
      "reward": 1.7313344478607178,
      "reward_std": 0.4855382740497589,
      "rewards/reward_function": 1.7313344478607178,
      "step": 864
    },
    {
      "completion_length": 242.8928680419922,
      "epoch": 0.4948512585812357,
      "grad_norm": 0.21132299304008484,
      "kl": 0.18847662210464478,
      "learning_rate": 2.9788467143239965e-06,
      "loss": 0.0075,
      "reward": 1.6256368160247803,
      "reward_std": 0.9588679075241089,
      "rewards/reward_function": 1.6256368160247803,
      "step": 865
    },
    {
      "completion_length": 290.2857360839844,
      "epoch": 0.4954233409610984,
      "grad_norm": 0.2163962870836258,
      "kl": 0.19699333608150482,
      "learning_rate": 2.9739452122391815e-06,
      "loss": 0.0079,
      "reward": 1.445899248123169,
      "reward_std": 0.9659876823425293,
      "rewards/reward_function": 1.445899248123169,
      "step": 866
    },
    {
      "completion_length": 327.25,
      "epoch": 0.4959954233409611,
      "grad_norm": 0.2185884565114975,
      "kl": 0.19911469519138336,
      "learning_rate": 2.969041819682163e-06,
      "loss": 0.008,
      "reward": 1.6919000148773193,
      "reward_std": 0.8453955054283142,
      "rewards/reward_function": 1.6919000148773193,
      "step": 867
    },
    {
      "completion_length": 307.6785888671875,
      "epoch": 0.4965675057208238,
      "grad_norm": 0.1785522997379303,
      "kl": 0.15444067120552063,
      "learning_rate": 2.9641365562115886e-06,
      "loss": 0.0062,
      "reward": 2.7261836528778076,
      "reward_std": 0.8582249283790588,
      "rewards/reward_function": 2.7261836528778076,
      "step": 868
    },
    {
      "completion_length": 287.14288330078125,
      "epoch": 0.4971395881006865,
      "grad_norm": 0.289832204580307,
      "kl": 0.28973039984703064,
      "learning_rate": 2.9592294413935683e-06,
      "loss": 0.0116,
      "reward": 1.6270426511764526,
      "reward_std": 1.3562644720077515,
      "rewards/reward_function": 1.6270426511764526,
      "step": 869
    },
    {
      "completion_length": 299.6785888671875,
      "epoch": 0.4977116704805492,
      "grad_norm": 0.20714303851127625,
      "kl": 0.17813436686992645,
      "learning_rate": 2.954320494801596e-06,
      "loss": 0.0071,
      "reward": 2.102156400680542,
      "reward_std": 1.2960364818572998,
      "rewards/reward_function": 2.102156400680542,
      "step": 870
    },
    {
      "completion_length": 293.1071472167969,
      "epoch": 0.4982837528604119,
      "grad_norm": 1.4663199186325073,
      "kl": 1.1808820962905884,
      "learning_rate": 2.949409736016474e-06,
      "loss": 0.0472,
      "reward": 2.012470006942749,
      "reward_std": 0.8153484463691711,
      "rewards/reward_function": 2.012470006942749,
      "step": 871
    },
    {
      "completion_length": 276.9285888671875,
      "epoch": 0.4988558352402746,
      "grad_norm": 0.24842403829097748,
      "kl": 0.1720389425754547,
      "learning_rate": 2.944497184626232e-06,
      "loss": 0.0069,
      "reward": 2.1589295864105225,
      "reward_std": 0.7184117436408997,
      "rewards/reward_function": 2.1589295864105225,
      "step": 872
    },
    {
      "completion_length": 341.25,
      "epoch": 0.4994279176201373,
      "grad_norm": 0.3981929421424866,
      "kl": 0.29600274562835693,
      "learning_rate": 2.93958286022605e-06,
      "loss": 0.0118,
      "reward": 1.9285370111465454,
      "reward_std": 0.9496399164199829,
      "rewards/reward_function": 1.9285370111465454,
      "step": 873
    },
    {
      "completion_length": 310.75,
      "epoch": 0.5,
      "grad_norm": 0.1844237744808197,
      "kl": 0.19341477751731873,
      "learning_rate": 2.9346667824181806e-06,
      "loss": 0.0077,
      "reward": 2.4296016693115234,
      "reward_std": 0.8336096405982971,
      "rewards/reward_function": 2.4296016693115234,
      "step": 874
    },
    {
      "completion_length": 356.5357360839844,
      "epoch": 0.5005720823798627,
      "grad_norm": 0.23108074069023132,
      "kl": 0.1624411940574646,
      "learning_rate": 2.9297489708118697e-06,
      "loss": 0.0065,
      "reward": 2.0535571575164795,
      "reward_std": 1.0768101215362549,
      "rewards/reward_function": 2.0535571575164795,
      "step": 875
    },
    {
      "completion_length": 288.8571472167969,
      "epoch": 0.5011441647597255,
      "grad_norm": 0.2294628769159317,
      "kl": 0.23944179713726044,
      "learning_rate": 2.9248294450232806e-06,
      "loss": 0.0096,
      "reward": 2.051074743270874,
      "reward_std": 1.0831785202026367,
      "rewards/reward_function": 2.051074743270874,
      "step": 876
    },
    {
      "completion_length": 293.8571472167969,
      "epoch": 0.5017162471395881,
      "grad_norm": 0.21515773236751556,
      "kl": 0.28141382336616516,
      "learning_rate": 2.919908224675412e-06,
      "loss": 0.0113,
      "reward": 2.2327868938446045,
      "reward_std": 0.9440484046936035,
      "rewards/reward_function": 2.2327868938446045,
      "step": 877
    },
    {
      "completion_length": 273.64288330078125,
      "epoch": 0.5022883295194508,
      "grad_norm": 0.21782232820987701,
      "kl": 0.17153361439704895,
      "learning_rate": 2.914985329398024e-06,
      "loss": 0.0069,
      "reward": 1.6796804666519165,
      "reward_std": 0.716663658618927,
      "rewards/reward_function": 1.6796804666519165,
      "step": 878
    },
    {
      "completion_length": 332.46429443359375,
      "epoch": 0.5028604118993135,
      "grad_norm": 0.2083820104598999,
      "kl": 0.1501208245754242,
      "learning_rate": 2.9100607788275547e-06,
      "loss": 0.006,
      "reward": 1.7736307382583618,
      "reward_std": 1.2384401559829712,
      "rewards/reward_function": 1.7736307382583618,
      "step": 879
    },
    {
      "completion_length": 296.64288330078125,
      "epoch": 0.5034324942791762,
      "grad_norm": 0.3489767611026764,
      "kl": 1.0714541673660278,
      "learning_rate": 2.9051345926070482e-06,
      "loss": 0.0429,
      "reward": 1.8878529071807861,
      "reward_std": 1.0620189905166626,
      "rewards/reward_function": 1.8878529071807861,
      "step": 880
    },
    {
      "completion_length": 282.6785888671875,
      "epoch": 0.5040045766590389,
      "grad_norm": 0.17624060809612274,
      "kl": 0.1411243975162506,
      "learning_rate": 2.9002067903860713e-06,
      "loss": 0.0056,
      "reward": 2.0570950508117676,
      "reward_std": 0.5238506197929382,
      "rewards/reward_function": 2.0570950508117676,
      "step": 881
    },
    {
      "completion_length": 318.71429443359375,
      "epoch": 0.5045766590389016,
      "grad_norm": 0.2465517222881317,
      "kl": 0.28859201073646545,
      "learning_rate": 2.8952773918206355e-06,
      "loss": 0.0115,
      "reward": 1.643612027168274,
      "reward_std": 1.0351427793502808,
      "rewards/reward_function": 1.643612027168274,
      "step": 882
    },
    {
      "completion_length": 309.0357360839844,
      "epoch": 0.5051487414187643,
      "grad_norm": 0.17678570747375488,
      "kl": 0.1430928260087967,
      "learning_rate": 2.8903464165731223e-06,
      "loss": 0.0057,
      "reward": 2.6382534503936768,
      "reward_std": 0.9080532193183899,
      "rewards/reward_function": 2.6382534503936768,
      "step": 883
    },
    {
      "completion_length": 329.96429443359375,
      "epoch": 0.505720823798627,
      "grad_norm": 0.1939430981874466,
      "kl": 0.1551240086555481,
      "learning_rate": 2.8854138843121997e-06,
      "loss": 0.0062,
      "reward": 1.1870672702789307,
      "reward_std": 0.38933393359184265,
      "rewards/reward_function": 1.1870672702789307,
      "step": 884
    },
    {
      "completion_length": 276.0,
      "epoch": 0.5062929061784897,
      "grad_norm": 0.17359107732772827,
      "kl": 0.37122294306755066,
      "learning_rate": 2.880479814712749e-06,
      "loss": 0.0148,
      "reward": 2.0701661109924316,
      "reward_std": 0.8208860754966736,
      "rewards/reward_function": 2.0701661109924316,
      "step": 885
    },
    {
      "completion_length": 245.8928680419922,
      "epoch": 0.5068649885583524,
      "grad_norm": 0.2894028425216675,
      "kl": 0.6627323627471924,
      "learning_rate": 2.8755442274557815e-06,
      "loss": 0.0265,
      "reward": 2.257594585418701,
      "reward_std": 0.7924765348434448,
      "rewards/reward_function": 2.257594585418701,
      "step": 886
    },
    {
      "completion_length": 277.2857360839844,
      "epoch": 0.5074370709382151,
      "grad_norm": 0.26471295952796936,
      "kl": 0.3769121468067169,
      "learning_rate": 2.8706071422283634e-06,
      "loss": 0.0151,
      "reward": 2.088155508041382,
      "reward_std": 0.8116816282272339,
      "rewards/reward_function": 2.088155508041382,
      "step": 887
    },
    {
      "completion_length": 302.1071472167969,
      "epoch": 0.5080091533180778,
      "grad_norm": 0.20810016989707947,
      "kl": 0.1653248816728592,
      "learning_rate": 2.865668578723534e-06,
      "loss": 0.0066,
      "reward": 1.567161202430725,
      "reward_std": 0.9151229858398438,
      "rewards/reward_function": 1.567161202430725,
      "step": 888
    },
    {
      "completion_length": 329.0714416503906,
      "epoch": 0.5085812356979404,
      "grad_norm": 0.18280254304409027,
      "kl": 0.17061491310596466,
      "learning_rate": 2.8607285566402326e-06,
      "loss": 0.0068,
      "reward": 2.224827766418457,
      "reward_std": 1.0494545698165894,
      "rewards/reward_function": 2.224827766418457,
      "step": 889
    },
    {
      "completion_length": 293.46429443359375,
      "epoch": 0.5091533180778032,
      "grad_norm": 0.23114271461963654,
      "kl": 0.17549803853034973,
      "learning_rate": 2.8557870956832135e-06,
      "loss": 0.007,
      "reward": 1.9855448007583618,
      "reward_std": 0.7224796414375305,
      "rewards/reward_function": 1.9855448007583618,
      "step": 890
    },
    {
      "completion_length": 241.7857208251953,
      "epoch": 0.5097254004576659,
      "grad_norm": 0.3457115590572357,
      "kl": 0.19242793321609497,
      "learning_rate": 2.850844215562971e-06,
      "loss": 0.0077,
      "reward": 1.5909394025802612,
      "reward_std": 1.0071433782577515,
      "rewards/reward_function": 1.5909394025802612,
      "step": 891
    },
    {
      "completion_length": 296.0714416503906,
      "epoch": 0.5102974828375286,
      "grad_norm": 0.2018698900938034,
      "kl": 0.152069091796875,
      "learning_rate": 2.8458999359956614e-06,
      "loss": 0.0061,
      "reward": 1.6008257865905762,
      "reward_std": 0.8541580438613892,
      "rewards/reward_function": 1.6008257865905762,
      "step": 892
    },
    {
      "completion_length": 284.7857360839844,
      "epoch": 0.5108695652173914,
      "grad_norm": 0.18537303805351257,
      "kl": 0.17076517641544342,
      "learning_rate": 2.840954276703022e-06,
      "loss": 0.0068,
      "reward": 2.0406973361968994,
      "reward_std": 0.9907272458076477,
      "rewards/reward_function": 2.0406973361968994,
      "step": 893
    },
    {
      "completion_length": 270.39288330078125,
      "epoch": 0.511441647597254,
      "grad_norm": 1.220893383026123,
      "kl": 0.3181988298892975,
      "learning_rate": 2.8360072574122933e-06,
      "loss": 0.0127,
      "reward": 2.186105251312256,
      "reward_std": 0.7045299410820007,
      "rewards/reward_function": 2.186105251312256,
      "step": 894
    },
    {
      "completion_length": 324.0,
      "epoch": 0.5120137299771167,
      "grad_norm": 0.5158973336219788,
      "kl": 0.2519814074039459,
      "learning_rate": 2.8310588978561417e-06,
      "loss": 0.0101,
      "reward": 2.6284842491149902,
      "reward_std": 0.4108975827693939,
      "rewards/reward_function": 2.6284842491149902,
      "step": 895
    },
    {
      "completion_length": 276.0714416503906,
      "epoch": 0.5125858123569794,
      "grad_norm": 0.1941145360469818,
      "kl": 0.15810814499855042,
      "learning_rate": 2.8261092177725786e-06,
      "loss": 0.0063,
      "reward": 2.386840343475342,
      "reward_std": 0.7830825448036194,
      "rewards/reward_function": 2.386840343475342,
      "step": 896
    },
    {
      "completion_length": 326.5714416503906,
      "epoch": 0.5131578947368421,
      "grad_norm": 4.82139778137207,
      "kl": 0.5552802085876465,
      "learning_rate": 2.821158236904883e-06,
      "loss": 0.0222,
      "reward": 1.7370579242706299,
      "reward_std": 0.7992799282073975,
      "rewards/reward_function": 1.7370579242706299,
      "step": 897
    },
    {
      "completion_length": 314.3214416503906,
      "epoch": 0.5137299771167048,
      "grad_norm": 0.21533654630184174,
      "kl": 0.1628369688987732,
      "learning_rate": 2.8162059750015234e-06,
      "loss": 0.0065,
      "reward": 2.220059394836426,
      "reward_std": 0.9455291628837585,
      "rewards/reward_function": 2.220059394836426,
      "step": 898
    },
    {
      "completion_length": 322.3214416503906,
      "epoch": 0.5143020594965675,
      "grad_norm": 0.2705165147781372,
      "kl": 0.5038074851036072,
      "learning_rate": 2.8112524518160774e-06,
      "loss": 0.0202,
      "reward": 2.506678819656372,
      "reward_std": 1.1595467329025269,
      "rewards/reward_function": 2.506678819656372,
      "step": 899
    },
    {
      "completion_length": 350.5357360839844,
      "epoch": 0.5148741418764302,
      "grad_norm": 0.23703180253505707,
      "kl": 0.23509792983531952,
      "learning_rate": 2.8062976871071513e-06,
      "loss": 0.0094,
      "reward": 1.2495816946029663,
      "reward_std": 0.6700268387794495,
      "rewards/reward_function": 1.2495816946029663,
      "step": 900
    },
    {
      "completion_length": 282.6071472167969,
      "epoch": 0.5154462242562929,
      "grad_norm": 0.21918922662734985,
      "kl": 0.18820494413375854,
      "learning_rate": 2.8013417006383078e-06,
      "loss": 0.0075,
      "reward": 1.9755468368530273,
      "reward_std": 0.9088695049285889,
      "rewards/reward_function": 1.9755468368530273,
      "step": 901
    },
    {
      "completion_length": 288.7857360839844,
      "epoch": 0.5160183066361556,
      "grad_norm": 0.3314061462879181,
      "kl": 0.38664162158966064,
      "learning_rate": 2.796384512177981e-06,
      "loss": 0.0155,
      "reward": 1.4428588151931763,
      "reward_std": 0.822701632976532,
      "rewards/reward_function": 1.4428588151931763,
      "step": 902
    },
    {
      "completion_length": 328.1071472167969,
      "epoch": 0.5165903890160183,
      "grad_norm": 0.5161841511726379,
      "kl": 0.5742769241333008,
      "learning_rate": 2.7914261414993983e-06,
      "loss": 0.023,
      "reward": 2.005040407180786,
      "reward_std": 0.820311427116394,
      "rewards/reward_function": 2.005040407180786,
      "step": 903
    },
    {
      "completion_length": 298.1071472167969,
      "epoch": 0.517162471395881,
      "grad_norm": 0.2694889008998871,
      "kl": 0.2267976999282837,
      "learning_rate": 2.786466608380505e-06,
      "loss": 0.0091,
      "reward": 1.8569588661193848,
      "reward_std": 0.9259413480758667,
      "rewards/reward_function": 1.8569588661193848,
      "step": 904
    },
    {
      "completion_length": 328.1785888671875,
      "epoch": 0.5177345537757437,
      "grad_norm": 0.2299042046070099,
      "kl": 0.20561997592449188,
      "learning_rate": 2.7815059326038814e-06,
      "loss": 0.0082,
      "reward": 2.5308852195739746,
      "reward_std": 0.7449536919593811,
      "rewards/reward_function": 2.5308852195739746,
      "step": 905
    },
    {
      "completion_length": 278.4285888671875,
      "epoch": 0.5183066361556065,
      "grad_norm": 0.1789190173149109,
      "kl": 0.133506178855896,
      "learning_rate": 2.7765441339566656e-06,
      "loss": 0.0053,
      "reward": 1.4018681049346924,
      "reward_std": 0.8881992101669312,
      "rewards/reward_function": 1.4018681049346924,
      "step": 906
    },
    {
      "completion_length": 315.3571472167969,
      "epoch": 0.5188787185354691,
      "grad_norm": 0.19214747846126556,
      "kl": 0.14083606004714966,
      "learning_rate": 2.7715812322304764e-06,
      "loss": 0.0056,
      "reward": 2.3394789695739746,
      "reward_std": 0.8824620246887207,
      "rewards/reward_function": 2.3394789695739746,
      "step": 907
    },
    {
      "completion_length": 318.21429443359375,
      "epoch": 0.5194508009153318,
      "grad_norm": 0.20507089793682098,
      "kl": 0.17751914262771606,
      "learning_rate": 2.766617247221331e-06,
      "loss": 0.0071,
      "reward": 1.6843163967132568,
      "reward_std": 1.1679449081420898,
      "rewards/reward_function": 1.6843163967132568,
      "step": 908
    },
    {
      "completion_length": 342.6071472167969,
      "epoch": 0.5200228832951945,
      "grad_norm": 0.31297117471694946,
      "kl": 0.20987924933433533,
      "learning_rate": 2.7616521987295675e-06,
      "loss": 0.0084,
      "reward": 1.7038334608078003,
      "reward_std": 1.03498113155365,
      "rewards/reward_function": 1.7038334608078003,
      "step": 909
    },
    {
      "completion_length": 334.6785888671875,
      "epoch": 0.5205949656750573,
      "grad_norm": 0.18672668933868408,
      "kl": 0.3343299925327301,
      "learning_rate": 2.7566861065597672e-06,
      "loss": 0.0134,
      "reward": 2.305371046066284,
      "reward_std": 0.7630388140678406,
      "rewards/reward_function": 2.305371046066284,
      "step": 910
    },
    {
      "completion_length": 349.8571472167969,
      "epoch": 0.5211670480549199,
      "grad_norm": 1.0887830257415771,
      "kl": 0.6239126920700073,
      "learning_rate": 2.7517189905206736e-06,
      "loss": 0.025,
      "reward": 2.043877601623535,
      "reward_std": 0.6332913637161255,
      "rewards/reward_function": 2.043877601623535,
      "step": 911
    },
    {
      "completion_length": 314.1785888671875,
      "epoch": 0.5217391304347826,
      "grad_norm": 0.19674569368362427,
      "kl": 0.1789151132106781,
      "learning_rate": 2.746750870425114e-06,
      "loss": 0.0072,
      "reward": 2.0411911010742188,
      "reward_std": 1.0113015174865723,
      "rewards/reward_function": 2.0411911010742188,
      "step": 912
    },
    {
      "completion_length": 306.71429443359375,
      "epoch": 0.5223112128146453,
      "grad_norm": 0.2041032910346985,
      "kl": 0.17256924510002136,
      "learning_rate": 2.7417817660899217e-06,
      "loss": 0.0069,
      "reward": 2.00935435295105,
      "reward_std": 0.8305166363716125,
      "rewards/reward_function": 2.00935435295105,
      "step": 913
    },
    {
      "completion_length": 349.2500305175781,
      "epoch": 0.522883295194508,
      "grad_norm": 0.1958686262369156,
      "kl": 0.18622325360774994,
      "learning_rate": 2.736811697335855e-06,
      "loss": 0.0074,
      "reward": 2.1760141849517822,
      "reward_std": 0.8973780274391174,
      "rewards/reward_function": 2.1760141849517822,
      "step": 914
    },
    {
      "completion_length": 316.1071472167969,
      "epoch": 0.5234553775743707,
      "grad_norm": 0.2193753868341446,
      "kl": 0.3887580931186676,
      "learning_rate": 2.7318406839875177e-06,
      "loss": 0.0156,
      "reward": 2.1969830989837646,
      "reward_std": 0.8329300284385681,
      "rewards/reward_function": 2.1969830989837646,
      "step": 915
    },
    {
      "completion_length": 292.4285888671875,
      "epoch": 0.5240274599542334,
      "grad_norm": 0.22898639738559723,
      "kl": 0.16913820803165436,
      "learning_rate": 2.726868745873286e-06,
      "loss": 0.0068,
      "reward": 1.9455881118774414,
      "reward_std": 0.9175885319709778,
      "rewards/reward_function": 1.9455881118774414,
      "step": 916
    },
    {
      "completion_length": 280.75,
      "epoch": 0.5245995423340961,
      "grad_norm": 0.22562840580940247,
      "kl": 0.15807250142097473,
      "learning_rate": 2.72189590282522e-06,
      "loss": 0.0063,
      "reward": 1.5462669134140015,
      "reward_std": 0.9198582172393799,
      "rewards/reward_function": 1.5462669134140015,
      "step": 917
    },
    {
      "completion_length": 298.3214416503906,
      "epoch": 0.5251716247139588,
      "grad_norm": 0.2525348365306854,
      "kl": 0.16116665303707123,
      "learning_rate": 2.7169221746789913e-06,
      "loss": 0.0064,
      "reward": 1.9521127939224243,
      "reward_std": 0.8213635087013245,
      "rewards/reward_function": 1.9521127939224243,
      "step": 918
    },
    {
      "completion_length": 309.25,
      "epoch": 0.5257437070938215,
      "grad_norm": 0.19093555212020874,
      "kl": 0.1533968150615692,
      "learning_rate": 2.711947581273802e-06,
      "loss": 0.0061,
      "reward": 2.036508560180664,
      "reward_std": 0.7052000164985657,
      "rewards/reward_function": 2.036508560180664,
      "step": 919
    },
    {
      "completion_length": 336.4285888671875,
      "epoch": 0.5263157894736842,
      "grad_norm": 0.5645521283149719,
      "kl": 0.8686953783035278,
      "learning_rate": 2.7069721424523067e-06,
      "loss": 0.0347,
      "reward": 1.723772406578064,
      "reward_std": 0.7210212349891663,
      "rewards/reward_function": 1.723772406578064,
      "step": 920
    },
    {
      "completion_length": 299.25,
      "epoch": 0.5268878718535469,
      "grad_norm": 108.82698059082031,
      "kl": 15.020988464355469,
      "learning_rate": 2.7019958780605283e-06,
      "loss": 0.6008,
      "reward": 2.2455036640167236,
      "reward_std": 0.8073368668556213,
      "rewards/reward_function": 2.2455036640167236,
      "step": 921
    },
    {
      "completion_length": 266.14288330078125,
      "epoch": 0.5274599542334096,
      "grad_norm": 0.2984023094177246,
      "kl": 0.19564394652843475,
      "learning_rate": 2.6970188079477883e-06,
      "loss": 0.0078,
      "reward": 1.8910008668899536,
      "reward_std": 0.6598604321479797,
      "rewards/reward_function": 1.8910008668899536,
      "step": 922
    },
    {
      "completion_length": 275.7857360839844,
      "epoch": 0.5280320366132724,
      "grad_norm": 0.2139962762594223,
      "kl": 0.14094752073287964,
      "learning_rate": 2.6920409519666173e-06,
      "loss": 0.0056,
      "reward": 1.7361171245574951,
      "reward_std": 1.0144038200378418,
      "rewards/reward_function": 1.7361171245574951,
      "step": 923
    },
    {
      "completion_length": 270.6785888671875,
      "epoch": 0.528604118993135,
      "grad_norm": 0.2060614824295044,
      "kl": 0.15059271454811096,
      "learning_rate": 2.6870623299726828e-06,
      "loss": 0.006,
      "reward": 2.401585817337036,
      "reward_std": 1.1612986326217651,
      "rewards/reward_function": 2.401585817337036,
      "step": 924
    },
    {
      "completion_length": 278.0,
      "epoch": 0.5291762013729977,
      "grad_norm": 0.9829635620117188,
      "kl": 0.8610355257987976,
      "learning_rate": 2.6820829618247086e-06,
      "loss": 0.0344,
      "reward": 1.9518945217132568,
      "reward_std": 0.8658051490783691,
      "rewards/reward_function": 1.9518945217132568,
      "step": 925
    },
    {
      "completion_length": 321.96429443359375,
      "epoch": 0.5297482837528604,
      "grad_norm": 0.1767490953207016,
      "kl": 0.15017801523208618,
      "learning_rate": 2.6771028673843917e-06,
      "loss": 0.006,
      "reward": 1.677519679069519,
      "reward_std": 0.9089478254318237,
      "rewards/reward_function": 1.677519679069519,
      "step": 926
    },
    {
      "completion_length": 325.9285888671875,
      "epoch": 0.5303203661327232,
      "grad_norm": 0.20040063560009003,
      "kl": 0.16222982108592987,
      "learning_rate": 2.6721220665163284e-06,
      "loss": 0.0065,
      "reward": 1.998309850692749,
      "reward_std": 0.542669415473938,
      "rewards/reward_function": 1.998309850692749,
      "step": 927
    },
    {
      "completion_length": 356.46429443359375,
      "epoch": 0.5308924485125858,
      "grad_norm": 0.3274592459201813,
      "kl": 0.5031225681304932,
      "learning_rate": 2.667140579087933e-06,
      "loss": 0.0201,
      "reward": 2.0517687797546387,
      "reward_std": 1.0637677907943726,
      "rewards/reward_function": 2.0517687797546387,
      "step": 928
    },
    {
      "completion_length": 298.1785888671875,
      "epoch": 0.5314645308924485,
      "grad_norm": 0.18925294280052185,
      "kl": 0.149303138256073,
      "learning_rate": 2.6621584249693577e-06,
      "loss": 0.006,
      "reward": 2.2763423919677734,
      "reward_std": 0.9855844378471375,
      "rewards/reward_function": 2.2763423919677734,
      "step": 929
    },
    {
      "completion_length": 316.0714416503906,
      "epoch": 0.5320366132723112,
      "grad_norm": 0.1934119313955307,
      "kl": 0.16068574786186218,
      "learning_rate": 2.6571756240334136e-06,
      "loss": 0.0064,
      "reward": 2.302855968475342,
      "reward_std": 0.8725823163986206,
      "rewards/reward_function": 2.302855968475342,
      "step": 930
    },
    {
      "completion_length": 322.9285888671875,
      "epoch": 0.532608695652174,
      "grad_norm": 0.21644353866577148,
      "kl": 0.22365084290504456,
      "learning_rate": 2.652192196155493e-06,
      "loss": 0.0089,
      "reward": 2.3917806148529053,
      "reward_std": 0.9737793207168579,
      "rewards/reward_function": 2.3917806148529053,
      "step": 931
    },
    {
      "completion_length": 317.9285888671875,
      "epoch": 0.5331807780320366,
      "grad_norm": 0.2032250165939331,
      "kl": 0.22234299778938293,
      "learning_rate": 2.6472081612134876e-06,
      "loss": 0.0089,
      "reward": 1.8390533924102783,
      "reward_std": 0.9904072880744934,
      "rewards/reward_function": 1.8390533924102783,
      "step": 932
    },
    {
      "completion_length": 295.2857360839844,
      "epoch": 0.5337528604118993,
      "grad_norm": 0.42885860800743103,
      "kl": 0.33825263381004333,
      "learning_rate": 2.6422235390877105e-06,
      "loss": 0.0135,
      "reward": 2.1293249130249023,
      "reward_std": 1.1919046640396118,
      "rewards/reward_function": 2.1293249130249023,
      "step": 933
    },
    {
      "completion_length": 294.1071472167969,
      "epoch": 0.534324942791762,
      "grad_norm": 0.9104035496711731,
      "kl": 0.29878556728363037,
      "learning_rate": 2.637238349660819e-06,
      "loss": 0.012,
      "reward": 1.4494441747665405,
      "reward_std": 0.7017114162445068,
      "rewards/reward_function": 1.4494441747665405,
      "step": 934
    },
    {
      "completion_length": 300.8571472167969,
      "epoch": 0.5348970251716247,
      "grad_norm": 0.22302979230880737,
      "kl": 0.2350396066904068,
      "learning_rate": 2.6322526128177314e-06,
      "loss": 0.0094,
      "reward": 1.7615400552749634,
      "reward_std": 0.8705003261566162,
      "rewards/reward_function": 1.7615400552749634,
      "step": 935
    },
    {
      "completion_length": 333.75,
      "epoch": 0.5354691075514875,
      "grad_norm": 0.21530823409557343,
      "kl": 0.19776687026023865,
      "learning_rate": 2.627266348445549e-06,
      "loss": 0.0079,
      "reward": 1.763790249824524,
      "reward_std": 0.7094086408615112,
      "rewards/reward_function": 1.763790249824524,
      "step": 936
    },
    {
      "completion_length": 289.75,
      "epoch": 0.5360411899313501,
      "grad_norm": 0.18076036870479584,
      "kl": 0.1495174616575241,
      "learning_rate": 2.622279576433479e-06,
      "loss": 0.006,
      "reward": 1.7033934593200684,
      "reward_std": 0.8600678443908691,
      "rewards/reward_function": 1.7033934593200684,
      "step": 937
    },
    {
      "completion_length": 280.64288330078125,
      "epoch": 0.5366132723112128,
      "grad_norm": 0.24105973541736603,
      "kl": 0.1299995630979538,
      "learning_rate": 2.617292316672754e-06,
      "loss": 0.0052,
      "reward": 1.6489492654800415,
      "reward_std": 0.7805511355400085,
      "rewards/reward_function": 1.6489492654800415,
      "step": 938
    },
    {
      "completion_length": 322.25,
      "epoch": 0.5371853546910755,
      "grad_norm": 0.18291842937469482,
      "kl": 0.13381455838680267,
      "learning_rate": 2.6123045890565483e-06,
      "loss": 0.0054,
      "reward": 1.4859169721603394,
      "reward_std": 0.7093538641929626,
      "rewards/reward_function": 1.4859169721603394,
      "step": 939
    },
    {
      "completion_length": 293.3214416503906,
      "epoch": 0.5377574370709383,
      "grad_norm": 1.0492188930511475,
      "kl": 0.3072182536125183,
      "learning_rate": 2.607316413479908e-06,
      "loss": 0.0123,
      "reward": 2.22265625,
      "reward_std": 1.034524917602539,
      "rewards/reward_function": 2.22265625,
      "step": 940
    },
    {
      "completion_length": 302.3571472167969,
      "epoch": 0.5383295194508009,
      "grad_norm": 0.20267429947853088,
      "kl": 0.1524999737739563,
      "learning_rate": 2.602327809839661e-06,
      "loss": 0.0061,
      "reward": 2.0567517280578613,
      "reward_std": 0.9992033243179321,
      "rewards/reward_function": 2.0567517280578613,
      "step": 941
    },
    {
      "completion_length": 303.5714416503906,
      "epoch": 0.5389016018306636,
      "grad_norm": 3.281285524368286,
      "kl": 1.1591461896896362,
      "learning_rate": 2.5973387980343447e-06,
      "loss": 0.0464,
      "reward": 1.45522141456604,
      "reward_std": 0.6133396029472351,
      "rewards/reward_function": 1.45522141456604,
      "step": 942
    },
    {
      "completion_length": 316.71429443359375,
      "epoch": 0.5394736842105263,
      "grad_norm": 0.1858331263065338,
      "kl": 0.1396237015724182,
      "learning_rate": 2.592349397964125e-06,
      "loss": 0.0056,
      "reward": 2.2095603942871094,
      "reward_std": 0.7319374680519104,
      "rewards/reward_function": 2.2095603942871094,
      "step": 943
    },
    {
      "completion_length": 354.0714416503906,
      "epoch": 0.540045766590389,
      "grad_norm": 0.214190274477005,
      "kl": 0.1503533571958542,
      "learning_rate": 2.587359629530717e-06,
      "loss": 0.006,
      "reward": 2.680302858352661,
      "reward_std": 0.915367841720581,
      "rewards/reward_function": 2.680302858352661,
      "step": 944
    },
    {
      "completion_length": 293.9285888671875,
      "epoch": 0.5406178489702517,
      "grad_norm": 0.197563037276268,
      "kl": 0.31232133507728577,
      "learning_rate": 2.582369512637302e-06,
      "loss": 0.0125,
      "reward": 2.152522563934326,
      "reward_std": 1.1442620754241943,
      "rewards/reward_function": 2.152522563934326,
      "step": 945
    },
    {
      "completion_length": 275.64288330078125,
      "epoch": 0.5411899313501144,
      "grad_norm": 0.2575356364250183,
      "kl": 0.12911240756511688,
      "learning_rate": 2.577379067188455e-06,
      "loss": 0.0052,
      "reward": 1.8733941316604614,
      "reward_std": 0.9108062386512756,
      "rewards/reward_function": 1.8733941316604614,
      "step": 946
    },
    {
      "completion_length": 327.21429443359375,
      "epoch": 0.5417620137299771,
      "grad_norm": 13.221973419189453,
      "kl": 1.5138881206512451,
      "learning_rate": 2.5723883130900608e-06,
      "loss": 0.0606,
      "reward": 1.772392988204956,
      "reward_std": 0.9499253034591675,
      "rewards/reward_function": 1.772392988204956,
      "step": 947
    },
    {
      "completion_length": 315.39288330078125,
      "epoch": 0.5423340961098398,
      "grad_norm": 0.20120497047901154,
      "kl": 0.16237466037273407,
      "learning_rate": 2.5673972702492316e-06,
      "loss": 0.0065,
      "reward": 2.692049741744995,
      "reward_std": 0.8480951189994812,
      "rewards/reward_function": 2.692049741744995,
      "step": 948
    },
    {
      "completion_length": 317.5714416503906,
      "epoch": 0.5429061784897025,
      "grad_norm": 3.208444595336914,
      "kl": 0.9736066460609436,
      "learning_rate": 2.562405958574238e-06,
      "loss": 0.0389,
      "reward": 2.9331469535827637,
      "reward_std": 0.5517695546150208,
      "rewards/reward_function": 2.9331469535827637,
      "step": 949
    },
    {
      "completion_length": 268.1071472167969,
      "epoch": 0.5434782608695652,
      "grad_norm": 0.20534805953502655,
      "kl": 0.16129174828529358,
      "learning_rate": 2.5574143979744164e-06,
      "loss": 0.0065,
      "reward": 1.7062658071517944,
      "reward_std": 1.0325053930282593,
      "rewards/reward_function": 1.7062658071517944,
      "step": 950
    },
    {
      "completion_length": 334.3571472167969,
      "epoch": 0.5440503432494279,
      "grad_norm": 0.18682649731636047,
      "kl": 0.16327594220638275,
      "learning_rate": 2.552422608360099e-06,
      "loss": 0.0065,
      "reward": 2.0748023986816406,
      "reward_std": 1.238512635231018,
      "rewards/reward_function": 2.0748023986816406,
      "step": 951
    },
    {
      "completion_length": 308.4285888671875,
      "epoch": 0.5446224256292906,
      "grad_norm": 1.0867397785186768,
      "kl": 0.42517679929733276,
      "learning_rate": 2.5474306096425333e-06,
      "loss": 0.017,
      "reward": 2.256049156188965,
      "reward_std": 0.8723558187484741,
      "rewards/reward_function": 2.256049156188965,
      "step": 952
    },
    {
      "completion_length": 331.25,
      "epoch": 0.5451945080091534,
      "grad_norm": 0.55674809217453,
      "kl": 0.35854169726371765,
      "learning_rate": 2.5424384217337964e-06,
      "loss": 0.0143,
      "reward": 2.2674388885498047,
      "reward_std": 0.825494647026062,
      "rewards/reward_function": 2.2674388885498047,
      "step": 953
    },
    {
      "completion_length": 343.9285888671875,
      "epoch": 0.545766590389016,
      "grad_norm": 0.23536553978919983,
      "kl": 0.3397100567817688,
      "learning_rate": 2.5374460645467253e-06,
      "loss": 0.0136,
      "reward": 2.0097944736480713,
      "reward_std": 0.9695807099342346,
      "rewards/reward_function": 2.0097944736480713,
      "step": 954
    },
    {
      "completion_length": 321.64288330078125,
      "epoch": 0.5463386727688787,
      "grad_norm": 2.651715040206909,
      "kl": 0.8927242755889893,
      "learning_rate": 2.5324535579948274e-06,
      "loss": 0.0357,
      "reward": 2.2809033393859863,
      "reward_std": 0.9490068554878235,
      "rewards/reward_function": 2.2809033393859863,
      "step": 955
    },
    {
      "completion_length": 334.0714416503906,
      "epoch": 0.5469107551487414,
      "grad_norm": 0.18931110203266144,
      "kl": 0.15052980184555054,
      "learning_rate": 2.5274609219922093e-06,
      "loss": 0.006,
      "reward": 2.342000722885132,
      "reward_std": 1.2853225469589233,
      "rewards/reward_function": 2.342000722885132,
      "step": 956
    },
    {
      "completion_length": 318.7857360839844,
      "epoch": 0.5474828375286042,
      "grad_norm": 0.787505567073822,
      "kl": 0.4072675406932831,
      "learning_rate": 2.5224681764534914e-06,
      "loss": 0.0163,
      "reward": 1.8551183938980103,
      "reward_std": 0.9390087127685547,
      "rewards/reward_function": 1.8551183938980103,
      "step": 957
    },
    {
      "completion_length": 308.46429443359375,
      "epoch": 0.5480549199084668,
      "grad_norm": 0.38508620858192444,
      "kl": 0.3399064838886261,
      "learning_rate": 2.5174753412937335e-06,
      "loss": 0.0136,
      "reward": 1.9832805395126343,
      "reward_std": 1.133129596710205,
      "rewards/reward_function": 1.9832805395126343,
      "step": 958
    },
    {
      "completion_length": 312.6071472167969,
      "epoch": 0.5486270022883295,
      "grad_norm": 791.404541015625,
      "kl": 231.45681762695312,
      "learning_rate": 2.5124824364283517e-06,
      "loss": 9.2583,
      "reward": 2.127643585205078,
      "reward_std": 1.1600841283798218,
      "rewards/reward_function": 2.127643585205078,
      "step": 959
    },
    {
      "completion_length": 285.1071472167969,
      "epoch": 0.5491990846681922,
      "grad_norm": 0.32602086663246155,
      "kl": 0.22927340865135193,
      "learning_rate": 2.5074894817730387e-06,
      "loss": 0.0092,
      "reward": 2.2716026306152344,
      "reward_std": 1.017473578453064,
      "rewards/reward_function": 2.2716026306152344,
      "step": 960
    },
    {
      "completion_length": 298.1071472167969,
      "epoch": 0.549771167048055,
      "grad_norm": 1.0374300479888916,
      "kl": 0.4304857850074768,
      "learning_rate": 2.502496497243689e-06,
      "loss": 0.0172,
      "reward": 2.5011842250823975,
      "reward_std": 0.8925716280937195,
      "rewards/reward_function": 2.5011842250823975,
      "step": 961
    },
    {
      "completion_length": 306.21429443359375,
      "epoch": 0.5503432494279176,
      "grad_norm": 0.18165358901023865,
      "kl": 0.14266559481620789,
      "learning_rate": 2.4975035027563123e-06,
      "loss": 0.0057,
      "reward": 2.024153232574463,
      "reward_std": 1.251166582107544,
      "rewards/reward_function": 2.024153232574463,
      "step": 962
    },
    {
      "completion_length": 334.3214416503906,
      "epoch": 0.5509153318077803,
      "grad_norm": 0.17951355874538422,
      "kl": 0.12764668464660645,
      "learning_rate": 2.492510518226962e-06,
      "loss": 0.0051,
      "reward": 2.295952558517456,
      "reward_std": 0.8377647995948792,
      "rewards/reward_function": 2.295952558517456,
      "step": 963
    },
    {
      "completion_length": 356.46429443359375,
      "epoch": 0.551487414187643,
      "grad_norm": 0.21070684492588043,
      "kl": 0.19286149740219116,
      "learning_rate": 2.4875175635716487e-06,
      "loss": 0.0077,
      "reward": 1.9713971614837646,
      "reward_std": 1.070521354675293,
      "rewards/reward_function": 1.9713971614837646,
      "step": 964
    },
    {
      "completion_length": 331.46429443359375,
      "epoch": 0.5520594965675057,
      "grad_norm": 0.2109554409980774,
      "kl": 0.27632132172584534,
      "learning_rate": 2.4825246587062673e-06,
      "loss": 0.0111,
      "reward": 1.348264455795288,
      "reward_std": 0.528099536895752,
      "rewards/reward_function": 1.348264455795288,
      "step": 965
    },
    {
      "completion_length": 272.5714416503906,
      "epoch": 0.5526315789473685,
      "grad_norm": 0.26093775033950806,
      "kl": 0.3208455741405487,
      "learning_rate": 2.4775318235465094e-06,
      "loss": 0.0128,
      "reward": 2.1988182067871094,
      "reward_std": 1.0762161016464233,
      "rewards/reward_function": 2.1988182067871094,
      "step": 966
    },
    {
      "completion_length": 303.46429443359375,
      "epoch": 0.5532036613272311,
      "grad_norm": 0.20761121809482574,
      "kl": 0.1604282110929489,
      "learning_rate": 2.472539078007791e-06,
      "loss": 0.0064,
      "reward": 2.2133378982543945,
      "reward_std": 1.0264358520507812,
      "rewards/reward_function": 2.2133378982543945,
      "step": 967
    },
    {
      "completion_length": 349.0000305175781,
      "epoch": 0.5537757437070938,
      "grad_norm": 0.18160149455070496,
      "kl": 0.16677340865135193,
      "learning_rate": 2.4675464420051735e-06,
      "loss": 0.0067,
      "reward": 2.0083742141723633,
      "reward_std": 1.0880887508392334,
      "rewards/reward_function": 2.0083742141723633,
      "step": 968
    },
    {
      "completion_length": 324.5357360839844,
      "epoch": 0.5543478260869565,
      "grad_norm": 0.5348715782165527,
      "kl": 0.5892442464828491,
      "learning_rate": 2.4625539354532755e-06,
      "loss": 0.0236,
      "reward": 2.2008252143859863,
      "reward_std": 1.1632028818130493,
      "rewards/reward_function": 2.2008252143859863,
      "step": 969
    },
    {
      "completion_length": 309.8571472167969,
      "epoch": 0.5549199084668193,
      "grad_norm": 0.550473153591156,
      "kl": 0.26897963881492615,
      "learning_rate": 2.457561578266204e-06,
      "loss": 0.0108,
      "reward": 1.5434519052505493,
      "reward_std": 1.1616437435150146,
      "rewards/reward_function": 1.5434519052505493,
      "step": 970
    },
    {
      "completion_length": 326.14288330078125,
      "epoch": 0.5554919908466819,
      "grad_norm": 0.33748704195022583,
      "kl": 0.22323979437351227,
      "learning_rate": 2.4525693903574676e-06,
      "loss": 0.0089,
      "reward": 1.9328892230987549,
      "reward_std": 0.9540339112281799,
      "rewards/reward_function": 1.9328892230987549,
      "step": 971
    },
    {
      "completion_length": 269.5,
      "epoch": 0.5560640732265446,
      "grad_norm": 0.2135508507490158,
      "kl": 0.14957712590694427,
      "learning_rate": 2.4475773916399015e-06,
      "loss": 0.006,
      "reward": 2.4423329830169678,
      "reward_std": 0.8509246706962585,
      "rewards/reward_function": 2.4423329830169678,
      "step": 972
    },
    {
      "completion_length": 319.3214416503906,
      "epoch": 0.5566361556064073,
      "grad_norm": 2.082305431365967,
      "kl": 0.9945375323295593,
      "learning_rate": 2.4425856020255844e-06,
      "loss": 0.0398,
      "reward": 1.845250129699707,
      "reward_std": 0.6708447933197021,
      "rewards/reward_function": 1.845250129699707,
      "step": 973
    },
    {
      "completion_length": 348.2500305175781,
      "epoch": 0.5572082379862701,
      "grad_norm": 0.1825430542230606,
      "kl": 0.14374065399169922,
      "learning_rate": 2.4375940414257637e-06,
      "loss": 0.0057,
      "reward": 1.9925810098648071,
      "reward_std": 1.1306254863739014,
      "rewards/reward_function": 1.9925810098648071,
      "step": 974
    },
    {
      "completion_length": 306.0,
      "epoch": 0.5577803203661327,
      "grad_norm": 0.1863701045513153,
      "kl": 0.1323855072259903,
      "learning_rate": 2.4326027297507688e-06,
      "loss": 0.0053,
      "reward": 2.0558969974517822,
      "reward_std": 1.037285566329956,
      "rewards/reward_function": 2.0558969974517822,
      "step": 975
    },
    {
      "completion_length": 286.46429443359375,
      "epoch": 0.5583524027459954,
      "grad_norm": 0.1999882608652115,
      "kl": 0.142306387424469,
      "learning_rate": 2.4276116869099396e-06,
      "loss": 0.0057,
      "reward": 2.532799243927002,
      "reward_std": 1.0187888145446777,
      "rewards/reward_function": 2.532799243927002,
      "step": 976
    },
    {
      "completion_length": 306.5357360839844,
      "epoch": 0.5589244851258581,
      "grad_norm": 0.2193925678730011,
      "kl": 0.14679914712905884,
      "learning_rate": 2.4226209328115457e-06,
      "loss": 0.0059,
      "reward": 2.0467429161071777,
      "reward_std": 0.9370306134223938,
      "rewards/reward_function": 2.0467429161071777,
      "step": 977
    },
    {
      "completion_length": 336.8571472167969,
      "epoch": 0.5594965675057209,
      "grad_norm": 0.25908854603767395,
      "kl": 0.17298288643360138,
      "learning_rate": 2.4176304873626983e-06,
      "loss": 0.0069,
      "reward": 1.9627119302749634,
      "reward_std": 1.1564587354660034,
      "rewards/reward_function": 1.9627119302749634,
      "step": 978
    },
    {
      "completion_length": 294.71429443359375,
      "epoch": 0.5600686498855835,
      "grad_norm": 1.1439634561538696,
      "kl": 0.3069126307964325,
      "learning_rate": 2.4126403704692843e-06,
      "loss": 0.0123,
      "reward": 2.285231590270996,
      "reward_std": 1.0323759317398071,
      "rewards/reward_function": 2.285231590270996,
      "step": 979
    },
    {
      "completion_length": 300.89288330078125,
      "epoch": 0.5606407322654462,
      "grad_norm": 0.19557146728038788,
      "kl": 0.15446589887142181,
      "learning_rate": 2.4076506020358752e-06,
      "loss": 0.0062,
      "reward": 2.201336622238159,
      "reward_std": 0.8785970211029053,
      "rewards/reward_function": 2.201336622238159,
      "step": 980
    },
    {
      "completion_length": 334.6071472167969,
      "epoch": 0.5612128146453089,
      "grad_norm": 0.24162332713603973,
      "kl": 0.1618763953447342,
      "learning_rate": 2.402661201965656e-06,
      "loss": 0.0065,
      "reward": 2.441218137741089,
      "reward_std": 1.237820029258728,
      "rewards/reward_function": 2.441218137741089,
      "step": 981
    },
    {
      "completion_length": 311.21429443359375,
      "epoch": 0.5617848970251716,
      "grad_norm": 0.3233824372291565,
      "kl": 0.1583796739578247,
      "learning_rate": 2.3976721901603396e-06,
      "loss": 0.0063,
      "reward": 2.4580259323120117,
      "reward_std": 1.1514509916305542,
      "rewards/reward_function": 2.4580259323120117,
      "step": 982
    },
    {
      "completion_length": 332.8571472167969,
      "epoch": 0.5623569794050344,
      "grad_norm": 0.1966945379972458,
      "kl": 0.1742265820503235,
      "learning_rate": 2.3926835865200934e-06,
      "loss": 0.007,
      "reward": 1.8115129470825195,
      "reward_std": 0.6199323534965515,
      "rewards/reward_function": 1.8115129470825195,
      "step": 983
    },
    {
      "completion_length": 332.64288330078125,
      "epoch": 0.562929061784897,
      "grad_norm": 0.2807498872280121,
      "kl": 0.1869150549173355,
      "learning_rate": 2.387695410943452e-06,
      "loss": 0.0075,
      "reward": 2.4849042892456055,
      "reward_std": 1.0071088075637817,
      "rewards/reward_function": 2.4849042892456055,
      "step": 984
    },
    {
      "completion_length": 330.14288330078125,
      "epoch": 0.5635011441647597,
      "grad_norm": 0.19645382463932037,
      "kl": 0.15087464451789856,
      "learning_rate": 2.382707683327246e-06,
      "loss": 0.006,
      "reward": 1.8893910646438599,
      "reward_std": 1.3086683750152588,
      "rewards/reward_function": 1.8893910646438599,
      "step": 985
    },
    {
      "completion_length": 338.7857360839844,
      "epoch": 0.5640732265446224,
      "grad_norm": 0.20251227915287018,
      "kl": 0.17656566202640533,
      "learning_rate": 2.3777204235665217e-06,
      "loss": 0.0071,
      "reward": 1.437700629234314,
      "reward_std": 1.0101406574249268,
      "rewards/reward_function": 1.437700629234314,
      "step": 986
    },
    {
      "completion_length": 338.6071472167969,
      "epoch": 0.5646453089244852,
      "grad_norm": 0.21496835350990295,
      "kl": 0.13344793021678925,
      "learning_rate": 2.3727336515544516e-06,
      "loss": 0.0053,
      "reward": 2.234539747238159,
      "reward_std": 0.36063066124916077,
      "rewards/reward_function": 2.234539747238159,
      "step": 987
    },
    {
      "completion_length": 324.1785888671875,
      "epoch": 0.5652173913043478,
      "grad_norm": 0.3320208489894867,
      "kl": 0.23384493589401245,
      "learning_rate": 2.36774738718227e-06,
      "loss": 0.0094,
      "reward": 2.02480411529541,
      "reward_std": 1.0886099338531494,
      "rewards/reward_function": 2.02480411529541,
      "step": 988
    },
    {
      "completion_length": 340.7857360839844,
      "epoch": 0.5657894736842105,
      "grad_norm": 0.17748446762561798,
      "kl": 0.12978079915046692,
      "learning_rate": 2.3627616503391813e-06,
      "loss": 0.0052,
      "reward": 1.9909427165985107,
      "reward_std": 1.153581976890564,
      "rewards/reward_function": 1.9909427165985107,
      "step": 989
    },
    {
      "completion_length": 318.3571472167969,
      "epoch": 0.5663615560640732,
      "grad_norm": 0.17102119326591492,
      "kl": 0.16289015114307404,
      "learning_rate": 2.3577764609122903e-06,
      "loss": 0.0065,
      "reward": 2.485705852508545,
      "reward_std": 0.4646953344345093,
      "rewards/reward_function": 2.485705852508545,
      "step": 990
    },
    {
      "completion_length": 353.64288330078125,
      "epoch": 0.566933638443936,
      "grad_norm": 0.17140349745750427,
      "kl": 0.15067175030708313,
      "learning_rate": 2.3527918387865133e-06,
      "loss": 0.006,
      "reward": 2.1191086769104004,
      "reward_std": 0.9076591730117798,
      "rewards/reward_function": 2.1191086769104004,
      "step": 991
    },
    {
      "completion_length": 294.5357360839844,
      "epoch": 0.5675057208237986,
      "grad_norm": 0.2576044499874115,
      "kl": 0.17300499975681305,
      "learning_rate": 2.3478078038445073e-06,
      "loss": 0.0069,
      "reward": 1.4818031787872314,
      "reward_std": 0.5684291124343872,
      "rewards/reward_function": 1.4818031787872314,
      "step": 992
    },
    {
      "completion_length": 328.4285888671875,
      "epoch": 0.5680778032036613,
      "grad_norm": 8.070863723754883,
      "kl": 1.560430884361267,
      "learning_rate": 2.3428243759665873e-06,
      "loss": 0.0624,
      "reward": 2.6016769409179688,
      "reward_std": 0.7864677906036377,
      "rewards/reward_function": 2.6016769409179688,
      "step": 993
    },
    {
      "completion_length": 345.6071472167969,
      "epoch": 0.568649885583524,
      "grad_norm": 0.1825365126132965,
      "kl": 0.15494655072689056,
      "learning_rate": 2.3378415750306423e-06,
      "loss": 0.0062,
      "reward": 2.471651554107666,
      "reward_std": 1.229562759399414,
      "rewards/reward_function": 2.471651554107666,
      "step": 994
    },
    {
      "completion_length": 291.3214416503906,
      "epoch": 0.5692219679633868,
      "grad_norm": 0.2086450606584549,
      "kl": 0.13215303421020508,
      "learning_rate": 2.3328594209120676e-06,
      "loss": 0.0053,
      "reward": 1.6447961330413818,
      "reward_std": 0.8625169396400452,
      "rewards/reward_function": 1.6447961330413818,
      "step": 995
    },
    {
      "completion_length": 334.3571472167969,
      "epoch": 0.5697940503432495,
      "grad_norm": 0.21304139494895935,
      "kl": 0.16743415594100952,
      "learning_rate": 2.3278779334836724e-06,
      "loss": 0.0067,
      "reward": 1.9673043489456177,
      "reward_std": 1.0222128629684448,
      "rewards/reward_function": 1.9673043489456177,
      "step": 996
    },
    {
      "completion_length": 271.64288330078125,
      "epoch": 0.5703661327231121,
      "grad_norm": 0.2113388329744339,
      "kl": 0.16681046783924103,
      "learning_rate": 2.3228971326156096e-06,
      "loss": 0.0067,
      "reward": 1.7290345430374146,
      "reward_std": 0.9562646746635437,
      "rewards/reward_function": 1.7290345430374146,
      "step": 997
    },
    {
      "completion_length": 349.7857360839844,
      "epoch": 0.5709382151029748,
      "grad_norm": 0.19354139268398285,
      "kl": 0.16759732365608215,
      "learning_rate": 2.3179170381752922e-06,
      "loss": 0.0067,
      "reward": 2.4231948852539062,
      "reward_std": 1.0988638401031494,
      "rewards/reward_function": 2.4231948852539062,
      "step": 998
    },
    {
      "completion_length": 324.21429443359375,
      "epoch": 0.5715102974828375,
      "grad_norm": 0.2949443757534027,
      "kl": 0.31311333179473877,
      "learning_rate": 2.3129376700273176e-06,
      "loss": 0.0125,
      "reward": 2.1395769119262695,
      "reward_std": 0.9058817028999329,
      "rewards/reward_function": 2.1395769119262695,
      "step": 999
    },
    {
      "completion_length": 327.8214416503906,
      "epoch": 0.5720823798627003,
      "grad_norm": 1.8125606775283813,
      "kl": 0.9317894577980042,
      "learning_rate": 2.307959048033383e-06,
      "loss": 0.0373,
      "reward": 2.4942634105682373,
      "reward_std": 0.7756945490837097,
      "rewards/reward_function": 2.4942634105682373,
      "step": 1000
    },
    {
      "completion_length": 337.89288330078125,
      "epoch": 0.5726544622425629,
      "grad_norm": 0.1912020742893219,
      "kl": 0.232317715883255,
      "learning_rate": 2.3029811920522117e-06,
      "loss": 0.0093,
      "reward": 2.179244041442871,
      "reward_std": 1.28500235080719,
      "rewards/reward_function": 2.179244041442871,
      "step": 1001
    },
    {
      "completion_length": 311.0357360839844,
      "epoch": 0.5732265446224256,
      "grad_norm": 0.2844683825969696,
      "kl": 0.20750585198402405,
      "learning_rate": 2.298004121939472e-06,
      "loss": 0.0083,
      "reward": 1.3182306289672852,
      "reward_std": 0.834410548210144,
      "rewards/reward_function": 1.3182306289672852,
      "step": 1002
    },
    {
      "completion_length": 316.7857360839844,
      "epoch": 0.5737986270022883,
      "grad_norm": 8.552783966064453,
      "kl": 0.42681992053985596,
      "learning_rate": 2.2930278575476937e-06,
      "loss": 0.0171,
      "reward": 1.808375597000122,
      "reward_std": 0.8161570429801941,
      "rewards/reward_function": 1.808375597000122,
      "step": 1003
    },
    {
      "completion_length": 346.89288330078125,
      "epoch": 0.5743707093821511,
      "grad_norm": 1.0847864151000977,
      "kl": 0.3395709693431854,
      "learning_rate": 2.2880524187261988e-06,
      "loss": 0.0136,
      "reward": 1.949497938156128,
      "reward_std": 1.018181324005127,
      "rewards/reward_function": 1.949497938156128,
      "step": 1004
    },
    {
      "completion_length": 318.71429443359375,
      "epoch": 0.5749427917620137,
      "grad_norm": 0.27824223041534424,
      "kl": 0.29197758436203003,
      "learning_rate": 2.283077825321009e-06,
      "loss": 0.0117,
      "reward": 1.5234626531600952,
      "reward_std": 0.9431173205375671,
      "rewards/reward_function": 1.5234626531600952,
      "step": 1005
    },
    {
      "completion_length": 325.5,
      "epoch": 0.5755148741418764,
      "grad_norm": 0.3443797826766968,
      "kl": 0.2107534259557724,
      "learning_rate": 2.278104097174781e-06,
      "loss": 0.0084,
      "reward": 2.2862727642059326,
      "reward_std": 0.8010655045509338,
      "rewards/reward_function": 2.2862727642059326,
      "step": 1006
    },
    {
      "completion_length": 284.0,
      "epoch": 0.5760869565217391,
      "grad_norm": 0.24215635657310486,
      "kl": 0.1610656976699829,
      "learning_rate": 2.2731312541267144e-06,
      "loss": 0.0064,
      "reward": 2.1232008934020996,
      "reward_std": 0.6035890579223633,
      "rewards/reward_function": 2.1232008934020996,
      "step": 1007
    },
    {
      "completion_length": 326.0714416503906,
      "epoch": 0.5766590389016019,
      "grad_norm": 0.20047397911548615,
      "kl": 0.14770233631134033,
      "learning_rate": 2.2681593160124827e-06,
      "loss": 0.0059,
      "reward": 2.396878242492676,
      "reward_std": 0.9996731877326965,
      "rewards/reward_function": 2.396878242492676,
      "step": 1008
    },
    {
      "completion_length": 270.39288330078125,
      "epoch": 0.5772311212814645,
      "grad_norm": 0.22728164494037628,
      "kl": 0.15712690353393555,
      "learning_rate": 2.263188302664146e-06,
      "loss": 0.0063,
      "reward": 1.9025800228118896,
      "reward_std": 0.749031662940979,
      "rewards/reward_function": 1.9025800228118896,
      "step": 1009
    },
    {
      "completion_length": 348.7857360839844,
      "epoch": 0.5778032036613272,
      "grad_norm": 0.1978304237127304,
      "kl": 0.15696656703948975,
      "learning_rate": 2.2582182339100787e-06,
      "loss": 0.0063,
      "reward": 2.152287006378174,
      "reward_std": 0.9213839173316956,
      "rewards/reward_function": 2.152287006378174,
      "step": 1010
    },
    {
      "completion_length": 305.1071472167969,
      "epoch": 0.5783752860411899,
      "grad_norm": 0.19917714595794678,
      "kl": 0.17782394587993622,
      "learning_rate": 2.253249129574887e-06,
      "loss": 0.0071,
      "reward": 1.9694836139678955,
      "reward_std": 1.3379676342010498,
      "rewards/reward_function": 1.9694836139678955,
      "step": 1011
    },
    {
      "completion_length": 364.64288330078125,
      "epoch": 0.5789473684210527,
      "grad_norm": 1.3129031658172607,
      "kl": 0.4905127286911011,
      "learning_rate": 2.2482810094793264e-06,
      "loss": 0.0196,
      "reward": 1.8586671352386475,
      "reward_std": 0.9371863007545471,
      "rewards/reward_function": 1.8586671352386475,
      "step": 1012
    },
    {
      "completion_length": 331.14288330078125,
      "epoch": 0.5795194508009154,
      "grad_norm": 0.1791488379240036,
      "kl": 0.14149662852287292,
      "learning_rate": 2.2433138934402336e-06,
      "loss": 0.0057,
      "reward": 1.502926230430603,
      "reward_std": 1.0278675556182861,
      "rewards/reward_function": 1.502926230430603,
      "step": 1013
    },
    {
      "completion_length": 308.75,
      "epoch": 0.580091533180778,
      "grad_norm": 14.473724365234375,
      "kl": 0.4222281575202942,
      "learning_rate": 2.2383478012704334e-06,
      "loss": 0.0169,
      "reward": 2.133173942565918,
      "reward_std": 1.0202759504318237,
      "rewards/reward_function": 2.133173942565918,
      "step": 1014
    },
    {
      "completion_length": 312.9285888671875,
      "epoch": 0.5806636155606407,
      "grad_norm": 0.3098878562450409,
      "kl": 0.249923974275589,
      "learning_rate": 2.23338275277867e-06,
      "loss": 0.01,
      "reward": 2.585780382156372,
      "reward_std": 1.066593885421753,
      "rewards/reward_function": 2.585780382156372,
      "step": 1015
    },
    {
      "completion_length": 353.5357360839844,
      "epoch": 0.5812356979405034,
      "grad_norm": 0.1935473382472992,
      "kl": 0.2568837106227875,
      "learning_rate": 2.2284187677695245e-06,
      "loss": 0.0103,
      "reward": 2.4201366901397705,
      "reward_std": 0.8679095506668091,
      "rewards/reward_function": 2.4201366901397705,
      "step": 1016
    },
    {
      "completion_length": 295.75,
      "epoch": 0.5818077803203662,
      "grad_norm": 1.428004264831543,
      "kl": 0.5135065913200378,
      "learning_rate": 2.2234558660433352e-06,
      "loss": 0.0205,
      "reward": 2.12947154045105,
      "reward_std": 0.7373562455177307,
      "rewards/reward_function": 2.12947154045105,
      "step": 1017
    },
    {
      "completion_length": 290.4285888671875,
      "epoch": 0.5823798627002288,
      "grad_norm": 1.2860522270202637,
      "kl": 0.3030727505683899,
      "learning_rate": 2.21849406739612e-06,
      "loss": 0.0121,
      "reward": 1.954169750213623,
      "reward_std": 0.6427761316299438,
      "rewards/reward_function": 1.954169750213623,
      "step": 1018
    },
    {
      "completion_length": 352.64288330078125,
      "epoch": 0.5829519450800915,
      "grad_norm": 7.306486129760742,
      "kl": 0.7503597140312195,
      "learning_rate": 2.2135333916194955e-06,
      "loss": 0.03,
      "reward": 2.367323637008667,
      "reward_std": 0.8646979331970215,
      "rewards/reward_function": 2.367323637008667,
      "step": 1019
    },
    {
      "completion_length": 339.8571472167969,
      "epoch": 0.5835240274599542,
      "grad_norm": 0.4326465129852295,
      "kl": 0.17359229922294617,
      "learning_rate": 2.2085738585006026e-06,
      "loss": 0.0069,
      "reward": 2.1216554641723633,
      "reward_std": 0.6415894627571106,
      "rewards/reward_function": 2.1216554641723633,
      "step": 1020
    },
    {
      "completion_length": 297.2857360839844,
      "epoch": 0.584096109839817,
      "grad_norm": 0.5906347036361694,
      "kl": 0.18415533006191254,
      "learning_rate": 2.203615487822019e-06,
      "loss": 0.0074,
      "reward": 1.7615400552749634,
      "reward_std": 0.936487078666687,
      "rewards/reward_function": 1.7615400552749634,
      "step": 1021
    },
    {
      "completion_length": 302.5,
      "epoch": 0.5846681922196796,
      "grad_norm": 0.1828601062297821,
      "kl": 0.14322859048843384,
      "learning_rate": 2.1986582993616926e-06,
      "loss": 0.0057,
      "reward": 1.3719488382339478,
      "reward_std": 0.5705829858779907,
      "rewards/reward_function": 1.3719488382339478,
      "step": 1022
    },
    {
      "completion_length": 302.9285888671875,
      "epoch": 0.5852402745995423,
      "grad_norm": 0.2272598296403885,
      "kl": 0.13672207295894623,
      "learning_rate": 2.1937023128928496e-06,
      "loss": 0.0055,
      "reward": 2.123927116394043,
      "reward_std": 0.5348818302154541,
      "rewards/reward_function": 2.123927116394043,
      "step": 1023
    },
    {
      "completion_length": 314.3214416503906,
      "epoch": 0.585812356979405,
      "grad_norm": 0.2888711988925934,
      "kl": 0.20177675783634186,
      "learning_rate": 2.1887475481839243e-06,
      "loss": 0.0081,
      "reward": 1.589815378189087,
      "reward_std": 0.9701445698738098,
      "rewards/reward_function": 1.589815378189087,
      "step": 1024
    },
    {
      "completion_length": 284.7857360839844,
      "epoch": 0.5863844393592678,
      "grad_norm": 0.18806390464305878,
      "kl": 0.16117139160633087,
      "learning_rate": 2.183794024998477e-06,
      "loss": 0.0064,
      "reward": 2.4833984375,
      "reward_std": 0.8567025661468506,
      "rewards/reward_function": 2.4833984375,
      "step": 1025
    },
    {
      "completion_length": 277.46429443359375,
      "epoch": 0.5869565217391305,
      "grad_norm": 0.21700099110603333,
      "kl": 0.17878685891628265,
      "learning_rate": 2.1788417630951173e-06,
      "loss": 0.0072,
      "reward": 2.393078565597534,
      "reward_std": 0.9306991100311279,
      "rewards/reward_function": 2.393078565597534,
      "step": 1026
    },
    {
      "completion_length": 361.0714416503906,
      "epoch": 0.5875286041189931,
      "grad_norm": 2.8886055946350098,
      "kl": 0.2587167024612427,
      "learning_rate": 2.173890782227422e-06,
      "loss": 0.0103,
      "reward": 2.313011884689331,
      "reward_std": 1.42891263961792,
      "rewards/reward_function": 2.313011884689331,
      "step": 1027
    },
    {
      "completion_length": 298.96429443359375,
      "epoch": 0.5881006864988558,
      "grad_norm": 0.767573893070221,
      "kl": 0.3155961036682129,
      "learning_rate": 2.1689411021438587e-06,
      "loss": 0.0126,
      "reward": 1.670512080192566,
      "reward_std": 0.7317466139793396,
      "rewards/reward_function": 1.670512080192566,
      "step": 1028
    },
    {
      "completion_length": 339.46429443359375,
      "epoch": 0.5886727688787186,
      "grad_norm": 1.750015139579773,
      "kl": 0.6226204633712769,
      "learning_rate": 2.1639927425877075e-06,
      "loss": 0.0249,
      "reward": 2.517667770385742,
      "reward_std": 1.0562007427215576,
      "rewards/reward_function": 2.517667770385742,
      "step": 1029
    },
    {
      "completion_length": 343.64288330078125,
      "epoch": 0.5892448512585813,
      "grad_norm": 0.19668224453926086,
      "kl": 0.1756916344165802,
      "learning_rate": 2.159045723296978e-06,
      "loss": 0.007,
      "reward": 2.244999408721924,
      "reward_std": 0.8807112574577332,
      "rewards/reward_function": 2.244999408721924,
      "step": 1030
    },
    {
      "completion_length": 328.1071472167969,
      "epoch": 0.5898169336384439,
      "grad_norm": 5.987061023712158,
      "kl": 1.1780160665512085,
      "learning_rate": 2.15410006400434e-06,
      "loss": 0.0471,
      "reward": 1.9675230979919434,
      "reward_std": 1.093263864517212,
      "rewards/reward_function": 1.9675230979919434,
      "step": 1031
    },
    {
      "completion_length": 362.6785888671875,
      "epoch": 0.5903890160183066,
      "grad_norm": 0.7567712068557739,
      "kl": 0.2949730157852173,
      "learning_rate": 2.14915578443703e-06,
      "loss": 0.0118,
      "reward": 1.7581632137298584,
      "reward_std": 0.8129567503929138,
      "rewards/reward_function": 1.7581632137298584,
      "step": 1032
    },
    {
      "completion_length": 326.8571472167969,
      "epoch": 0.5909610983981693,
      "grad_norm": 0.2130144238471985,
      "kl": 0.1669134646654129,
      "learning_rate": 2.1442129043167877e-06,
      "loss": 0.0067,
      "reward": 1.7175947427749634,
      "reward_std": 0.7818310856819153,
      "rewards/reward_function": 1.7175947427749634,
      "step": 1033
    },
    {
      "completion_length": 334.5,
      "epoch": 0.5915331807780321,
      "grad_norm": 0.3299359381198883,
      "kl": 0.22751319408416748,
      "learning_rate": 2.139271443359768e-06,
      "loss": 0.0091,
      "reward": 2.3759946823120117,
      "reward_std": 1.01981520652771,
      "rewards/reward_function": 2.3759946823120117,
      "step": 1034
    },
    {
      "completion_length": 367.2500305175781,
      "epoch": 0.5921052631578947,
      "grad_norm": 156.0352020263672,
      "kl": 18.745803833007812,
      "learning_rate": 2.1343314212764665e-06,
      "loss": 0.7498,
      "reward": 2.587236166000366,
      "reward_std": 0.6966283321380615,
      "rewards/reward_function": 2.587236166000366,
      "step": 1035
    },
    {
      "completion_length": 381.9285888671875,
      "epoch": 0.5926773455377574,
      "grad_norm": 0.3233250081539154,
      "kl": 0.18117420375347137,
      "learning_rate": 2.129392857771638e-06,
      "loss": 0.0072,
      "reward": 2.365799903869629,
      "reward_std": 0.9813275933265686,
      "rewards/reward_function": 2.365799903869629,
      "step": 1036
    },
    {
      "completion_length": 310.2857360839844,
      "epoch": 0.5932494279176201,
      "grad_norm": 0.2902023494243622,
      "kl": 0.24722783267498016,
      "learning_rate": 2.124455772544219e-06,
      "loss": 0.0099,
      "reward": 2.0497727394104004,
      "reward_std": 1.0336813926696777,
      "rewards/reward_function": 2.0497727394104004,
      "step": 1037
    },
    {
      "completion_length": 308.5357360839844,
      "epoch": 0.5938215102974829,
      "grad_norm": 0.28752535581588745,
      "kl": 0.19398429989814758,
      "learning_rate": 2.1195201852872522e-06,
      "loss": 0.0078,
      "reward": 1.9709858894348145,
      "reward_std": 0.9330247640609741,
      "rewards/reward_function": 1.9709858894348145,
      "step": 1038
    },
    {
      "completion_length": 372.8571472167969,
      "epoch": 0.5943935926773455,
      "grad_norm": 1.5704669952392578,
      "kl": 0.5319435596466064,
      "learning_rate": 2.1145861156878007e-06,
      "loss": 0.0213,
      "reward": 2.579878568649292,
      "reward_std": 0.8719170689582825,
      "rewards/reward_function": 2.579878568649292,
      "step": 1039
    },
    {
      "completion_length": 340.5714416503906,
      "epoch": 0.5949656750572082,
      "grad_norm": 1.0397648811340332,
      "kl": 0.2583724558353424,
      "learning_rate": 2.109653583426879e-06,
      "loss": 0.0103,
      "reward": 2.2401771545410156,
      "reward_std": 0.8393760919570923,
      "rewards/reward_function": 2.2401771545410156,
      "step": 1040
    },
    {
      "completion_length": 341.7857360839844,
      "epoch": 0.5955377574370709,
      "grad_norm": 0.48193132877349854,
      "kl": 0.22696200013160706,
      "learning_rate": 2.104722608179365e-06,
      "loss": 0.0091,
      "reward": 2.091731071472168,
      "reward_std": 1.1664211750030518,
      "rewards/reward_function": 2.091731071472168,
      "step": 1041
    },
    {
      "completion_length": 321.1785888671875,
      "epoch": 0.5961098398169337,
      "grad_norm": 0.17843230068683624,
      "kl": 0.13616584241390228,
      "learning_rate": 2.09979320961393e-06,
      "loss": 0.0054,
      "reward": 2.0772738456726074,
      "reward_std": 0.9431192278862,
      "rewards/reward_function": 2.0772738456726074,
      "step": 1042
    },
    {
      "completion_length": 301.3214416503906,
      "epoch": 0.5966819221967964,
      "grad_norm": 0.18397055566310883,
      "kl": 0.14306551218032837,
      "learning_rate": 2.094865407392952e-06,
      "loss": 0.0057,
      "reward": 2.61319899559021,
      "reward_std": 0.9963331818580627,
      "rewards/reward_function": 2.61319899559021,
      "step": 1043
    },
    {
      "completion_length": 323.4285888671875,
      "epoch": 0.597254004576659,
      "grad_norm": 4.604989528656006,
      "kl": 0.4957977831363678,
      "learning_rate": 2.089939221172446e-06,
      "loss": 0.0198,
      "reward": 2.515796661376953,
      "reward_std": 0.6994448304176331,
      "rewards/reward_function": 2.515796661376953,
      "step": 1044
    },
    {
      "completion_length": 332.5,
      "epoch": 0.5978260869565217,
      "grad_norm": 0.35101446509361267,
      "kl": 0.24889902770519257,
      "learning_rate": 2.0850146706019766e-06,
      "loss": 0.01,
      "reward": 2.2037084102630615,
      "reward_std": 1.012711763381958,
      "rewards/reward_function": 2.2037084102630615,
      "step": 1045
    },
    {
      "completion_length": 281.8214416503906,
      "epoch": 0.5983981693363845,
      "grad_norm": 0.1902020126581192,
      "kl": 0.15346656739711761,
      "learning_rate": 2.080091775324588e-06,
      "loss": 0.0061,
      "reward": 2.0153746604919434,
      "reward_std": 0.8767405152320862,
      "rewards/reward_function": 2.0153746604919434,
      "step": 1046
    },
    {
      "completion_length": 327.5714416503906,
      "epoch": 0.5989702517162472,
      "grad_norm": 0.6578503251075745,
      "kl": 0.15589940547943115,
      "learning_rate": 2.0751705549767202e-06,
      "loss": 0.0062,
      "reward": 1.7749543190002441,
      "reward_std": 1.0500338077545166,
      "rewards/reward_function": 1.7749543190002441,
      "step": 1047
    },
    {
      "completion_length": 330.5357360839844,
      "epoch": 0.5995423340961098,
      "grad_norm": 0.3004372715950012,
      "kl": 0.16544431447982788,
      "learning_rate": 2.0702510291881307e-06,
      "loss": 0.0066,
      "reward": 1.8107150793075562,
      "reward_std": 1.12462317943573,
      "rewards/reward_function": 1.8107150793075562,
      "step": 1048
    },
    {
      "completion_length": 278.9285888671875,
      "epoch": 0.6001144164759725,
      "grad_norm": 110.1727294921875,
      "kl": 3.388174295425415,
      "learning_rate": 2.0653332175818206e-06,
      "loss": 0.1355,
      "reward": 1.8664542436599731,
      "reward_std": 1.0273792743682861,
      "rewards/reward_function": 1.8664542436599731,
      "step": 1049
    },
    {
      "completion_length": 322.46429443359375,
      "epoch": 0.6006864988558352,
      "grad_norm": 0.28148153424263,
      "kl": 0.1901276856660843,
      "learning_rate": 2.0604171397739508e-06,
      "loss": 0.0076,
      "reward": 1.8828197717666626,
      "reward_std": 0.6973152756690979,
      "rewards/reward_function": 1.8828197717666626,
      "step": 1050
    },
    {
      "completion_length": 320.25,
      "epoch": 0.601258581235698,
      "grad_norm": 0.23639242351055145,
      "kl": 0.17261405289173126,
      "learning_rate": 2.055502815373769e-06,
      "loss": 0.0069,
      "reward": 2.2763888835906982,
      "reward_std": 1.0949151515960693,
      "rewards/reward_function": 2.2763888835906982,
      "step": 1051
    },
    {
      "completion_length": 281.8214416503906,
      "epoch": 0.6018306636155606,
      "grad_norm": 0.2511940002441406,
      "kl": 0.15335600078105927,
      "learning_rate": 2.0505902639835266e-06,
      "loss": 0.0061,
      "reward": 1.7311341762542725,
      "reward_std": 0.7294654250144958,
      "rewards/reward_function": 1.7311341762542725,
      "step": 1052
    },
    {
      "completion_length": 381.89288330078125,
      "epoch": 0.6024027459954233,
      "grad_norm": 0.23145802319049835,
      "kl": 0.1905820518732071,
      "learning_rate": 2.045679505198404e-06,
      "loss": 0.0076,
      "reward": 2.161991596221924,
      "reward_std": 1.0559331178665161,
      "rewards/reward_function": 2.161991596221924,
      "step": 1053
    },
    {
      "completion_length": 339.46429443359375,
      "epoch": 0.602974828375286,
      "grad_norm": 0.7313336730003357,
      "kl": 0.2764993906021118,
      "learning_rate": 2.0407705586064325e-06,
      "loss": 0.0111,
      "reward": 1.8859713077545166,
      "reward_std": 1.2317856550216675,
      "rewards/reward_function": 1.8859713077545166,
      "step": 1054
    },
    {
      "completion_length": 371.1071472167969,
      "epoch": 0.6035469107551488,
      "grad_norm": 0.18876773118972778,
      "kl": 0.15353438258171082,
      "learning_rate": 2.0358634437884114e-06,
      "loss": 0.0061,
      "reward": 2.0156502723693848,
      "reward_std": 0.9025229215621948,
      "rewards/reward_function": 2.0156502723693848,
      "step": 1055
    },
    {
      "completion_length": 350.64288330078125,
      "epoch": 0.6041189931350115,
      "grad_norm": 0.20318087935447693,
      "kl": 0.16329126060009003,
      "learning_rate": 2.0309581803178373e-06,
      "loss": 0.0065,
      "reward": 2.045612335205078,
      "reward_std": 1.2901644706726074,
      "rewards/reward_function": 2.045612335205078,
      "step": 1056
    },
    {
      "completion_length": 349.5714416503906,
      "epoch": 0.6046910755148741,
      "grad_norm": 87.25923156738281,
      "kl": 1.7713508605957031,
      "learning_rate": 2.0260547877608194e-06,
      "loss": 0.0709,
      "reward": 1.7902896404266357,
      "reward_std": 1.1219569444656372,
      "rewards/reward_function": 1.7902896404266357,
      "step": 1057
    },
    {
      "completion_length": 322.3571472167969,
      "epoch": 0.6052631578947368,
      "grad_norm": 126.80311584472656,
      "kl": 7.726560115814209,
      "learning_rate": 2.021153285676005e-06,
      "loss": 0.3091,
      "reward": 2.883016347885132,
      "reward_std": 0.6147007942199707,
      "rewards/reward_function": 2.883016347885132,
      "step": 1058
    },
    {
      "completion_length": 370.39288330078125,
      "epoch": 0.6058352402745996,
      "grad_norm": 0.24242612719535828,
      "kl": 0.1744418442249298,
      "learning_rate": 2.016253693614501e-06,
      "loss": 0.007,
      "reward": 2.784837007522583,
      "reward_std": 0.8289033770561218,
      "rewards/reward_function": 2.784837007522583,
      "step": 1059
    },
    {
      "completion_length": 357.64288330078125,
      "epoch": 0.6064073226544623,
      "grad_norm": 0.3324011564254761,
      "kl": 0.15743029117584229,
      "learning_rate": 2.0113560311197965e-06,
      "loss": 0.0063,
      "reward": 1.8418327569961548,
      "reward_std": 1.0359687805175781,
      "rewards/reward_function": 1.8418327569961548,
      "step": 1060
    },
    {
      "completion_length": 284.9285888671875,
      "epoch": 0.6069794050343249,
      "grad_norm": 2.2128167152404785,
      "kl": 0.15750321745872498,
      "learning_rate": 2.0064603177276815e-06,
      "loss": 0.0063,
      "reward": 1.507265329360962,
      "reward_std": 0.7412401437759399,
      "rewards/reward_function": 1.507265329360962,
      "step": 1061
    },
    {
      "completion_length": 341.14288330078125,
      "epoch": 0.6075514874141876,
      "grad_norm": 5.057772636413574,
      "kl": 0.3906545341014862,
      "learning_rate": 2.001566572966173e-06,
      "loss": 0.0156,
      "reward": 1.7942674160003662,
      "reward_std": 0.8478716611862183,
      "rewards/reward_function": 1.7942674160003662,
      "step": 1062
    },
    {
      "completion_length": 306.7857360839844,
      "epoch": 0.6081235697940504,
      "grad_norm": 0.8931918144226074,
      "kl": 0.18458357453346252,
      "learning_rate": 1.996674816355437e-06,
      "loss": 0.0074,
      "reward": 2.0666747093200684,
      "reward_std": 1.0747385025024414,
      "rewards/reward_function": 2.0666747093200684,
      "step": 1063
    },
    {
      "completion_length": 318.8214416503906,
      "epoch": 0.6086956521739131,
      "grad_norm": 1.2413456439971924,
      "kl": 0.2312837541103363,
      "learning_rate": 1.9917850674077056e-06,
      "loss": 0.0093,
      "reward": 2.240534782409668,
      "reward_std": 0.6689039468765259,
      "rewards/reward_function": 2.240534782409668,
      "step": 1064
    },
    {
      "completion_length": 294.8571472167969,
      "epoch": 0.6092677345537757,
      "grad_norm": 1.6724172830581665,
      "kl": 0.3789401948451996,
      "learning_rate": 1.9868973456272067e-06,
      "loss": 0.0152,
      "reward": 2.4195752143859863,
      "reward_std": 1.003045916557312,
      "rewards/reward_function": 2.4195752143859863,
      "step": 1065
    },
    {
      "completion_length": 270.7857360839844,
      "epoch": 0.6098398169336384,
      "grad_norm": 71.75444030761719,
      "kl": 4.5926666259765625,
      "learning_rate": 1.9820116705100778e-06,
      "loss": 0.1837,
      "reward": 1.9455702304840088,
      "reward_std": 0.8621322512626648,
      "rewards/reward_function": 1.9455702304840088,
      "step": 1066
    },
    {
      "completion_length": 343.14288330078125,
      "epoch": 0.6104118993135011,
      "grad_norm": 1.73850679397583,
      "kl": 0.2400248646736145,
      "learning_rate": 1.9771280615442967e-06,
      "loss": 0.0096,
      "reward": 2.467172384262085,
      "reward_std": 0.7328288555145264,
      "rewards/reward_function": 2.467172384262085,
      "step": 1067
    },
    {
      "completion_length": 364.89288330078125,
      "epoch": 0.6109839816933639,
      "grad_norm": 291657.9375,
      "kl": 1767.41015625,
      "learning_rate": 1.972246538209597e-06,
      "loss": 70.6964,
      "reward": 1.7902538776397705,
      "reward_std": 0.8036648035049438,
      "rewards/reward_function": 1.7902538776397705,
      "step": 1068
    },
    {
      "completion_length": 310.2857360839844,
      "epoch": 0.6115560640732265,
      "grad_norm": 0.23382511734962463,
      "kl": 0.19792047142982483,
      "learning_rate": 1.967367119977396e-06,
      "loss": 0.0079,
      "reward": 1.9264001846313477,
      "reward_std": 0.7839677929878235,
      "rewards/reward_function": 1.9264001846313477,
      "step": 1069
    },
    {
      "completion_length": 320.71429443359375,
      "epoch": 0.6121281464530892,
      "grad_norm": 5.713137626647949,
      "kl": 1.981060266494751,
      "learning_rate": 1.96248982631071e-06,
      "loss": 0.0792,
      "reward": 1.9035029411315918,
      "reward_std": 1.0037848949432373,
      "rewards/reward_function": 1.9035029411315918,
      "step": 1070
    },
    {
      "completion_length": 337.1785888671875,
      "epoch": 0.6127002288329519,
      "grad_norm": 0.7873445749282837,
      "kl": 0.18097512423992157,
      "learning_rate": 1.957614676664085e-06,
      "loss": 0.0072,
      "reward": 1.9369635581970215,
      "reward_std": 1.0108269453048706,
      "rewards/reward_function": 1.9369635581970215,
      "step": 1071
    },
    {
      "completion_length": 338.0714416503906,
      "epoch": 0.6132723112128147,
      "grad_norm": 3.738955020904541,
      "kl": 0.5565629601478577,
      "learning_rate": 1.9527416904835133e-06,
      "loss": 0.0223,
      "reward": 2.0228333473205566,
      "reward_std": 0.5884087681770325,
      "rewards/reward_function": 2.0228333473205566,
      "step": 1072
    },
    {
      "completion_length": 303.89288330078125,
      "epoch": 0.6138443935926774,
      "grad_norm": 0.23005208373069763,
      "kl": 0.1752651184797287,
      "learning_rate": 1.9478708872063563e-06,
      "loss": 0.007,
      "reward": 1.7210108041763306,
      "reward_std": 0.5258492231369019,
      "rewards/reward_function": 1.7210108041763306,
      "step": 1073
    },
    {
      "completion_length": 277.2857360839844,
      "epoch": 0.61441647597254,
      "grad_norm": 0.2490849643945694,
      "kl": 0.1736275553703308,
      "learning_rate": 1.9430022862612714e-06,
      "loss": 0.0069,
      "reward": 2.0293328762054443,
      "reward_std": 1.2626196146011353,
      "rewards/reward_function": 2.0293328762054443,
      "step": 1074
    },
    {
      "completion_length": 337.46429443359375,
      "epoch": 0.6149885583524027,
      "grad_norm": 1.0400874614715576,
      "kl": 0.2182181030511856,
      "learning_rate": 1.9381359070681265e-06,
      "loss": 0.0087,
      "reward": 1.9752211570739746,
      "reward_std": 0.8590238094329834,
      "rewards/reward_function": 1.9752211570739746,
      "step": 1075
    },
    {
      "completion_length": 297.1785888671875,
      "epoch": 0.6155606407322655,
      "grad_norm": 0.16826029121875763,
      "kl": 0.14013658463954926,
      "learning_rate": 1.9332717690379324e-06,
      "loss": 0.0056,
      "reward": 2.719834089279175,
      "reward_std": 1.0073574781417847,
      "rewards/reward_function": 2.719834089279175,
      "step": 1076
    },
    {
      "completion_length": 367.64288330078125,
      "epoch": 0.6161327231121282,
      "grad_norm": 0.17574772238731384,
      "kl": 0.14610809087753296,
      "learning_rate": 1.928409891572757e-06,
      "loss": 0.0058,
      "reward": 2.2135562896728516,
      "reward_std": 1.0727205276489258,
      "rewards/reward_function": 2.2135562896728516,
      "step": 1077
    },
    {
      "completion_length": 350.14288330078125,
      "epoch": 0.6167048054919908,
      "grad_norm": 0.17408595979213715,
      "kl": 0.14534339308738708,
      "learning_rate": 1.923550294065653e-06,
      "loss": 0.0058,
      "reward": 1.8132442235946655,
      "reward_std": 1.0108531713485718,
      "rewards/reward_function": 1.8132442235946655,
      "step": 1078
    },
    {
      "completion_length": 329.0714416503906,
      "epoch": 0.6172768878718535,
      "grad_norm": 0.19551192224025726,
      "kl": 0.1304793357849121,
      "learning_rate": 1.918692995900578e-06,
      "loss": 0.0052,
      "reward": 1.361092209815979,
      "reward_std": 0.4576855003833771,
      "rewards/reward_function": 1.361092209815979,
      "step": 1079
    },
    {
      "completion_length": 300.0357360839844,
      "epoch": 0.6178489702517163,
      "grad_norm": 1.2328474521636963,
      "kl": 0.23558440804481506,
      "learning_rate": 1.9138380164523186e-06,
      "loss": 0.0094,
      "reward": 1.7950615882873535,
      "reward_std": 0.6225394010543823,
      "rewards/reward_function": 1.7950615882873535,
      "step": 1080
    },
    {
      "completion_length": 322.75,
      "epoch": 0.618421052631579,
      "grad_norm": 1.773610234260559,
      "kl": 0.25136232376098633,
      "learning_rate": 1.9089853750864116e-06,
      "loss": 0.0101,
      "reward": 2.092061758041382,
      "reward_std": 1.2878423929214478,
      "rewards/reward_function": 2.092061758041382,
      "step": 1081
    },
    {
      "completion_length": 333.6785888671875,
      "epoch": 0.6189931350114416,
      "grad_norm": 0.16315858066082,
      "kl": 0.11269695311784744,
      "learning_rate": 1.9041350911590686e-06,
      "loss": 0.0045,
      "reward": 2.1760854721069336,
      "reward_std": 0.9166327118873596,
      "rewards/reward_function": 2.1760854721069336,
      "step": 1082
    },
    {
      "completion_length": 306.4285888671875,
      "epoch": 0.6195652173913043,
      "grad_norm": 0.1929519921541214,
      "kl": 0.15988807380199432,
      "learning_rate": 1.899287184017098e-06,
      "loss": 0.0064,
      "reward": 2.066632032394409,
      "reward_std": 1.0643939971923828,
      "rewards/reward_function": 2.066632032394409,
      "step": 1083
    },
    {
      "completion_length": 337.1785888671875,
      "epoch": 0.620137299771167,
      "grad_norm": 0.18027924001216888,
      "kl": 0.16182442009449005,
      "learning_rate": 1.8944416729978253e-06,
      "loss": 0.0065,
      "reward": 1.9852193593978882,
      "reward_std": 1.1447467803955078,
      "rewards/reward_function": 1.9852193593978882,
      "step": 1084
    },
    {
      "completion_length": 322.25,
      "epoch": 0.6207093821510298,
      "grad_norm": 22.314428329467773,
      "kl": 0.8049201369285583,
      "learning_rate": 1.889598577429022e-06,
      "loss": 0.0322,
      "reward": 2.158811330795288,
      "reward_std": 1.0466272830963135,
      "rewards/reward_function": 2.158811330795288,
      "step": 1085
    },
    {
      "completion_length": 338.46429443359375,
      "epoch": 0.6212814645308925,
      "grad_norm": 0.2899605929851532,
      "kl": 0.13485506176948547,
      "learning_rate": 1.8847579166288218e-06,
      "loss": 0.0054,
      "reward": 2.465383529663086,
      "reward_std": 0.6876826882362366,
      "rewards/reward_function": 2.465383529663086,
      "step": 1086
    },
    {
      "completion_length": 316.2857360839844,
      "epoch": 0.6218535469107551,
      "grad_norm": 0.2220635712146759,
      "kl": 0.1452186554670334,
      "learning_rate": 1.8799197099056487e-06,
      "loss": 0.0058,
      "reward": 2.31534743309021,
      "reward_std": 0.777035653591156,
      "rewards/reward_function": 2.31534743309021,
      "step": 1087
    },
    {
      "completion_length": 307.1785888671875,
      "epoch": 0.6224256292906178,
      "grad_norm": 1.1704092025756836,
      "kl": 0.31595051288604736,
      "learning_rate": 1.875083976558136e-06,
      "loss": 0.0126,
      "reward": 2.28351092338562,
      "reward_std": 1.0009040832519531,
      "rewards/reward_function": 2.28351092338562,
      "step": 1088
    },
    {
      "completion_length": 373.14288330078125,
      "epoch": 0.6229977116704806,
      "grad_norm": 1.1939876079559326,
      "kl": 0.2195790559053421,
      "learning_rate": 1.870250735875053e-06,
      "loss": 0.0088,
      "reward": 2.064650058746338,
      "reward_std": 1.1086556911468506,
      "rewards/reward_function": 2.064650058746338,
      "step": 1089
    },
    {
      "completion_length": 373.0000305175781,
      "epoch": 0.6235697940503433,
      "grad_norm": 0.7687636613845825,
      "kl": 0.19946907460689545,
      "learning_rate": 1.8654200071352259e-06,
      "loss": 0.008,
      "reward": 2.3886184692382812,
      "reward_std": 0.9597442150115967,
      "rewards/reward_function": 2.3886184692382812,
      "step": 1090
    },
    {
      "completion_length": 325.39288330078125,
      "epoch": 0.6241418764302059,
      "grad_norm": 1.649415135383606,
      "kl": 0.17828021943569183,
      "learning_rate": 1.860591809607459e-06,
      "loss": 0.0071,
      "reward": 1.7362531423568726,
      "reward_std": 0.738692045211792,
      "rewards/reward_function": 1.7362531423568726,
      "step": 1091
    },
    {
      "completion_length": 345.64288330078125,
      "epoch": 0.6247139588100686,
      "grad_norm": 0.18186812102794647,
      "kl": 0.12185943126678467,
      "learning_rate": 1.8557661625504646e-06,
      "loss": 0.0049,
      "reward": 2.58095121383667,
      "reward_std": 1.0907772779464722,
      "rewards/reward_function": 2.58095121383667,
      "step": 1092
    },
    {
      "completion_length": 332.6785888671875,
      "epoch": 0.6252860411899314,
      "grad_norm": 0.7163596153259277,
      "kl": 0.13435952365398407,
      "learning_rate": 1.8509430852127764e-06,
      "loss": 0.0054,
      "reward": 1.8739714622497559,
      "reward_std": 0.7739185690879822,
      "rewards/reward_function": 1.8739714622497559,
      "step": 1093
    },
    {
      "completion_length": 318.5,
      "epoch": 0.6258581235697941,
      "grad_norm": 0.19217640161514282,
      "kl": 0.14351066946983337,
      "learning_rate": 1.846122596832682e-06,
      "loss": 0.0057,
      "reward": 2.0486817359924316,
      "reward_std": 0.9617774486541748,
      "rewards/reward_function": 2.0486817359924316,
      "step": 1094
    },
    {
      "completion_length": 287.5714416503906,
      "epoch": 0.6264302059496567,
      "grad_norm": 1.8011494874954224,
      "kl": 0.255168616771698,
      "learning_rate": 1.8413047166381404e-06,
      "loss": 0.0102,
      "reward": 1.6204750537872314,
      "reward_std": 1.1672121286392212,
      "rewards/reward_function": 1.6204750537872314,
      "step": 1095
    },
    {
      "completion_length": 364.3214416503906,
      "epoch": 0.6270022883295194,
      "grad_norm": 0.18060146272182465,
      "kl": 0.15084406733512878,
      "learning_rate": 1.8364894638467078e-06,
      "loss": 0.006,
      "reward": 2.3330509662628174,
      "reward_std": 0.7574965953826904,
      "rewards/reward_function": 2.3330509662628174,
      "step": 1096
    },
    {
      "completion_length": 327.14288330078125,
      "epoch": 0.6275743707093822,
      "grad_norm": 0.6838092803955078,
      "kl": 0.2641604542732239,
      "learning_rate": 1.8316768576654591e-06,
      "loss": 0.0106,
      "reward": 2.0832619667053223,
      "reward_std": 0.8120837807655334,
      "rewards/reward_function": 2.0832619667053223,
      "step": 1097
    },
    {
      "completion_length": 349.8214416503906,
      "epoch": 0.6281464530892449,
      "grad_norm": 3.6582086086273193,
      "kl": 0.22082620859146118,
      "learning_rate": 1.8268669172909137e-06,
      "loss": 0.0088,
      "reward": 1.488646149635315,
      "reward_std": 0.8730930089950562,
      "rewards/reward_function": 1.488646149635315,
      "step": 1098
    },
    {
      "completion_length": 323.0,
      "epoch": 0.6287185354691075,
      "grad_norm": 0.21478597819805145,
      "kl": 0.14385077357292175,
      "learning_rate": 1.8220596619089576e-06,
      "loss": 0.0058,
      "reward": 1.9934325218200684,
      "reward_std": 1.117966651916504,
      "rewards/reward_function": 1.9934325218200684,
      "step": 1099
    },
    {
      "completion_length": 369.5000305175781,
      "epoch": 0.6292906178489702,
      "grad_norm": 0.19913142919540405,
      "kl": 0.12517811357975006,
      "learning_rate": 1.8172551106947656e-06,
      "loss": 0.005,
      "reward": 2.3623266220092773,
      "reward_std": 0.9545746445655823,
      "rewards/reward_function": 2.3623266220092773,
      "step": 1100
    },
    {
      "completion_length": 340.1785888671875,
      "epoch": 0.629862700228833,
      "grad_norm": 0.5911720395088196,
      "kl": 0.16484320163726807,
      "learning_rate": 1.8124532828127283e-06,
      "loss": 0.0066,
      "reward": 2.002819061279297,
      "reward_std": 1.2344284057617188,
      "rewards/reward_function": 2.002819061279297,
      "step": 1101
    },
    {
      "completion_length": 366.6785888671875,
      "epoch": 0.6304347826086957,
      "grad_norm": 0.20001782476902008,
      "kl": 0.13328823447227478,
      "learning_rate": 1.8076541974163699e-06,
      "loss": 0.0053,
      "reward": 1.862307071685791,
      "reward_std": 0.9026513695716858,
      "rewards/reward_function": 1.862307071685791,
      "step": 1102
    },
    {
      "completion_length": 334.75,
      "epoch": 0.6310068649885584,
      "grad_norm": 0.27485454082489014,
      "kl": 0.20149727165699005,
      "learning_rate": 1.8028578736482793e-06,
      "loss": 0.0081,
      "reward": 1.8340418338775635,
      "reward_std": 1.0098894834518433,
      "rewards/reward_function": 1.8340418338775635,
      "step": 1103
    },
    {
      "completion_length": 317.8571472167969,
      "epoch": 0.631578947368421,
      "grad_norm": 0.24069935083389282,
      "kl": 0.15606604516506195,
      "learning_rate": 1.7980643306400272e-06,
      "loss": 0.0062,
      "reward": 1.4957683086395264,
      "reward_std": 0.9543678760528564,
      "rewards/reward_function": 1.4957683086395264,
      "step": 1104
    },
    {
      "completion_length": 275.0714416503906,
      "epoch": 0.6321510297482837,
      "grad_norm": 0.20282234251499176,
      "kl": 0.1505909413099289,
      "learning_rate": 1.7932735875120935e-06,
      "loss": 0.006,
      "reward": 2.6728806495666504,
      "reward_std": 0.9261201024055481,
      "rewards/reward_function": 2.6728806495666504,
      "step": 1105
    },
    {
      "completion_length": 337.1071472167969,
      "epoch": 0.6327231121281465,
      "grad_norm": 0.2598014771938324,
      "kl": 0.19530431926250458,
      "learning_rate": 1.7884856633737907e-06,
      "loss": 0.0078,
      "reward": 1.8289942741394043,
      "reward_std": 0.9685468673706055,
      "rewards/reward_function": 1.8289942741394043,
      "step": 1106
    },
    {
      "completion_length": 337.9285888671875,
      "epoch": 0.6332951945080092,
      "grad_norm": 0.18474864959716797,
      "kl": 0.1592569351196289,
      "learning_rate": 1.7837005773231844e-06,
      "loss": 0.0064,
      "reward": 2.502368211746216,
      "reward_std": 0.8152681589126587,
      "rewards/reward_function": 2.502368211746216,
      "step": 1107
    },
    {
      "completion_length": 306.71429443359375,
      "epoch": 0.6338672768878718,
      "grad_norm": 0.4965636730194092,
      "kl": 0.13603857159614563,
      "learning_rate": 1.7789183484470226e-06,
      "loss": 0.0054,
      "reward": 2.4903740882873535,
      "reward_std": 1.0518121719360352,
      "rewards/reward_function": 2.4903740882873535,
      "step": 1108
    },
    {
      "completion_length": 295.7857360839844,
      "epoch": 0.6344393592677345,
      "grad_norm": 0.4897773563861847,
      "kl": 0.20155377686023712,
      "learning_rate": 1.774138995820654e-06,
      "loss": 0.0081,
      "reward": 2.1157960891723633,
      "reward_std": 0.9447885751724243,
      "rewards/reward_function": 2.1157960891723633,
      "step": 1109
    },
    {
      "completion_length": 331.96429443359375,
      "epoch": 0.6350114416475973,
      "grad_norm": 0.20737309753894806,
      "kl": 0.1644778847694397,
      "learning_rate": 1.7693625385079576e-06,
      "loss": 0.0066,
      "reward": 1.9379186630249023,
      "reward_std": 0.5425484776496887,
      "rewards/reward_function": 1.9379186630249023,
      "step": 1110
    },
    {
      "completion_length": 300.5357360839844,
      "epoch": 0.63558352402746,
      "grad_norm": 0.3236565589904785,
      "kl": 0.1442820131778717,
      "learning_rate": 1.7645889955612595e-06,
      "loss": 0.0058,
      "reward": 1.4757505655288696,
      "reward_std": 0.6946955919265747,
      "rewards/reward_function": 1.4757505655288696,
      "step": 1111
    },
    {
      "completion_length": 347.6071472167969,
      "epoch": 0.6361556064073226,
      "grad_norm": 104.62155151367188,
      "kl": 1.6773569583892822,
      "learning_rate": 1.7598183860212644e-06,
      "loss": 0.0671,
      "reward": 2.222381114959717,
      "reward_std": 0.8995649814605713,
      "rewards/reward_function": 2.222381114959717,
      "step": 1112
    },
    {
      "completion_length": 301.71429443359375,
      "epoch": 0.6367276887871853,
      "grad_norm": 13.325573921203613,
      "kl": 1.2199747562408447,
      "learning_rate": 1.7550507289169743e-06,
      "loss": 0.0488,
      "reward": 2.5085954666137695,
      "reward_std": 1.22212815284729,
      "rewards/reward_function": 2.5085954666137695,
      "step": 1113
    },
    {
      "completion_length": 292.8214416503906,
      "epoch": 0.6372997711670481,
      "grad_norm": 11.794377326965332,
      "kl": 0.8908405303955078,
      "learning_rate": 1.7502860432656156e-06,
      "loss": 0.0356,
      "reward": 1.9531002044677734,
      "reward_std": 0.7531242966651917,
      "rewards/reward_function": 1.9531002044677734,
      "step": 1114
    },
    {
      "completion_length": 347.9285888671875,
      "epoch": 0.6378718535469108,
      "grad_norm": 5.362925052642822,
      "kl": 0.24167566001415253,
      "learning_rate": 1.7455243480725628e-06,
      "loss": 0.0097,
      "reward": 2.0226006507873535,
      "reward_std": 1.054322600364685,
      "rewards/reward_function": 2.0226006507873535,
      "step": 1115
    },
    {
      "completion_length": 306.7857360839844,
      "epoch": 0.6384439359267735,
      "grad_norm": 0.21931323409080505,
      "kl": 0.15863186120986938,
      "learning_rate": 1.7407656623312594e-06,
      "loss": 0.0063,
      "reward": 2.2918834686279297,
      "reward_std": 0.8792473077774048,
      "rewards/reward_function": 2.2918834686279297,
      "step": 1116
    },
    {
      "completion_length": 324.0714416503906,
      "epoch": 0.6390160183066361,
      "grad_norm": 7.360823154449463,
      "kl": 0.879558801651001,
      "learning_rate": 1.7360100050231483e-06,
      "loss": 0.0352,
      "reward": 1.695087194442749,
      "reward_std": 0.9789000153541565,
      "rewards/reward_function": 1.695087194442749,
      "step": 1117
    },
    {
      "completion_length": 316.6071472167969,
      "epoch": 0.6395881006864989,
      "grad_norm": 0.27127528190612793,
      "kl": 0.1434551477432251,
      "learning_rate": 1.7312573951175883e-06,
      "loss": 0.0057,
      "reward": 2.4468650817871094,
      "reward_std": 1.1938021183013916,
      "rewards/reward_function": 2.4468650817871094,
      "step": 1118
    },
    {
      "completion_length": 284.4285888671875,
      "epoch": 0.6401601830663616,
      "grad_norm": 0.3496883809566498,
      "kl": 0.14097601175308228,
      "learning_rate": 1.7265078515717882e-06,
      "loss": 0.0056,
      "reward": 1.6669243574142456,
      "reward_std": 1.1488525867462158,
      "rewards/reward_function": 1.6669243574142456,
      "step": 1119
    },
    {
      "completion_length": 334.5357360839844,
      "epoch": 0.6407322654462243,
      "grad_norm": 0.3037044405937195,
      "kl": 0.15846580266952515,
      "learning_rate": 1.7217613933307198e-06,
      "loss": 0.0063,
      "reward": 1.8457533121109009,
      "reward_std": 1.1476702690124512,
      "rewards/reward_function": 1.8457533121109009,
      "step": 1120
    },
    {
      "completion_length": 275.8571472167969,
      "epoch": 0.6413043478260869,
      "grad_norm": 0.19376401603221893,
      "kl": 0.1349031776189804,
      "learning_rate": 1.7170180393270533e-06,
      "loss": 0.0054,
      "reward": 2.197702169418335,
      "reward_std": 0.9091899394989014,
      "rewards/reward_function": 2.197702169418335,
      "step": 1121
    },
    {
      "completion_length": 348.21429443359375,
      "epoch": 0.6418764302059496,
      "grad_norm": 0.3183380663394928,
      "kl": 0.16709266602993011,
      "learning_rate": 1.712277808481073e-06,
      "loss": 0.0067,
      "reward": 2.220245361328125,
      "reward_std": 0.8062126040458679,
      "rewards/reward_function": 2.220245361328125,
      "step": 1122
    },
    {
      "completion_length": 318.89288330078125,
      "epoch": 0.6424485125858124,
      "grad_norm": 0.25102755427360535,
      "kl": 0.18935607373714447,
      "learning_rate": 1.707540719700608e-06,
      "loss": 0.0076,
      "reward": 2.4805550575256348,
      "reward_std": 1.1372554302215576,
      "rewards/reward_function": 2.4805550575256348,
      "step": 1123
    },
    {
      "completion_length": 361.0714416503906,
      "epoch": 0.6430205949656751,
      "grad_norm": 3.0860486030578613,
      "kl": 0.2973494529724121,
      "learning_rate": 1.7028067918809539e-06,
      "loss": 0.0119,
      "reward": 2.3449935913085938,
      "reward_std": 0.9321015477180481,
      "rewards/reward_function": 2.3449935913085938,
      "step": 1124
    },
    {
      "completion_length": 359.7857360839844,
      "epoch": 0.6435926773455377,
      "grad_norm": 0.5820218324661255,
      "kl": 0.16994664072990417,
      "learning_rate": 1.698076043904796e-06,
      "loss": 0.0068,
      "reward": 1.6389366388320923,
      "reward_std": 1.0099661350250244,
      "rewards/reward_function": 1.6389366388320923,
      "step": 1125
    },
    {
      "completion_length": 326.6785888671875,
      "epoch": 0.6441647597254004,
      "grad_norm": 0.4853680729866028,
      "kl": 0.18956102430820465,
      "learning_rate": 1.6933484946421392e-06,
      "loss": 0.0076,
      "reward": 2.198249578475952,
      "reward_std": 0.9716740250587463,
      "rewards/reward_function": 2.198249578475952,
      "step": 1126
    },
    {
      "completion_length": 336.4285888671875,
      "epoch": 0.6447368421052632,
      "grad_norm": 0.17941902577877045,
      "kl": 0.1555303931236267,
      "learning_rate": 1.6886241629502249e-06,
      "loss": 0.0062,
      "reward": 2.019177198410034,
      "reward_std": 0.5352837443351746,
      "rewards/reward_function": 2.019177198410034,
      "step": 1127
    },
    {
      "completion_length": 355.3214416503906,
      "epoch": 0.6453089244851259,
      "grad_norm": 0.3105866014957428,
      "kl": 0.1633284091949463,
      "learning_rate": 1.6839030676734652e-06,
      "loss": 0.0065,
      "reward": 2.1660802364349365,
      "reward_std": 0.9781515002250671,
      "rewards/reward_function": 2.1660802364349365,
      "step": 1128
    },
    {
      "completion_length": 305.64288330078125,
      "epoch": 0.6458810068649885,
      "grad_norm": 0.24898774921894073,
      "kl": 0.13893859088420868,
      "learning_rate": 1.6791852276433588e-06,
      "loss": 0.0056,
      "reward": 2.6175954341888428,
      "reward_std": 1.0555685758590698,
      "rewards/reward_function": 2.6175954341888428,
      "step": 1129
    },
    {
      "completion_length": 346.0000305175781,
      "epoch": 0.6464530892448512,
      "grad_norm": 98.28045654296875,
      "kl": 2.685899257659912,
      "learning_rate": 1.6744706616784224e-06,
      "loss": 0.1074,
      "reward": 2.260098457336426,
      "reward_std": 0.9050644636154175,
      "rewards/reward_function": 2.260098457336426,
      "step": 1130
    },
    {
      "completion_length": 374.3214416503906,
      "epoch": 0.647025171624714,
      "grad_norm": 0.21383044123649597,
      "kl": 0.16655264794826508,
      "learning_rate": 1.6697593885841107e-06,
      "loss": 0.0067,
      "reward": 2.0992302894592285,
      "reward_std": 1.0913515090942383,
      "rewards/reward_function": 2.0992302894592285,
      "step": 1131
    },
    {
      "completion_length": 334.6071472167969,
      "epoch": 0.6475972540045767,
      "grad_norm": 0.270504891872406,
      "kl": 0.13549122214317322,
      "learning_rate": 1.6650514271527468e-06,
      "loss": 0.0054,
      "reward": 1.6098401546478271,
      "reward_std": 0.8649271726608276,
      "rewards/reward_function": 1.6098401546478271,
      "step": 1132
    },
    {
      "completion_length": 321.9285888671875,
      "epoch": 0.6481693363844394,
      "grad_norm": 3.6485140323638916,
      "kl": 0.6959154605865479,
      "learning_rate": 1.6603467961634424e-06,
      "loss": 0.0278,
      "reward": 2.4131577014923096,
      "reward_std": 1.0551866292953491,
      "rewards/reward_function": 2.4131577014923096,
      "step": 1133
    },
    {
      "completion_length": 350.21429443359375,
      "epoch": 0.648741418764302,
      "grad_norm": 67.51324462890625,
      "kl": 3.197885751724243,
      "learning_rate": 1.6556455143820244e-06,
      "loss": 0.1279,
      "reward": 1.9777467250823975,
      "reward_std": 0.8242555260658264,
      "rewards/reward_function": 1.9777467250823975,
      "step": 1134
    },
    {
      "completion_length": 371.64288330078125,
      "epoch": 0.6493135011441648,
      "grad_norm": 0.21284809708595276,
      "kl": 0.15188615024089813,
      "learning_rate": 1.6509476005609624e-06,
      "loss": 0.0061,
      "reward": 2.9424080848693848,
      "reward_std": 0.905396044254303,
      "rewards/reward_function": 2.9424080848693848,
      "step": 1135
    },
    {
      "completion_length": 350.7500305175781,
      "epoch": 0.6498855835240275,
      "grad_norm": 1.7400672435760498,
      "kl": 0.17528396844863892,
      "learning_rate": 1.646253073439288e-06,
      "loss": 0.007,
      "reward": 1.708830714225769,
      "reward_std": 1.1310800313949585,
      "rewards/reward_function": 1.708830714225769,
      "step": 1136
    },
    {
      "completion_length": 308.21429443359375,
      "epoch": 0.6504576659038902,
      "grad_norm": 0.432870477437973,
      "kl": 0.18318085372447968,
      "learning_rate": 1.6415619517425296e-06,
      "loss": 0.0073,
      "reward": 1.749899983406067,
      "reward_std": 0.9216949343681335,
      "rewards/reward_function": 1.749899983406067,
      "step": 1137
    },
    {
      "completion_length": 325.25,
      "epoch": 0.6510297482837528,
      "grad_norm": 7746.0927734375,
      "kl": 179.8416290283203,
      "learning_rate": 1.636874254182626e-06,
      "loss": 7.1937,
      "reward": 2.442908763885498,
      "reward_std": 0.7041344046592712,
      "rewards/reward_function": 2.442908763885498,
      "step": 1138
    },
    {
      "completion_length": 328.2857360839844,
      "epoch": 0.6516018306636155,
      "grad_norm": 2.3617358207702637,
      "kl": 0.2625889182090759,
      "learning_rate": 1.6321899994578627e-06,
      "loss": 0.0105,
      "reward": 2.501728057861328,
      "reward_std": 0.4395984709262848,
      "rewards/reward_function": 2.501728057861328,
      "step": 1139
    },
    {
      "completion_length": 309.89288330078125,
      "epoch": 0.6521739130434783,
      "grad_norm": 0.1765773743391037,
      "kl": 0.1372670978307724,
      "learning_rate": 1.6275092062527892e-06,
      "loss": 0.0055,
      "reward": 2.092390775680542,
      "reward_std": 0.9223259687423706,
      "rewards/reward_function": 2.092390775680542,
      "step": 1140
    },
    {
      "completion_length": 367.0000305175781,
      "epoch": 0.652745995423341,
      "grad_norm": 4.402590274810791,
      "kl": 0.4149474799633026,
      "learning_rate": 1.622831893238149e-06,
      "loss": 0.0166,
      "reward": 2.2136778831481934,
      "reward_std": 1.1656996011734009,
      "rewards/reward_function": 2.2136778831481934,
      "step": 1141
    },
    {
      "completion_length": 319.46429443359375,
      "epoch": 0.6533180778032036,
      "grad_norm": 0.1528187394142151,
      "kl": 0.11745802313089371,
      "learning_rate": 1.6181580790708049e-06,
      "loss": 0.0047,
      "reward": 1.9330933094024658,
      "reward_std": 0.8229712843894958,
      "rewards/reward_function": 1.9330933094024658,
      "step": 1142
    },
    {
      "completion_length": 331.71429443359375,
      "epoch": 0.6538901601830663,
      "grad_norm": 0.7569029331207275,
      "kl": 0.16234025359153748,
      "learning_rate": 1.613487782393661e-06,
      "loss": 0.0065,
      "reward": 2.49646258354187,
      "reward_std": 0.9091151356697083,
      "rewards/reward_function": 2.49646258354187,
      "step": 1143
    },
    {
      "completion_length": 311.39288330078125,
      "epoch": 0.6544622425629291,
      "grad_norm": 0.6095835566520691,
      "kl": 0.20721186697483063,
      "learning_rate": 1.6088210218355938e-06,
      "loss": 0.0083,
      "reward": 1.8585630655288696,
      "reward_std": 1.1102958917617798,
      "rewards/reward_function": 1.8585630655288696,
      "step": 1144
    },
    {
      "completion_length": 310.64288330078125,
      "epoch": 0.6550343249427918,
      "grad_norm": 0.206709623336792,
      "kl": 0.1466379463672638,
      "learning_rate": 1.6041578160113703e-06,
      "loss": 0.0059,
      "reward": 2.5480592250823975,
      "reward_std": 0.3077634572982788,
      "rewards/reward_function": 2.5480592250823975,
      "step": 1145
    },
    {
      "completion_length": 343.8571472167969,
      "epoch": 0.6556064073226545,
      "grad_norm": 4.242599010467529,
      "kl": 0.30751320719718933,
      "learning_rate": 1.599498183521585e-06,
      "loss": 0.0123,
      "reward": 1.903739094734192,
      "reward_std": 0.9986927509307861,
      "rewards/reward_function": 1.903739094734192,
      "step": 1146
    },
    {
      "completion_length": 300.21429443359375,
      "epoch": 0.6561784897025171,
      "grad_norm": 0.2447453886270523,
      "kl": 0.18119841814041138,
      "learning_rate": 1.5948421429525726e-06,
      "loss": 0.0072,
      "reward": 2.56825590133667,
      "reward_std": 0.6456741690635681,
      "rewards/reward_function": 2.56825590133667,
      "step": 1147
    },
    {
      "completion_length": 328.4285888671875,
      "epoch": 0.6567505720823799,
      "grad_norm": 1.7115975618362427,
      "kl": 0.18196648359298706,
      "learning_rate": 1.590189712876345e-06,
      "loss": 0.0073,
      "reward": 2.1034157276153564,
      "reward_std": 1.0083723068237305,
      "rewards/reward_function": 2.1034157276153564,
      "step": 1148
    },
    {
      "completion_length": 355.5714416503906,
      "epoch": 0.6573226544622426,
      "grad_norm": 4.845210075378418,
      "kl": 0.3310038447380066,
      "learning_rate": 1.5855409118505093e-06,
      "loss": 0.0132,
      "reward": 2.398730993270874,
      "reward_std": 1.2659754753112793,
      "rewards/reward_function": 2.398730993270874,
      "step": 1149
    },
    {
      "completion_length": 344.64288330078125,
      "epoch": 0.6578947368421053,
      "grad_norm": 0.6990411877632141,
      "kl": 0.31144627928733826,
      "learning_rate": 1.5808957584181997e-06,
      "loss": 0.0125,
      "reward": 1.8206095695495605,
      "reward_std": 1.0002244710922241,
      "rewards/reward_function": 1.8206095695495605,
      "step": 1150
    },
    {
      "completion_length": 364.8214416503906,
      "epoch": 0.6584668192219679,
      "grad_norm": 873.46484375,
      "kl": 12.407258033752441,
      "learning_rate": 1.5762542711079997e-06,
      "loss": 0.4963,
      "reward": 1.892946720123291,
      "reward_std": 1.134818196296692,
      "rewards/reward_function": 1.892946720123291,
      "step": 1151
    },
    {
      "completion_length": 311.1071472167969,
      "epoch": 0.6590389016018307,
      "grad_norm": 15.856904983520508,
      "kl": 1.4455586671829224,
      "learning_rate": 1.5716164684338686e-06,
      "loss": 0.0578,
      "reward": 1.8586238622665405,
      "reward_std": 0.6922212243080139,
      "rewards/reward_function": 1.8586238622665405,
      "step": 1152
    },
    {
      "completion_length": 312.96429443359375,
      "epoch": 0.6596109839816934,
      "grad_norm": 0.2348310500383377,
      "kl": 0.1604197770357132,
      "learning_rate": 1.566982368895071e-06,
      "loss": 0.0064,
      "reward": 1.996594786643982,
      "reward_std": 1.1803417205810547,
      "rewards/reward_function": 1.996594786643982,
      "step": 1153
    },
    {
      "completion_length": 316.71429443359375,
      "epoch": 0.6601830663615561,
      "grad_norm": 2.6097280979156494,
      "kl": 0.23769792914390564,
      "learning_rate": 1.5623519909760953e-06,
      "loss": 0.0095,
      "reward": 1.9352680444717407,
      "reward_std": 1.214856743812561,
      "rewards/reward_function": 1.9352680444717407,
      "step": 1154
    },
    {
      "completion_length": 308.21429443359375,
      "epoch": 0.6607551487414187,
      "grad_norm": 0.17026136815547943,
      "kl": 0.15249748528003693,
      "learning_rate": 1.5577253531465925e-06,
      "loss": 0.0061,
      "reward": 2.0102128982543945,
      "reward_std": 0.9555670619010925,
      "rewards/reward_function": 2.0102128982543945,
      "step": 1155
    },
    {
      "completion_length": 337.5714416503906,
      "epoch": 0.6613272311212814,
      "grad_norm": 0.20267190039157867,
      "kl": 0.15204213559627533,
      "learning_rate": 1.5531024738612876e-06,
      "loss": 0.0061,
      "reward": 1.8538697957992554,
      "reward_std": 0.9490690231323242,
      "rewards/reward_function": 1.8538697957992554,
      "step": 1156
    },
    {
      "completion_length": 332.6071472167969,
      "epoch": 0.6618993135011442,
      "grad_norm": 0.25553202629089355,
      "kl": 0.18844395875930786,
      "learning_rate": 1.5484833715599193e-06,
      "loss": 0.0075,
      "reward": 2.2034828662872314,
      "reward_std": 0.9595307111740112,
      "rewards/reward_function": 2.2034828662872314,
      "step": 1157
    },
    {
      "completion_length": 339.64288330078125,
      "epoch": 0.6624713958810069,
      "grad_norm": 0.23491047322750092,
      "kl": 0.13118883967399597,
      "learning_rate": 1.5438680646671572e-06,
      "loss": 0.0052,
      "reward": 1.8280786275863647,
      "reward_std": 1.114801287651062,
      "rewards/reward_function": 1.8280786275863647,
      "step": 1158
    },
    {
      "completion_length": 280.71429443359375,
      "epoch": 0.6630434782608695,
      "grad_norm": 0.5083250999450684,
      "kl": 0.30815720558166504,
      "learning_rate": 1.5392565715925331e-06,
      "loss": 0.0123,
      "reward": 1.8581159114837646,
      "reward_std": 0.7138462662696838,
      "rewards/reward_function": 1.8581159114837646,
      "step": 1159
    },
    {
      "completion_length": 301.8214416503906,
      "epoch": 0.6636155606407322,
      "grad_norm": 4.5067458152771,
      "kl": 0.9550285935401917,
      "learning_rate": 1.534648910730367e-06,
      "loss": 0.0382,
      "reward": 2.0814483165740967,
      "reward_std": 0.8202261328697205,
      "rewards/reward_function": 2.0814483165740967,
      "step": 1160
    },
    {
      "completion_length": 368.1071472167969,
      "epoch": 0.664187643020595,
      "grad_norm": 0.21369092166423798,
      "kl": 0.1644049882888794,
      "learning_rate": 1.5300451004596912e-06,
      "loss": 0.0066,
      "reward": 1.3086189031600952,
      "reward_std": 0.5508645176887512,
      "rewards/reward_function": 1.3086189031600952,
      "step": 1161
    },
    {
      "completion_length": 343.3214416503906,
      "epoch": 0.6647597254004577,
      "grad_norm": 0.6883459091186523,
      "kl": 0.25539863109588623,
      "learning_rate": 1.5254451591441805e-06,
      "loss": 0.0102,
      "reward": 2.041287660598755,
      "reward_std": 1.0819778442382812,
      "rewards/reward_function": 2.041287660598755,
      "step": 1162
    },
    {
      "completion_length": 348.3214416503906,
      "epoch": 0.6653318077803204,
      "grad_norm": 0.17704984545707703,
      "kl": 0.13924573361873627,
      "learning_rate": 1.5208491051320745e-06,
      "loss": 0.0056,
      "reward": 2.2404420375823975,
      "reward_std": 0.7854861617088318,
      "rewards/reward_function": 2.2404420375823975,
      "step": 1163
    },
    {
      "completion_length": 318.46429443359375,
      "epoch": 0.665903890160183,
      "grad_norm": 1.8582780361175537,
      "kl": 0.6610764265060425,
      "learning_rate": 1.5162569567561117e-06,
      "loss": 0.0264,
      "reward": 2.3993606567382812,
      "reward_std": 1.2312391996383667,
      "rewards/reward_function": 2.3993606567382812,
      "step": 1164
    },
    {
      "completion_length": 344.21429443359375,
      "epoch": 0.6664759725400458,
      "grad_norm": 0.16266076266765594,
      "kl": 0.1306222677230835,
      "learning_rate": 1.5116687323334467e-06,
      "loss": 0.0052,
      "reward": 2.464186191558838,
      "reward_std": 0.8418284058570862,
      "rewards/reward_function": 2.464186191558838,
      "step": 1165
    },
    {
      "completion_length": 407.71429443359375,
      "epoch": 0.6670480549199085,
      "grad_norm": 0.8939342498779297,
      "kl": 0.21182234585285187,
      "learning_rate": 1.5070844501655857e-06,
      "loss": 0.0085,
      "reward": 2.0563580989837646,
      "reward_std": 1.3669164180755615,
      "rewards/reward_function": 2.0563580989837646,
      "step": 1166
    },
    {
      "completion_length": 339.1785888671875,
      "epoch": 0.6676201372997712,
      "grad_norm": 0.6310370564460754,
      "kl": 0.17577148973941803,
      "learning_rate": 1.5025041285383093e-06,
      "loss": 0.007,
      "reward": 1.8592320680618286,
      "reward_std": 0.9380041360855103,
      "rewards/reward_function": 1.8592320680618286,
      "step": 1167
    },
    {
      "completion_length": 341.0,
      "epoch": 0.6681922196796338,
      "grad_norm": 0.2543943524360657,
      "kl": 0.16902002692222595,
      "learning_rate": 1.497927785721599e-06,
      "loss": 0.0068,
      "reward": 1.49201238155365,
      "reward_std": 0.7474342584609985,
      "rewards/reward_function": 1.49201238155365,
      "step": 1168
    },
    {
      "completion_length": 359.5357360839844,
      "epoch": 0.6687643020594966,
      "grad_norm": 2117.13232421875,
      "kl": 46.17893600463867,
      "learning_rate": 1.493355439969568e-06,
      "loss": 1.8472,
      "reward": 2.2535953521728516,
      "reward_std": 1.064886212348938,
      "rewards/reward_function": 2.2535953521728516,
      "step": 1169
    },
    {
      "completion_length": 285.5357360839844,
      "epoch": 0.6693363844393593,
      "grad_norm": 0.4708089530467987,
      "kl": 0.1723165661096573,
      "learning_rate": 1.488787109520383e-06,
      "loss": 0.0069,
      "reward": 2.2924251556396484,
      "reward_std": 0.7879359722137451,
      "rewards/reward_function": 2.2924251556396484,
      "step": 1170
    },
    {
      "completion_length": 351.0357360839844,
      "epoch": 0.669908466819222,
      "grad_norm": 0.2080860584974289,
      "kl": 0.15329474210739136,
      "learning_rate": 1.4842228125961978e-06,
      "loss": 0.0061,
      "reward": 1.5413949489593506,
      "reward_std": 0.9684211611747742,
      "rewards/reward_function": 1.5413949489593506,
      "step": 1171
    },
    {
      "completion_length": 345.2500305175781,
      "epoch": 0.6704805491990846,
      "grad_norm": 0.6354350447654724,
      "kl": 0.22366207838058472,
      "learning_rate": 1.4796625674030734e-06,
      "loss": 0.0089,
      "reward": 2.7437331676483154,
      "reward_std": 0.8869529962539673,
      "rewards/reward_function": 2.7437331676483154,
      "step": 1172
    },
    {
      "completion_length": 349.1785888671875,
      "epoch": 0.6710526315789473,
      "grad_norm": 0.5395358204841614,
      "kl": 0.14116345345973969,
      "learning_rate": 1.475106392130914e-06,
      "loss": 0.0056,
      "reward": 1.839153528213501,
      "reward_std": 1.010190725326538,
      "rewards/reward_function": 1.839153528213501,
      "step": 1173
    },
    {
      "completion_length": 349.6071472167969,
      "epoch": 0.6716247139588101,
      "grad_norm": 4.02215576171875,
      "kl": 0.7369453310966492,
      "learning_rate": 1.4705543049533843e-06,
      "loss": 0.0295,
      "reward": 1.3551433086395264,
      "reward_std": 1.0710238218307495,
      "rewards/reward_function": 1.3551433086395264,
      "step": 1174
    },
    {
      "completion_length": 323.64288330078125,
      "epoch": 0.6721967963386728,
      "grad_norm": 0.3182370364665985,
      "kl": 0.2238457202911377,
      "learning_rate": 1.4660063240278472e-06,
      "loss": 0.009,
      "reward": 1.6739535331726074,
      "reward_std": 1.0527147054672241,
      "rewards/reward_function": 1.6739535331726074,
      "step": 1175
    },
    {
      "completion_length": 317.5,
      "epoch": 0.6727688787185355,
      "grad_norm": 0.20498108863830566,
      "kl": 0.1923564374446869,
      "learning_rate": 1.4614624674952843e-06,
      "loss": 0.0077,
      "reward": 1.7354161739349365,
      "reward_std": 0.8597069382667542,
      "rewards/reward_function": 1.7354161739349365,
      "step": 1176
    },
    {
      "completion_length": 330.1071472167969,
      "epoch": 0.6733409610983981,
      "grad_norm": 0.26927220821380615,
      "kl": 0.1823723018169403,
      "learning_rate": 1.456922753480225e-06,
      "loss": 0.0073,
      "reward": 2.044210195541382,
      "reward_std": 1.0113441944122314,
      "rewards/reward_function": 2.044210195541382,
      "step": 1177
    },
    {
      "completion_length": 361.8571472167969,
      "epoch": 0.6739130434782609,
      "grad_norm": 0.2245093733072281,
      "kl": 0.14169907569885254,
      "learning_rate": 1.4523872000906785e-06,
      "loss": 0.0057,
      "reward": 2.2797977924346924,
      "reward_std": 0.8398749232292175,
      "rewards/reward_function": 2.2797977924346924,
      "step": 1178
    },
    {
      "completion_length": 304.1071472167969,
      "epoch": 0.6744851258581236,
      "grad_norm": 0.22986851632595062,
      "kl": 0.1340750902891159,
      "learning_rate": 1.4478558254180536e-06,
      "loss": 0.0054,
      "reward": 1.7031681537628174,
      "reward_std": 0.6509934663772583,
      "rewards/reward_function": 1.7031681537628174,
      "step": 1179
    },
    {
      "completion_length": 341.14288330078125,
      "epoch": 0.6750572082379863,
      "grad_norm": 0.16343948245048523,
      "kl": 0.15253639221191406,
      "learning_rate": 1.4433286475370945e-06,
      "loss": 0.0061,
      "reward": 2.2072997093200684,
      "reward_std": 1.1930735111236572,
      "rewards/reward_function": 2.2072997093200684,
      "step": 1180
    },
    {
      "completion_length": 304.3571472167969,
      "epoch": 0.6756292906178489,
      "grad_norm": 0.6404942870140076,
      "kl": 0.1430700570344925,
      "learning_rate": 1.438805684505803e-06,
      "loss": 0.0057,
      "reward": 2.178650379180908,
      "reward_std": 0.9477179646492004,
      "rewards/reward_function": 2.178650379180908,
      "step": 1181
    },
    {
      "completion_length": 370.0000305175781,
      "epoch": 0.6762013729977117,
      "grad_norm": 0.2539541721343994,
      "kl": 0.16365490853786469,
      "learning_rate": 1.4342869543653694e-06,
      "loss": 0.0065,
      "reward": 2.124086856842041,
      "reward_std": 0.7304180860519409,
      "rewards/reward_function": 2.124086856842041,
      "step": 1182
    },
    {
      "completion_length": 380.8571472167969,
      "epoch": 0.6767734553775744,
      "grad_norm": 0.42060017585754395,
      "kl": 0.2240549772977829,
      "learning_rate": 1.4297724751401012e-06,
      "loss": 0.009,
      "reward": 1.7882922887802124,
      "reward_std": 0.7127362489700317,
      "rewards/reward_function": 1.7882922887802124,
      "step": 1183
    },
    {
      "completion_length": 352.8214416503906,
      "epoch": 0.6773455377574371,
      "grad_norm": 0.7802943587303162,
      "kl": 0.4134688973426819,
      "learning_rate": 1.4252622648373482e-06,
      "loss": 0.0165,
      "reward": 1.8013181686401367,
      "reward_std": 0.5346314311027527,
      "rewards/reward_function": 1.8013181686401367,
      "step": 1184
    },
    {
      "completion_length": 295.5714416503906,
      "epoch": 0.6779176201372997,
      "grad_norm": 0.3461554944515228,
      "kl": 0.16658082604408264,
      "learning_rate": 1.4207563414474343e-06,
      "loss": 0.0067,
      "reward": 1.7877857685089111,
      "reward_std": 0.5231627225875854,
      "rewards/reward_function": 1.7877857685089111,
      "step": 1185
    },
    {
      "completion_length": 372.4285888671875,
      "epoch": 0.6784897025171625,
      "grad_norm": 95.81034851074219,
      "kl": 2.452226161956787,
      "learning_rate": 1.41625472294358e-06,
      "loss": 0.0981,
      "reward": 1.6333028078079224,
      "reward_std": 0.7996233701705933,
      "rewards/reward_function": 1.6333028078079224,
      "step": 1186
    },
    {
      "completion_length": 308.75,
      "epoch": 0.6790617848970252,
      "grad_norm": 0.19565363228321075,
      "kl": 0.1532176285982132,
      "learning_rate": 1.4117574272818388e-06,
      "loss": 0.0061,
      "reward": 2.0584938526153564,
      "reward_std": 0.9205247759819031,
      "rewards/reward_function": 2.0584938526153564,
      "step": 1187
    },
    {
      "completion_length": 360.6071472167969,
      "epoch": 0.6796338672768879,
      "grad_norm": 0.632017195224762,
      "kl": 0.19117864966392517,
      "learning_rate": 1.4072644724010175e-06,
      "loss": 0.0076,
      "reward": 1.4859278202056885,
      "reward_std": 0.7466169595718384,
      "rewards/reward_function": 1.4859278202056885,
      "step": 1188
    },
    {
      "completion_length": 338.6071472167969,
      "epoch": 0.6802059496567505,
      "grad_norm": 0.544471800327301,
      "kl": 0.2175166755914688,
      "learning_rate": 1.402775876222611e-06,
      "loss": 0.0087,
      "reward": 1.6642630100250244,
      "reward_std": 0.8802015781402588,
      "rewards/reward_function": 1.6642630100250244,
      "step": 1189
    },
    {
      "completion_length": 316.14288330078125,
      "epoch": 0.6807780320366132,
      "grad_norm": 0.22017423808574677,
      "kl": 0.14212356507778168,
      "learning_rate": 1.3982916566507276e-06,
      "loss": 0.0057,
      "reward": 2.230206251144409,
      "reward_std": 1.1462205648422241,
      "rewards/reward_function": 2.230206251144409,
      "step": 1190
    },
    {
      "completion_length": 396.1785888671875,
      "epoch": 0.681350114416476,
      "grad_norm": 0.21290507912635803,
      "kl": 0.16016808152198792,
      "learning_rate": 1.3938118315720186e-06,
      "loss": 0.0064,
      "reward": 2.0184619426727295,
      "reward_std": 0.7850914597511292,
      "rewards/reward_function": 2.0184619426727295,
      "step": 1191
    },
    {
      "completion_length": 302.75,
      "epoch": 0.6819221967963387,
      "grad_norm": 0.17241616547107697,
      "kl": 0.12829279899597168,
      "learning_rate": 1.3893364188556044e-06,
      "loss": 0.0051,
      "reward": 1.5443209409713745,
      "reward_std": 0.7788754105567932,
      "rewards/reward_function": 1.5443209409713745,
      "step": 1192
    },
    {
      "completion_length": 326.14288330078125,
      "epoch": 0.6824942791762014,
      "grad_norm": 0.30140620470046997,
      "kl": 0.14784406125545502,
      "learning_rate": 1.3848654363530078e-06,
      "loss": 0.0059,
      "reward": 1.7205493450164795,
      "reward_std": 1.0921719074249268,
      "rewards/reward_function": 1.7205493450164795,
      "step": 1193
    },
    {
      "completion_length": 333.0,
      "epoch": 0.683066361556064,
      "grad_norm": 0.3123094439506531,
      "kl": 0.15629152953624725,
      "learning_rate": 1.3803989018980801e-06,
      "loss": 0.0063,
      "reward": 1.8342671394348145,
      "reward_std": 0.7073037028312683,
      "rewards/reward_function": 1.8342671394348145,
      "step": 1194
    },
    {
      "completion_length": 314.1785888671875,
      "epoch": 0.6836384439359268,
      "grad_norm": 0.2403012365102768,
      "kl": 0.15080390870571136,
      "learning_rate": 1.3759368333069274e-06,
      "loss": 0.006,
      "reward": 2.0053553581237793,
      "reward_std": 0.7569484114646912,
      "rewards/reward_function": 2.0053553581237793,
      "step": 1195
    },
    {
      "completion_length": 320.39288330078125,
      "epoch": 0.6842105263157895,
      "grad_norm": 0.3336333632469177,
      "kl": 0.18663755059242249,
      "learning_rate": 1.3714792483778477e-06,
      "loss": 0.0075,
      "reward": 2.0294008255004883,
      "reward_std": 0.8317741751670837,
      "rewards/reward_function": 2.0294008255004883,
      "step": 1196
    },
    {
      "completion_length": 351.71429443359375,
      "epoch": 0.6847826086956522,
      "grad_norm": 0.17286840081214905,
      "kl": 0.14103923738002777,
      "learning_rate": 1.367026164891249e-06,
      "loss": 0.0056,
      "reward": 2.3622868061065674,
      "reward_std": 0.9955798983573914,
      "rewards/reward_function": 2.3622868061065674,
      "step": 1197
    },
    {
      "completion_length": 265.7857360839844,
      "epoch": 0.6853546910755148,
      "grad_norm": 0.2483995109796524,
      "kl": 0.1840333491563797,
      "learning_rate": 1.3625776006095882e-06,
      "loss": 0.0074,
      "reward": 2.0969409942626953,
      "reward_std": 1.2313801050186157,
      "rewards/reward_function": 2.0969409942626953,
      "step": 1198
    },
    {
      "completion_length": 314.46429443359375,
      "epoch": 0.6859267734553776,
      "grad_norm": 0.17885664105415344,
      "kl": 0.13237479329109192,
      "learning_rate": 1.3581335732772922e-06,
      "loss": 0.0053,
      "reward": 2.0800604820251465,
      "reward_std": 1.002368688583374,
      "rewards/reward_function": 2.0800604820251465,
      "step": 1199
    },
    {
      "completion_length": 324.0,
      "epoch": 0.6864988558352403,
      "grad_norm": 0.18450875580310822,
      "kl": 0.1595897078514099,
      "learning_rate": 1.353694100620694e-06,
      "loss": 0.0064,
      "reward": 2.3114984035491943,
      "reward_std": 0.8189448118209839,
      "rewards/reward_function": 2.3114984035491943,
      "step": 1200
    },
    {
      "completion_length": 415.8571472167969,
      "epoch": 0.687070938215103,
      "grad_norm": 0.23664912581443787,
      "kl": 0.19611868262290955,
      "learning_rate": 1.3492592003479577e-06,
      "loss": 0.0078,
      "reward": 1.919775366783142,
      "reward_std": 1.3345247507095337,
      "rewards/reward_function": 1.919775366783142,
      "step": 1201
    },
    {
      "completion_length": 329.5714416503906,
      "epoch": 0.6876430205949656,
      "grad_norm": 0.1735672652721405,
      "kl": 0.1364441066980362,
      "learning_rate": 1.3448288901490094e-06,
      "loss": 0.0055,
      "reward": 1.7873778343200684,
      "reward_std": 0.8339072465896606,
      "rewards/reward_function": 1.7873778343200684,
      "step": 1202
    },
    {
      "completion_length": 348.21429443359375,
      "epoch": 0.6882151029748284,
      "grad_norm": 0.4026508927345276,
      "kl": 0.17807237803936005,
      "learning_rate": 1.3404031876954671e-06,
      "loss": 0.0071,
      "reward": 1.6773946285247803,
      "reward_std": 0.7374661564826965,
      "rewards/reward_function": 1.6773946285247803,
      "step": 1203
    },
    {
      "completion_length": 327.5357360839844,
      "epoch": 0.6887871853546911,
      "grad_norm": 0.24898108839988708,
      "kl": 0.16023042798042297,
      "learning_rate": 1.3359821106405662e-06,
      "loss": 0.0064,
      "reward": 2.236689567565918,
      "reward_std": 1.2121977806091309,
      "rewards/reward_function": 2.236689567565918,
      "step": 1204
    },
    {
      "completion_length": 332.25,
      "epoch": 0.6893592677345538,
      "grad_norm": 0.2994310259819031,
      "kl": 0.19694827497005463,
      "learning_rate": 1.3315656766190971e-06,
      "loss": 0.0079,
      "reward": 2.0629758834838867,
      "reward_std": 0.8150213956832886,
      "rewards/reward_function": 2.0629758834838867,
      "step": 1205
    },
    {
      "completion_length": 353.7857360839844,
      "epoch": 0.6899313501144165,
      "grad_norm": 2.7753756046295166,
      "kl": 0.17018505930900574,
      "learning_rate": 1.327153903247325e-06,
      "loss": 0.0068,
      "reward": 2.025628089904785,
      "reward_std": 0.9706907868385315,
      "rewards/reward_function": 2.025628089904785,
      "step": 1206
    },
    {
      "completion_length": 323.0357360839844,
      "epoch": 0.6905034324942791,
      "grad_norm": 0.49145349860191345,
      "kl": 0.1672387570142746,
      "learning_rate": 1.3227468081229286e-06,
      "loss": 0.0067,
      "reward": 2.018901824951172,
      "reward_std": 1.118575930595398,
      "rewards/reward_function": 2.018901824951172,
      "step": 1207
    },
    {
      "completion_length": 294.0357360839844,
      "epoch": 0.6910755148741419,
      "grad_norm": 3.494807243347168,
      "kl": 0.2745448350906372,
      "learning_rate": 1.3183444088249244e-06,
      "loss": 0.011,
      "reward": 2.433046817779541,
      "reward_std": 0.43890145421028137,
      "rewards/reward_function": 2.433046817779541,
      "step": 1208
    },
    {
      "completion_length": 285.6785888671875,
      "epoch": 0.6916475972540046,
      "grad_norm": 0.18900363147258759,
      "kl": 0.12232780456542969,
      "learning_rate": 1.3139467229135999e-06,
      "loss": 0.0049,
      "reward": 1.9271299839019775,
      "reward_std": 0.9365607500076294,
      "rewards/reward_function": 1.9271299839019775,
      "step": 1209
    },
    {
      "completion_length": 384.4285888671875,
      "epoch": 0.6922196796338673,
      "grad_norm": 14.187193870544434,
      "kl": 1.7001057863235474,
      "learning_rate": 1.309553767930438e-06,
      "loss": 0.068,
      "reward": 2.4286515712738037,
      "reward_std": 0.8855085968971252,
      "rewards/reward_function": 2.4286515712738037,
      "step": 1210
    },
    {
      "completion_length": 341.14288330078125,
      "epoch": 0.6927917620137299,
      "grad_norm": 3.122781753540039,
      "kl": 0.1773904263973236,
      "learning_rate": 1.3051655613980544e-06,
      "loss": 0.0071,
      "reward": 1.169024109840393,
      "reward_std": 0.7596145868301392,
      "rewards/reward_function": 1.169024109840393,
      "step": 1211
    },
    {
      "completion_length": 350.8571472167969,
      "epoch": 0.6933638443935927,
      "grad_norm": 0.20104488730430603,
      "kl": 0.1493329554796219,
      "learning_rate": 1.3007821208201238e-06,
      "loss": 0.006,
      "reward": 2.610781192779541,
      "reward_std": 0.917763352394104,
      "rewards/reward_function": 2.610781192779541,
      "step": 1212
    },
    {
      "completion_length": 386.71429443359375,
      "epoch": 0.6939359267734554,
      "grad_norm": 0.4620758891105652,
      "kl": 0.1828593611717224,
      "learning_rate": 1.2964034636813075e-06,
      "loss": 0.0073,
      "reward": 1.5278983116149902,
      "reward_std": 0.6334531903266907,
      "rewards/reward_function": 1.5278983116149902,
      "step": 1213
    },
    {
      "completion_length": 320.5,
      "epoch": 0.6945080091533181,
      "grad_norm": 0.2159460037946701,
      "kl": 0.1353958696126938,
      "learning_rate": 1.2920296074471921e-06,
      "loss": 0.0054,
      "reward": 2.054551839828491,
      "reward_std": 0.5432683229446411,
      "rewards/reward_function": 2.054551839828491,
      "step": 1214
    },
    {
      "completion_length": 355.96429443359375,
      "epoch": 0.6950800915331807,
      "grad_norm": 0.3821556866168976,
      "kl": 0.1773122400045395,
      "learning_rate": 1.2876605695642086e-06,
      "loss": 0.0071,
      "reward": 2.0401101112365723,
      "reward_std": 0.6662929058074951,
      "rewards/reward_function": 2.0401101112365723,
      "step": 1215
    },
    {
      "completion_length": 331.75,
      "epoch": 0.6956521739130435,
      "grad_norm": 0.9766490459442139,
      "kl": 0.17596913874149323,
      "learning_rate": 1.2832963674595727e-06,
      "loss": 0.007,
      "reward": 1.822312355041504,
      "reward_std": 0.8206509351730347,
      "rewards/reward_function": 1.822312355041504,
      "step": 1216
    },
    {
      "completion_length": 362.5000305175781,
      "epoch": 0.6962242562929062,
      "grad_norm": 14.428997039794922,
      "kl": 0.5743314623832703,
      "learning_rate": 1.2789370185412077e-06,
      "loss": 0.023,
      "reward": 2.3049702644348145,
      "reward_std": 0.9289882183074951,
      "rewards/reward_function": 2.3049702644348145,
      "step": 1217
    },
    {
      "completion_length": 312.64288330078125,
      "epoch": 0.6967963386727689,
      "grad_norm": 0.189418762922287,
      "kl": 0.14099538326263428,
      "learning_rate": 1.2745825401976814e-06,
      "loss": 0.0056,
      "reward": 1.9908391237258911,
      "reward_std": 0.7928318381309509,
      "rewards/reward_function": 1.9908391237258911,
      "step": 1218
    },
    {
      "completion_length": 344.0357360839844,
      "epoch": 0.6973684210526315,
      "grad_norm": 5.057568073272705,
      "kl": 1.0808563232421875,
      "learning_rate": 1.2702329497981326e-06,
      "loss": 0.0432,
      "reward": 2.289245128631592,
      "reward_std": 0.9295623302459717,
      "rewards/reward_function": 2.289245128631592,
      "step": 1219
    },
    {
      "completion_length": 369.8571472167969,
      "epoch": 0.6979405034324943,
      "grad_norm": 0.2727314531803131,
      "kl": 0.1977793425321579,
      "learning_rate": 1.2658882646922036e-06,
      "loss": 0.0079,
      "reward": 2.524918556213379,
      "reward_std": 0.9903700351715088,
      "rewards/reward_function": 2.524918556213379,
      "step": 1220
    },
    {
      "completion_length": 322.71429443359375,
      "epoch": 0.698512585812357,
      "grad_norm": 0.7971591949462891,
      "kl": 0.34511029720306396,
      "learning_rate": 1.2615485022099709e-06,
      "loss": 0.0138,
      "reward": 1.6733702421188354,
      "reward_std": 0.9126988053321838,
      "rewards/reward_function": 1.6733702421188354,
      "step": 1221
    },
    {
      "completion_length": 337.0357360839844,
      "epoch": 0.6990846681922197,
      "grad_norm": 0.23309069871902466,
      "kl": 0.15291273593902588,
      "learning_rate": 1.2572136796618728e-06,
      "loss": 0.0061,
      "reward": 2.3840038776397705,
      "reward_std": 1.1558811664581299,
      "rewards/reward_function": 2.3840038776397705,
      "step": 1222
    },
    {
      "completion_length": 367.9285888671875,
      "epoch": 0.6996567505720824,
      "grad_norm": 0.2897517681121826,
      "kl": 0.1536458283662796,
      "learning_rate": 1.2528838143386474e-06,
      "loss": 0.0061,
      "reward": 1.8396114110946655,
      "reward_std": 0.8631329536437988,
      "rewards/reward_function": 1.8396114110946655,
      "step": 1223
    },
    {
      "completion_length": 364.0357360839844,
      "epoch": 0.700228832951945,
      "grad_norm": 0.2805930972099304,
      "kl": 0.1573413461446762,
      "learning_rate": 1.2485589235112557e-06,
      "loss": 0.0063,
      "reward": 1.8642938137054443,
      "reward_std": 0.9585059285163879,
      "rewards/reward_function": 1.8642938137054443,
      "step": 1224
    },
    {
      "completion_length": 344.1071472167969,
      "epoch": 0.7008009153318078,
      "grad_norm": 0.7965362668037415,
      "kl": 0.1555909365415573,
      "learning_rate": 1.2442390244308185e-06,
      "loss": 0.0062,
      "reward": 1.837661862373352,
      "reward_std": 0.9334724545478821,
      "rewards/reward_function": 1.837661862373352,
      "step": 1225
    },
    {
      "completion_length": 320.0714416503906,
      "epoch": 0.7013729977116705,
      "grad_norm": 1.3954691886901855,
      "kl": 0.24509698152542114,
      "learning_rate": 1.2399241343285453e-06,
      "loss": 0.0098,
      "reward": 1.656089186668396,
      "reward_std": 1.152642846107483,
      "rewards/reward_function": 1.656089186668396,
      "step": 1226
    },
    {
      "completion_length": 349.71429443359375,
      "epoch": 0.7019450800915332,
      "grad_norm": 2.2701733112335205,
      "kl": 0.21291343867778778,
      "learning_rate": 1.2356142704156657e-06,
      "loss": 0.0085,
      "reward": 2.2620694637298584,
      "reward_std": 0.6368263363838196,
      "rewards/reward_function": 2.2620694637298584,
      "step": 1227
    },
    {
      "completion_length": 352.2857360839844,
      "epoch": 0.7025171624713958,
      "grad_norm": 0.45842766761779785,
      "kl": 0.21976548433303833,
      "learning_rate": 1.2313094498833612e-06,
      "loss": 0.0088,
      "reward": 1.740431308746338,
      "reward_std": 1.0705604553222656,
      "rewards/reward_function": 1.740431308746338,
      "step": 1228
    },
    {
      "completion_length": 342.39288330078125,
      "epoch": 0.7030892448512586,
      "grad_norm": 0.16942894458770752,
      "kl": 0.16834107041358948,
      "learning_rate": 1.2270096899026942e-06,
      "loss": 0.0067,
      "reward": 2.0022823810577393,
      "reward_std": 1.0608819723129272,
      "rewards/reward_function": 2.0022823810577393,
      "step": 1229
    },
    {
      "completion_length": 321.5714416503906,
      "epoch": 0.7036613272311213,
      "grad_norm": 2.286022424697876,
      "kl": 0.29320645332336426,
      "learning_rate": 1.2227150076245452e-06,
      "loss": 0.0117,
      "reward": 1.5047613382339478,
      "reward_std": 1.1099146604537964,
      "rewards/reward_function": 1.5047613382339478,
      "step": 1230
    },
    {
      "completion_length": 307.2857360839844,
      "epoch": 0.704233409610984,
      "grad_norm": 0.46743884682655334,
      "kl": 0.1351780742406845,
      "learning_rate": 1.2184254201795364e-06,
      "loss": 0.0054,
      "reward": 1.6873677968978882,
      "reward_std": 0.4649975299835205,
      "rewards/reward_function": 1.6873677968978882,
      "step": 1231
    },
    {
      "completion_length": 302.89288330078125,
      "epoch": 0.7048054919908466,
      "grad_norm": 2.0684986114501953,
      "kl": 0.3510928750038147,
      "learning_rate": 1.2141409446779713e-06,
      "loss": 0.014,
      "reward": 1.8896986246109009,
      "reward_std": 0.5637810826301575,
      "rewards/reward_function": 1.8896986246109009,
      "step": 1232
    },
    {
      "completion_length": 359.6071472167969,
      "epoch": 0.7053775743707094,
      "grad_norm": 0.17563658952713013,
      "kl": 0.13139581680297852,
      "learning_rate": 1.2098615982097617e-06,
      "loss": 0.0053,
      "reward": 1.9720733165740967,
      "reward_std": 0.9006944298744202,
      "rewards/reward_function": 1.9720733165740967,
      "step": 1233
    },
    {
      "completion_length": 316.2857360839844,
      "epoch": 0.7059496567505721,
      "grad_norm": 0.18203264474868774,
      "kl": 0.1162971630692482,
      "learning_rate": 1.2055873978443613e-06,
      "loss": 0.0047,
      "reward": 1.890092134475708,
      "reward_std": 0.8179863691329956,
      "rewards/reward_function": 1.890092134475708,
      "step": 1234
    },
    {
      "completion_length": 287.7857360839844,
      "epoch": 0.7065217391304348,
      "grad_norm": 0.19999311864376068,
      "kl": 0.12704116106033325,
      "learning_rate": 1.2013183606306947e-06,
      "loss": 0.0051,
      "reward": 2.1923043727874756,
      "reward_std": 0.7301133871078491,
      "rewards/reward_function": 2.1923043727874756,
      "step": 1235
    },
    {
      "completion_length": 308.14288330078125,
      "epoch": 0.7070938215102975,
      "grad_norm": 0.2724687159061432,
      "kl": 0.14152126014232635,
      "learning_rate": 1.1970545035970942e-06,
      "loss": 0.0057,
      "reward": 1.787703514099121,
      "reward_std": 0.69990074634552,
      "rewards/reward_function": 1.787703514099121,
      "step": 1236
    },
    {
      "completion_length": 330.6071472167969,
      "epoch": 0.7076659038901602,
      "grad_norm": 677.69775390625,
      "kl": 8.739831924438477,
      "learning_rate": 1.1927958437512292e-06,
      "loss": 0.3496,
      "reward": 1.8468170166015625,
      "reward_std": 0.5492849349975586,
      "rewards/reward_function": 1.8468170166015625,
      "step": 1237
    },
    {
      "completion_length": 394.89288330078125,
      "epoch": 0.7082379862700229,
      "grad_norm": 0.18053746223449707,
      "kl": 0.13354045152664185,
      "learning_rate": 1.1885423980800368e-06,
      "loss": 0.0053,
      "reward": 1.447619915008545,
      "reward_std": 0.9977588653564453,
      "rewards/reward_function": 1.447619915008545,
      "step": 1238
    },
    {
      "completion_length": 398.5714416503906,
      "epoch": 0.7088100686498856,
      "grad_norm": 0.32204511761665344,
      "kl": 0.22526976466178894,
      "learning_rate": 1.1842941835496587e-06,
      "loss": 0.009,
      "reward": 1.7912590503692627,
      "reward_std": 1.1032133102416992,
      "rewards/reward_function": 1.7912590503692627,
      "step": 1239
    },
    {
      "completion_length": 401.5357360839844,
      "epoch": 0.7093821510297483,
      "grad_norm": 3.192819356918335,
      "kl": 0.5321924686431885,
      "learning_rate": 1.1800512171053674e-06,
      "loss": 0.0213,
      "reward": 2.193652868270874,
      "reward_std": 1.1894315481185913,
      "rewards/reward_function": 2.193652868270874,
      "step": 1240
    },
    {
      "completion_length": 325.5,
      "epoch": 0.709954233409611,
      "grad_norm": 0.17416207492351532,
      "kl": 0.14036276936531067,
      "learning_rate": 1.1758135156715043e-06,
      "loss": 0.0056,
      "reward": 1.9674803018569946,
      "reward_std": 0.7511993646621704,
      "rewards/reward_function": 1.9674803018569946,
      "step": 1241
    },
    {
      "completion_length": 313.14288330078125,
      "epoch": 0.7105263157894737,
      "grad_norm": 0.18654181063175201,
      "kl": 0.14174161851406097,
      "learning_rate": 1.1715810961514073e-06,
      "loss": 0.0057,
      "reward": 1.57412588596344,
      "reward_std": 0.7744311094284058,
      "rewards/reward_function": 1.57412588596344,
      "step": 1242
    },
    {
      "completion_length": 305.21429443359375,
      "epoch": 0.7110983981693364,
      "grad_norm": 0.7104048132896423,
      "kl": 0.1518244594335556,
      "learning_rate": 1.1673539754273485e-06,
      "loss": 0.0061,
      "reward": 2.338305711746216,
      "reward_std": 0.5267112851142883,
      "rewards/reward_function": 2.338305711746216,
      "step": 1243
    },
    {
      "completion_length": 329.3571472167969,
      "epoch": 0.7116704805491991,
      "grad_norm": 0.6441555619239807,
      "kl": 0.1850856989622116,
      "learning_rate": 1.1631321703604629e-06,
      "loss": 0.0074,
      "reward": 2.4280171394348145,
      "reward_std": 0.7338711023330688,
      "rewards/reward_function": 2.4280171394348145,
      "step": 1244
    },
    {
      "completion_length": 372.5714416503906,
      "epoch": 0.7122425629290617,
      "grad_norm": 0.28527653217315674,
      "kl": 0.15086595714092255,
      "learning_rate": 1.1589156977906819e-06,
      "loss": 0.006,
      "reward": 1.681644320487976,
      "reward_std": 0.9054322838783264,
      "rewards/reward_function": 1.681644320487976,
      "step": 1245
    },
    {
      "completion_length": 352.71429443359375,
      "epoch": 0.7128146453089245,
      "grad_norm": 0.7930838465690613,
      "kl": 0.16975818574428558,
      "learning_rate": 1.1547045745366686e-06,
      "loss": 0.0068,
      "reward": 2.1853179931640625,
      "reward_std": 0.8814895153045654,
      "rewards/reward_function": 2.1853179931640625,
      "step": 1246
    },
    {
      "completion_length": 295.2857360839844,
      "epoch": 0.7133867276887872,
      "grad_norm": 0.37143784761428833,
      "kl": 0.14931224286556244,
      "learning_rate": 1.1504988173957455e-06,
      "loss": 0.006,
      "reward": 1.9805188179016113,
      "reward_std": 1.1497629880905151,
      "rewards/reward_function": 1.9805188179016113,
      "step": 1247
    },
    {
      "completion_length": 305.2857360839844,
      "epoch": 0.7139588100686499,
      "grad_norm": 1.8873450756072998,
      "kl": 0.623467743396759,
      "learning_rate": 1.1462984431438346e-06,
      "loss": 0.0249,
      "reward": 1.7161389589309692,
      "reward_std": 1.0998311042785645,
      "rewards/reward_function": 1.7161389589309692,
      "step": 1248
    },
    {
      "completion_length": 318.1785888671875,
      "epoch": 0.7145308924485125,
      "grad_norm": 0.22110435366630554,
      "kl": 0.12978622317314148,
      "learning_rate": 1.1421034685353821e-06,
      "loss": 0.0052,
      "reward": 2.7661547660827637,
      "reward_std": 0.4049433469772339,
      "rewards/reward_function": 2.7661547660827637,
      "step": 1249
    },
    {
      "completion_length": 351.89288330078125,
      "epoch": 0.7151029748283753,
      "grad_norm": 0.21057532727718353,
      "kl": 0.1728118509054184,
      "learning_rate": 1.1379139103033005e-06,
      "loss": 0.0069,
      "reward": 1.6439481973648071,
      "reward_std": 0.9920302033424377,
      "rewards/reward_function": 1.6439481973648071,
      "step": 1250
    },
    {
      "completion_length": 316.71429443359375,
      "epoch": 0.715675057208238,
      "grad_norm": 1.7699816226959229,
      "kl": 0.20096100866794586,
      "learning_rate": 1.133729785158895e-06,
      "loss": 0.008,
      "reward": 2.2876102924346924,
      "reward_std": 0.9473558068275452,
      "rewards/reward_function": 2.2876102924346924,
      "step": 1251
    },
    {
      "completion_length": 317.3214416503906,
      "epoch": 0.7162471395881007,
      "grad_norm": 0.47391194105148315,
      "kl": 0.19094522297382355,
      "learning_rate": 1.129551109791801e-06,
      "loss": 0.0076,
      "reward": 1.8132191896438599,
      "reward_std": 0.917185366153717,
      "rewards/reward_function": 1.8132191896438599,
      "step": 1252
    },
    {
      "completion_length": 301.1785888671875,
      "epoch": 0.7168192219679634,
      "grad_norm": 499.9647521972656,
      "kl": 9.18834114074707,
      "learning_rate": 1.1253779008699131e-06,
      "loss": 0.3675,
      "reward": 1.972935438156128,
      "reward_std": 0.8966747522354126,
      "rewards/reward_function": 1.972935438156128,
      "step": 1253
    },
    {
      "completion_length": 332.14288330078125,
      "epoch": 0.717391304347826,
      "grad_norm": 0.262636661529541,
      "kl": 0.1473575383424759,
      "learning_rate": 1.1212101750393236e-06,
      "loss": 0.0059,
      "reward": 2.5315792560577393,
      "reward_std": 0.5666507482528687,
      "rewards/reward_function": 2.5315792560577393,
      "step": 1254
    },
    {
      "completion_length": 350.1785888671875,
      "epoch": 0.7179633867276888,
      "grad_norm": 266.7964172363281,
      "kl": 1.888623595237732,
      "learning_rate": 1.1170479489242537e-06,
      "loss": 0.0755,
      "reward": 2.5477943420410156,
      "reward_std": 0.9974485039710999,
      "rewards/reward_function": 2.5477943420410156,
      "step": 1255
    },
    {
      "completion_length": 360.5000305175781,
      "epoch": 0.7185354691075515,
      "grad_norm": 4.3814568519592285,
      "kl": 0.459209680557251,
      "learning_rate": 1.1128912391269864e-06,
      "loss": 0.0184,
      "reward": 1.7319289445877075,
      "reward_std": 0.8447801470756531,
      "rewards/reward_function": 1.7319289445877075,
      "step": 1256
    },
    {
      "completion_length": 321.4285888671875,
      "epoch": 0.7191075514874142,
      "grad_norm": 2.2790849208831787,
      "kl": 0.46113207936286926,
      "learning_rate": 1.108740062227803e-06,
      "loss": 0.0184,
      "reward": 1.871462345123291,
      "reward_std": 0.7232704758644104,
      "rewards/reward_function": 1.871462345123291,
      "step": 1257
    },
    {
      "completion_length": 367.2857360839844,
      "epoch": 0.7196796338672768,
      "grad_norm": 0.19819217920303345,
      "kl": 0.13735665380954742,
      "learning_rate": 1.1045944347849119e-06,
      "loss": 0.0055,
      "reward": 2.3844153881073,
      "reward_std": 1.1316884756088257,
      "rewards/reward_function": 2.3844153881073,
      "step": 1258
    },
    {
      "completion_length": 371.0714416503906,
      "epoch": 0.7202517162471396,
      "grad_norm": 98302.9140625,
      "kl": 3149.480224609375,
      "learning_rate": 1.1004543733343898e-06,
      "loss": 125.9792,
      "reward": 1.8329328298568726,
      "reward_std": 0.5571037530899048,
      "rewards/reward_function": 1.8329328298568726,
      "step": 1259
    },
    {
      "completion_length": 325.8571472167969,
      "epoch": 0.7208237986270023,
      "grad_norm": 0.23273810744285583,
      "kl": 0.18492360413074493,
      "learning_rate": 1.096319894390108e-06,
      "loss": 0.0074,
      "reward": 1.7074919939041138,
      "reward_std": 0.6979931592941284,
      "rewards/reward_function": 1.7074919939041138,
      "step": 1260
    },
    {
      "completion_length": 331.14288330078125,
      "epoch": 0.721395881006865,
      "grad_norm": 0.6561976671218872,
      "kl": 0.20930834114551544,
      "learning_rate": 1.0921910144436726e-06,
      "loss": 0.0084,
      "reward": 2.4557125568389893,
      "reward_std": 1.0106416940689087,
      "rewards/reward_function": 2.4557125568389893,
      "step": 1261
    },
    {
      "completion_length": 348.5000305175781,
      "epoch": 0.7219679633867276,
      "grad_norm": 5.608922004699707,
      "kl": 0.43487533926963806,
      "learning_rate": 1.0880677499643563e-06,
      "loss": 0.0174,
      "reward": 2.581695079803467,
      "reward_std": 0.7944502234458923,
      "rewards/reward_function": 2.581695079803467,
      "step": 1262
    },
    {
      "completion_length": 327.4285888671875,
      "epoch": 0.7225400457665904,
      "grad_norm": 0.3129083812236786,
      "kl": 0.15374386310577393,
      "learning_rate": 1.0839501173990325e-06,
      "loss": 0.0061,
      "reward": 2.5264534950256348,
      "reward_std": 0.8018366098403931,
      "rewards/reward_function": 2.5264534950256348,
      "step": 1263
    },
    {
      "completion_length": 382.7857360839844,
      "epoch": 0.7231121281464531,
      "grad_norm": 0.21425148844718933,
      "kl": 0.1458859145641327,
      "learning_rate": 1.079838133172111e-06,
      "loss": 0.0058,
      "reward": 2.019134283065796,
      "reward_std": 0.623065173625946,
      "rewards/reward_function": 2.019134283065796,
      "step": 1264
    },
    {
      "completion_length": 355.46429443359375,
      "epoch": 0.7236842105263158,
      "grad_norm": 1.769576072692871,
      "kl": 0.31142860651016235,
      "learning_rate": 1.0757318136854682e-06,
      "loss": 0.0125,
      "reward": 1.7880967855453491,
      "reward_std": 0.9522236585617065,
      "rewards/reward_function": 1.7880967855453491,
      "step": 1265
    },
    {
      "completion_length": 355.5714416503906,
      "epoch": 0.7242562929061785,
      "grad_norm": 0.18841975927352905,
      "kl": 0.19373519718647003,
      "learning_rate": 1.0716311753183895e-06,
      "loss": 0.0077,
      "reward": 1.5119515657424927,
      "reward_std": 0.6754130125045776,
      "rewards/reward_function": 1.5119515657424927,
      "step": 1266
    },
    {
      "completion_length": 292.0357360839844,
      "epoch": 0.7248283752860412,
      "grad_norm": 2.5387790203094482,
      "kl": 0.3730827569961548,
      "learning_rate": 1.0675362344274953e-06,
      "loss": 0.0149,
      "reward": 2.7591683864593506,
      "reward_std": 0.8548004031181335,
      "rewards/reward_function": 2.7591683864593506,
      "step": 1267
    },
    {
      "completion_length": 387.5357360839844,
      "epoch": 0.7254004576659039,
      "grad_norm": 2.6366026401519775,
      "kl": 0.18145853281021118,
      "learning_rate": 1.063447007346683e-06,
      "loss": 0.0073,
      "reward": 2.243879795074463,
      "reward_std": 0.6763830780982971,
      "rewards/reward_function": 2.243879795074463,
      "step": 1268
    },
    {
      "completion_length": 340.4285888671875,
      "epoch": 0.7259725400457666,
      "grad_norm": 0.4228467047214508,
      "kl": 0.14436636865139008,
      "learning_rate": 1.0593635103870572e-06,
      "loss": 0.0058,
      "reward": 2.2148690223693848,
      "reward_std": 0.9813318848609924,
      "rewards/reward_function": 2.2148690223693848,
      "step": 1269
    },
    {
      "completion_length": 323.14288330078125,
      "epoch": 0.7265446224256293,
      "grad_norm": 0.8638460636138916,
      "kl": 0.13321082293987274,
      "learning_rate": 1.055285759836868e-06,
      "loss": 0.0053,
      "reward": 2.5417776107788086,
      "reward_std": 0.5175011157989502,
      "rewards/reward_function": 2.5417776107788086,
      "step": 1270
    },
    {
      "completion_length": 374.5714416503906,
      "epoch": 0.727116704805492,
      "grad_norm": 1.6892176866531372,
      "kl": 0.24965481460094452,
      "learning_rate": 1.0512137719614407e-06,
      "loss": 0.01,
      "reward": 2.2925825119018555,
      "reward_std": 1.028584361076355,
      "rewards/reward_function": 2.2925825119018555,
      "step": 1271
    },
    {
      "completion_length": 361.3214416503906,
      "epoch": 0.7276887871853547,
      "grad_norm": 0.48626020550727844,
      "kl": 0.1447243094444275,
      "learning_rate": 1.0471475630031175e-06,
      "loss": 0.0058,
      "reward": 2.7179200649261475,
      "reward_std": 0.9969215393066406,
      "rewards/reward_function": 2.7179200649261475,
      "step": 1272
    },
    {
      "completion_length": 328.3571472167969,
      "epoch": 0.7282608695652174,
      "grad_norm": 0.21275663375854492,
      "kl": 0.13030372560024261,
      "learning_rate": 1.043087149181189e-06,
      "loss": 0.0052,
      "reward": 1.8447375297546387,
      "reward_std": 0.9471205472946167,
      "rewards/reward_function": 1.8447375297546387,
      "step": 1273
    },
    {
      "completion_length": 319.6785888671875,
      "epoch": 0.7288329519450801,
      "grad_norm": 0.4084039330482483,
      "kl": 0.12615469098091125,
      "learning_rate": 1.0390325466918296e-06,
      "loss": 0.005,
      "reward": 1.6070141792297363,
      "reward_std": 0.672260046005249,
      "rewards/reward_function": 1.6070141792297363,
      "step": 1274
    },
    {
      "completion_length": 348.4285888671875,
      "epoch": 0.7294050343249427,
      "grad_norm": 6.941281795501709,
      "kl": 0.8467923402786255,
      "learning_rate": 1.034983771708035e-06,
      "loss": 0.0339,
      "reward": 1.4682815074920654,
      "reward_std": 0.9348545074462891,
      "rewards/reward_function": 1.4682815074920654,
      "step": 1275
    },
    {
      "completion_length": 328.6785888671875,
      "epoch": 0.7299771167048055,
      "grad_norm": 0.2109302431344986,
      "kl": 0.14610545337200165,
      "learning_rate": 1.0309408403795528e-06,
      "loss": 0.0058,
      "reward": 2.756596565246582,
      "reward_std": 0.6928213834762573,
      "rewards/reward_function": 2.756596565246582,
      "step": 1276
    },
    {
      "completion_length": 329.7857360839844,
      "epoch": 0.7305491990846682,
      "grad_norm": 0.9451284408569336,
      "kl": 0.1513671725988388,
      "learning_rate": 1.0269037688328258e-06,
      "loss": 0.0061,
      "reward": 1.9599860906600952,
      "reward_std": 1.110528588294983,
      "rewards/reward_function": 1.9599860906600952,
      "step": 1277
    },
    {
      "completion_length": 330.0,
      "epoch": 0.7311212814645309,
      "grad_norm": 0.611292839050293,
      "kl": 0.13649559020996094,
      "learning_rate": 1.022872573170919e-06,
      "loss": 0.0055,
      "reward": 1.9023939371109009,
      "reward_std": 0.7799933552742004,
      "rewards/reward_function": 1.9023939371109009,
      "step": 1278
    },
    {
      "completion_length": 315.8214416503906,
      "epoch": 0.7316933638443935,
      "grad_norm": 11.249259948730469,
      "kl": 0.16968561708927155,
      "learning_rate": 1.0188472694734628e-06,
      "loss": 0.0068,
      "reward": 1.893841028213501,
      "reward_std": 1.0820512771606445,
      "rewards/reward_function": 1.893841028213501,
      "step": 1279
    },
    {
      "completion_length": 329.39288330078125,
      "epoch": 0.7322654462242563,
      "grad_norm": 0.6034091711044312,
      "kl": 0.23458020389080048,
      "learning_rate": 1.0148278737965845e-06,
      "loss": 0.0094,
      "reward": 1.8823692798614502,
      "reward_std": 1.0682947635650635,
      "rewards/reward_function": 1.8823692798614502,
      "step": 1280
    },
    {
      "completion_length": 313.3214416503906,
      "epoch": 0.732837528604119,
      "grad_norm": 0.19499224424362183,
      "kl": 0.149389386177063,
      "learning_rate": 1.0108144021728458e-06,
      "loss": 0.006,
      "reward": 1.6213657855987549,
      "reward_std": 0.8528889417648315,
      "rewards/reward_function": 1.6213657855987549,
      "step": 1281
    },
    {
      "completion_length": 346.0714416503906,
      "epoch": 0.7334096109839817,
      "grad_norm": 0.18856117129325867,
      "kl": 0.13200849294662476,
      "learning_rate": 1.0068068706111794e-06,
      "loss": 0.0053,
      "reward": 2.1366868019104004,
      "reward_std": 1.2406628131866455,
      "rewards/reward_function": 2.1366868019104004,
      "step": 1282
    },
    {
      "completion_length": 350.2500305175781,
      "epoch": 0.7339816933638444,
      "grad_norm": 2.734204053878784,
      "kl": 0.16468630731105804,
      "learning_rate": 1.0028052950968212e-06,
      "loss": 0.0066,
      "reward": 2.4672584533691406,
      "reward_std": 0.6308850646018982,
      "rewards/reward_function": 2.4672584533691406,
      "step": 1283
    },
    {
      "completion_length": 359.46429443359375,
      "epoch": 0.7345537757437071,
      "grad_norm": 0.19783446192741394,
      "kl": 0.16571040451526642,
      "learning_rate": 9.98809691591254e-07,
      "loss": 0.0066,
      "reward": 1.966017246246338,
      "reward_std": 0.9982368350028992,
      "rewards/reward_function": 1.966017246246338,
      "step": 1284
    },
    {
      "completion_length": 355.2857360839844,
      "epoch": 0.7351258581235698,
      "grad_norm": 0.19637972116470337,
      "kl": 0.15660089254379272,
      "learning_rate": 9.94820076032135e-07,
      "loss": 0.0063,
      "reward": 1.565208077430725,
      "reward_std": 0.8517073392868042,
      "rewards/reward_function": 1.565208077430725,
      "step": 1285
    },
    {
      "completion_length": 334.71429443359375,
      "epoch": 0.7356979405034325,
      "grad_norm": 1.2774688005447388,
      "kl": 0.19253064692020416,
      "learning_rate": 9.9083646433324e-07,
      "loss": 0.0077,
      "reward": 1.7739754915237427,
      "reward_std": 0.7756158709526062,
      "rewards/reward_function": 1.7739754915237427,
      "step": 1286
    },
    {
      "completion_length": 324.0357360839844,
      "epoch": 0.7362700228832952,
      "grad_norm": 0.26677846908569336,
      "kl": 0.16816596686840057,
      "learning_rate": 9.868588723843955e-07,
      "loss": 0.0067,
      "reward": 1.980919599533081,
      "reward_std": 0.6163951754570007,
      "rewards/reward_function": 1.980919599533081,
      "step": 1287
    },
    {
      "completion_length": 345.0714416503906,
      "epoch": 0.7368421052631579,
      "grad_norm": 2.074888229370117,
      "kl": 0.4809549152851105,
      "learning_rate": 9.828873160514163e-07,
      "loss": 0.0192,
      "reward": 2.409151554107666,
      "reward_std": 1.339752435684204,
      "rewards/reward_function": 2.409151554107666,
      "step": 1288
    },
    {
      "completion_length": 342.6785888671875,
      "epoch": 0.7374141876430206,
      "grad_norm": 0.21067743003368378,
      "kl": 0.14388996362686157,
      "learning_rate": 9.789218111760432e-07,
      "loss": 0.0058,
      "reward": 1.898267388343811,
      "reward_std": 1.0829455852508545,
      "rewards/reward_function": 1.898267388343811,
      "step": 1289
    },
    {
      "completion_length": 337.96429443359375,
      "epoch": 0.7379862700228833,
      "grad_norm": 3.8852691650390625,
      "kl": 0.4773253798484802,
      "learning_rate": 9.749623735758767e-07,
      "loss": 0.0191,
      "reward": 2.8175761699676514,
      "reward_std": 0.7735650539398193,
      "rewards/reward_function": 2.8175761699676514,
      "step": 1290
    },
    {
      "completion_length": 378.7857360839844,
      "epoch": 0.738558352402746,
      "grad_norm": 13.655024528503418,
      "kl": 1.1035475730895996,
      "learning_rate": 9.710090190443188e-07,
      "loss": 0.0441,
      "reward": 2.1953556537628174,
      "reward_std": 1.009286642074585,
      "rewards/reward_function": 2.1953556537628174,
      "step": 1291
    },
    {
      "completion_length": 346.7500305175781,
      "epoch": 0.7391304347826086,
      "grad_norm": 0.23712103068828583,
      "kl": 0.14600203931331635,
      "learning_rate": 9.670617633505055e-07,
      "loss": 0.0058,
      "reward": 1.6246854066848755,
      "reward_std": 0.7649444937705994,
      "rewards/reward_function": 1.6246854066848755,
      "step": 1292
    },
    {
      "completion_length": 349.3214416503906,
      "epoch": 0.7397025171624714,
      "grad_norm": 0.2365378737449646,
      "kl": 0.16985008120536804,
      "learning_rate": 9.63120622239248e-07,
      "loss": 0.0068,
      "reward": 1.6741001605987549,
      "reward_std": 0.8524614572525024,
      "rewards/reward_function": 1.6741001605987549,
      "step": 1293
    },
    {
      "completion_length": 348.96429443359375,
      "epoch": 0.7402745995423341,
      "grad_norm": 0.22564707696437836,
      "kl": 0.12051711976528168,
      "learning_rate": 9.59185611430964e-07,
      "loss": 0.0048,
      "reward": 2.454441785812378,
      "reward_std": 0.7498735785484314,
      "rewards/reward_function": 2.454441785812378,
      "step": 1294
    },
    {
      "completion_length": 374.1785888671875,
      "epoch": 0.7408466819221968,
      "grad_norm": 1.3906505107879639,
      "kl": 0.33241361379623413,
      "learning_rate": 9.55256746621623e-07,
      "loss": 0.0133,
      "reward": 1.0246037244796753,
      "reward_std": 0.561504602432251,
      "rewards/reward_function": 1.0246037244796753,
      "step": 1295
    },
    {
      "completion_length": 370.0714416503906,
      "epoch": 0.7414187643020596,
      "grad_norm": 0.6311622262001038,
      "kl": 0.17082318663597107,
      "learning_rate": 9.513340434826751e-07,
      "loss": 0.0068,
      "reward": 2.426483631134033,
      "reward_std": 0.9898043274879456,
      "rewards/reward_function": 2.426483631134033,
      "step": 1296
    },
    {
      "completion_length": 380.64288330078125,
      "epoch": 0.7419908466819222,
      "grad_norm": 0.8632773160934448,
      "kl": 0.16478972136974335,
      "learning_rate": 9.474175176609956e-07,
      "loss": 0.0066,
      "reward": 1.8598122596740723,
      "reward_std": 1.0432523488998413,
      "rewards/reward_function": 1.8598122596740723,
      "step": 1297
    },
    {
      "completion_length": 332.1071472167969,
      "epoch": 0.7425629290617849,
      "grad_norm": 0.36226579546928406,
      "kl": 0.18881551921367645,
      "learning_rate": 9.435071847788191e-07,
      "loss": 0.0076,
      "reward": 2.430506706237793,
      "reward_std": 1.0749120712280273,
      "rewards/reward_function": 2.430506706237793,
      "step": 1298
    },
    {
      "completion_length": 364.6071472167969,
      "epoch": 0.7431350114416476,
      "grad_norm": 1.1882848739624023,
      "kl": 0.3235842287540436,
      "learning_rate": 9.396030604336772e-07,
      "loss": 0.0129,
      "reward": 1.5816575288772583,
      "reward_std": 0.6789014339447021,
      "rewards/reward_function": 1.5816575288772583,
      "step": 1299
    },
    {
      "completion_length": 323.89288330078125,
      "epoch": 0.7437070938215103,
      "grad_norm": 0.21749719977378845,
      "kl": 0.17001326382160187,
      "learning_rate": 9.357051601983389e-07,
      "loss": 0.0068,
      "reward": 2.4591705799102783,
      "reward_std": 0.7535757422447205,
      "rewards/reward_function": 2.4591705799102783,
      "step": 1300
    },
    {
      "completion_length": 365.6785888671875,
      "epoch": 0.744279176201373,
      "grad_norm": 0.18257352709770203,
      "kl": 0.15462081134319305,
      "learning_rate": 9.318134996207423e-07,
      "loss": 0.0062,
      "reward": 2.3508830070495605,
      "reward_std": 0.9079025983810425,
      "rewards/reward_function": 2.3508830070495605,
      "step": 1301
    },
    {
      "completion_length": 341.7500305175781,
      "epoch": 0.7448512585812357,
      "grad_norm": 0.1881745606660843,
      "kl": 0.18154066801071167,
      "learning_rate": 9.279280942239418e-07,
      "loss": 0.0073,
      "reward": 1.9397679567337036,
      "reward_std": 1.1028603315353394,
      "rewards/reward_function": 1.9397679567337036,
      "step": 1302
    },
    {
      "completion_length": 318.5,
      "epoch": 0.7454233409610984,
      "grad_norm": 0.44698819518089294,
      "kl": 0.15142479538917542,
      "learning_rate": 9.240489595060368e-07,
      "loss": 0.0061,
      "reward": 2.180778741836548,
      "reward_std": 1.0423610210418701,
      "rewards/reward_function": 2.180778741836548,
      "step": 1303
    },
    {
      "completion_length": 330.7857360839844,
      "epoch": 0.7459954233409611,
      "grad_norm": 0.19734753668308258,
      "kl": 0.11282029002904892,
      "learning_rate": 9.201761109401164e-07,
      "loss": 0.0045,
      "reward": 1.7379306554794312,
      "reward_std": 0.6903936266899109,
      "rewards/reward_function": 1.7379306554794312,
      "step": 1304
    },
    {
      "completion_length": 294.0,
      "epoch": 0.7465675057208238,
      "grad_norm": 2.9873642921447754,
      "kl": 0.24618317186832428,
      "learning_rate": 9.163095639741959e-07,
      "loss": 0.0098,
      "reward": 2.382901906967163,
      "reward_std": 0.6495540142059326,
      "rewards/reward_function": 2.382901906967163,
      "step": 1305
    },
    {
      "completion_length": 383.3571472167969,
      "epoch": 0.7471395881006865,
      "grad_norm": 3.441599130630493,
      "kl": 0.41821563243865967,
      "learning_rate": 9.124493340311538e-07,
      "loss": 0.0167,
      "reward": 2.375751256942749,
      "reward_std": 0.7404726147651672,
      "rewards/reward_function": 2.375751256942749,
      "step": 1306
    },
    {
      "completion_length": 328.9285888671875,
      "epoch": 0.7477116704805492,
      "grad_norm": 0.211582750082016,
      "kl": 0.14330987632274628,
      "learning_rate": 9.085954365086728e-07,
      "loss": 0.0057,
      "reward": 1.8742525577545166,
      "reward_std": 0.7134398221969604,
      "rewards/reward_function": 1.8742525577545166,
      "step": 1307
    },
    {
      "completion_length": 341.5714416503906,
      "epoch": 0.7482837528604119,
      "grad_norm": 0.18190115690231323,
      "kl": 0.13693711161613464,
      "learning_rate": 9.047478867791732e-07,
      "loss": 0.0055,
      "reward": 2.0054051876068115,
      "reward_std": 0.674003005027771,
      "rewards/reward_function": 2.0054051876068115,
      "step": 1308
    },
    {
      "completion_length": 319.96429443359375,
      "epoch": 0.7488558352402745,
      "grad_norm": 0.21103109419345856,
      "kl": 0.1476384699344635,
      "learning_rate": 9.009067001897598e-07,
      "loss": 0.0059,
      "reward": 2.4830336570739746,
      "reward_std": 1.0122472047805786,
      "rewards/reward_function": 2.4830336570739746,
      "step": 1309
    },
    {
      "completion_length": 334.14288330078125,
      "epoch": 0.7494279176201373,
      "grad_norm": 1.216840147972107,
      "kl": 0.23205415904521942,
      "learning_rate": 8.970718920621513e-07,
      "loss": 0.0093,
      "reward": 2.178975820541382,
      "reward_std": 1.0777761936187744,
      "rewards/reward_function": 2.178975820541382,
      "step": 1310
    },
    {
      "completion_length": 381.9285888671875,
      "epoch": 0.75,
      "grad_norm": 0.22527667880058289,
      "kl": 0.1457098126411438,
      "learning_rate": 8.932434776926302e-07,
      "loss": 0.0058,
      "reward": 1.982572317123413,
      "reward_std": 1.0493512153625488,
      "rewards/reward_function": 1.982572317123413,
      "step": 1311
    },
    {
      "completion_length": 327.0714416503906,
      "epoch": 0.7505720823798627,
      "grad_norm": 2.023352861404419,
      "kl": 0.35076016187667847,
      "learning_rate": 8.894214723519693e-07,
      "loss": 0.014,
      "reward": 1.363867998123169,
      "reward_std": 0.668507993221283,
      "rewards/reward_function": 1.363867998123169,
      "step": 1312
    },
    {
      "completion_length": 350.6785888671875,
      "epoch": 0.7511441647597255,
      "grad_norm": 4.174585342407227,
      "kl": 0.7632587552070618,
      "learning_rate": 8.856058912853813e-07,
      "loss": 0.0305,
      "reward": 2.3804664611816406,
      "reward_std": 0.899351954460144,
      "rewards/reward_function": 2.3804664611816406,
      "step": 1313
    },
    {
      "completion_length": 371.14288330078125,
      "epoch": 0.7517162471395881,
      "grad_norm": 33.5223388671875,
      "kl": 0.6368253231048584,
      "learning_rate": 8.817967497124511e-07,
      "loss": 0.0255,
      "reward": 2.466754198074341,
      "reward_std": 0.7868172526359558,
      "rewards/reward_function": 2.466754198074341,
      "step": 1314
    },
    {
      "completion_length": 312.0,
      "epoch": 0.7522883295194508,
      "grad_norm": 0.17763957381248474,
      "kl": 0.13058555126190186,
      "learning_rate": 8.779940628270791e-07,
      "loss": 0.0052,
      "reward": 1.6083842515945435,
      "reward_std": 0.826134979724884,
      "rewards/reward_function": 1.6083842515945435,
      "step": 1315
    },
    {
      "completion_length": 327.8571472167969,
      "epoch": 0.7528604118993135,
      "grad_norm": 8.176822662353516,
      "kl": 0.41709694266319275,
      "learning_rate": 8.741978457974198e-07,
      "loss": 0.0167,
      "reward": 2.0969552993774414,
      "reward_std": 1.2309859991073608,
      "rewards/reward_function": 2.0969552993774414,
      "step": 1316
    },
    {
      "completion_length": 396.7500305175781,
      "epoch": 0.7534324942791762,
      "grad_norm": 3.270625352859497,
      "kl": 0.17694932222366333,
      "learning_rate": 8.704081137658196e-07,
      "loss": 0.0071,
      "reward": 2.504117250442505,
      "reward_std": 1.3439160585403442,
      "rewards/reward_function": 2.504117250442505,
      "step": 1317
    },
    {
      "completion_length": 360.89288330078125,
      "epoch": 0.7540045766590389,
      "grad_norm": 0.19804629683494568,
      "kl": 0.17738507688045502,
      "learning_rate": 8.666248818487588e-07,
      "loss": 0.0071,
      "reward": 2.062657594680786,
      "reward_std": 0.9243210554122925,
      "rewards/reward_function": 2.062657594680786,
      "step": 1318
    },
    {
      "completion_length": 337.75,
      "epoch": 0.7545766590389016,
      "grad_norm": 0.20654034614562988,
      "kl": 0.1960248500108719,
      "learning_rate": 8.628481651367876e-07,
      "loss": 0.0078,
      "reward": 1.8910043239593506,
      "reward_std": 0.8621404767036438,
      "rewards/reward_function": 1.8910043239593506,
      "step": 1319
    },
    {
      "completion_length": 345.6785888671875,
      "epoch": 0.7551487414187643,
      "grad_norm": 5.2600016593933105,
      "kl": 0.4814510643482208,
      "learning_rate": 8.590779786944714e-07,
      "loss": 0.0193,
      "reward": 2.327631711959839,
      "reward_std": 1.2424302101135254,
      "rewards/reward_function": 2.327631711959839,
      "step": 1320
    },
    {
      "completion_length": 323.39288330078125,
      "epoch": 0.755720823798627,
      "grad_norm": 0.17905817925930023,
      "kl": 0.12085702270269394,
      "learning_rate": 8.553143375603251e-07,
      "loss": 0.0048,
      "reward": 1.9672226905822754,
      "reward_std": 0.8729865550994873,
      "rewards/reward_function": 1.9672226905822754,
      "step": 1321
    },
    {
      "completion_length": 341.0714416503906,
      "epoch": 0.7562929061784897,
      "grad_norm": 0.1725624054670334,
      "kl": 0.1467761993408203,
      "learning_rate": 8.515572567467573e-07,
      "loss": 0.0059,
      "reward": 1.9698807001113892,
      "reward_std": 0.7620569467544556,
      "rewards/reward_function": 1.9698807001113892,
      "step": 1322
    },
    {
      "completion_length": 359.21429443359375,
      "epoch": 0.7568649885583524,
      "grad_norm": 0.8175433874130249,
      "kl": 0.20031018555164337,
      "learning_rate": 8.478067512400081e-07,
      "loss": 0.008,
      "reward": 1.5110642910003662,
      "reward_std": 1.038511872291565,
      "rewards/reward_function": 1.5110642910003662,
      "step": 1323
    },
    {
      "completion_length": 313.9285888671875,
      "epoch": 0.7574370709382151,
      "grad_norm": 1.9290615320205688,
      "kl": 0.349882572889328,
      "learning_rate": 8.440628360000907e-07,
      "loss": 0.014,
      "reward": 2.1461517810821533,
      "reward_std": 0.9335917234420776,
      "rewards/reward_function": 2.1461517810821533,
      "step": 1324
    },
    {
      "completion_length": 376.0714416503906,
      "epoch": 0.7580091533180778,
      "grad_norm": 1.385538101196289,
      "kl": 0.4470590651035309,
      "learning_rate": 8.403255259607312e-07,
      "loss": 0.0179,
      "reward": 2.1967685222625732,
      "reward_std": 0.8486936688423157,
      "rewards/reward_function": 2.1967685222625732,
      "step": 1325
    },
    {
      "completion_length": 338.6071472167969,
      "epoch": 0.7585812356979404,
      "grad_norm": 0.4682115912437439,
      "kl": 0.16484007239341736,
      "learning_rate": 8.365948360293069e-07,
      "loss": 0.0066,
      "reward": 2.111299514770508,
      "reward_std": 0.9532479047775269,
      "rewards/reward_function": 2.111299514770508,
      "step": 1326
    },
    {
      "completion_length": 335.3571472167969,
      "epoch": 0.7591533180778032,
      "grad_norm": 1521.49169921875,
      "kl": 2.3386895656585693,
      "learning_rate": 8.328707810867911e-07,
      "loss": 0.0935,
      "reward": 1.6176848411560059,
      "reward_std": 1.0614889860153198,
      "rewards/reward_function": 1.6176848411560059,
      "step": 1327
    },
    {
      "completion_length": 331.25,
      "epoch": 0.7597254004576659,
      "grad_norm": 0.1778520941734314,
      "kl": 0.16225551068782806,
      "learning_rate": 8.291533759876888e-07,
      "loss": 0.0065,
      "reward": 2.268794536590576,
      "reward_std": 0.70760178565979,
      "rewards/reward_function": 2.268794536590576,
      "step": 1328
    },
    {
      "completion_length": 318.7857360839844,
      "epoch": 0.7602974828375286,
      "grad_norm": 0.19255512952804565,
      "kl": 0.14946705102920532,
      "learning_rate": 8.254426355599849e-07,
      "loss": 0.006,
      "reward": 2.5665206909179688,
      "reward_std": 0.639181911945343,
      "rewards/reward_function": 2.5665206909179688,
      "step": 1329
    },
    {
      "completion_length": 347.5000305175781,
      "epoch": 0.7608695652173914,
      "grad_norm": 0.23484541475772858,
      "kl": 0.15259991586208344,
      "learning_rate": 8.217385746050743e-07,
      "loss": 0.0061,
      "reward": 1.7940170764923096,
      "reward_std": 0.9508223533630371,
      "rewards/reward_function": 1.7940170764923096,
      "step": 1330
    },
    {
      "completion_length": 344.8214416503906,
      "epoch": 0.761441647597254,
      "grad_norm": 0.19816499948501587,
      "kl": 0.17508703470230103,
      "learning_rate": 8.180412078977135e-07,
      "loss": 0.007,
      "reward": 2.0389411449432373,
      "reward_std": 0.6215721964836121,
      "rewards/reward_function": 2.0389411449432373,
      "step": 1331
    },
    {
      "completion_length": 329.6785888671875,
      "epoch": 0.7620137299771167,
      "grad_norm": 0.5620640516281128,
      "kl": 0.15333867073059082,
      "learning_rate": 8.143505501859553e-07,
      "loss": 0.0061,
      "reward": 2.033475160598755,
      "reward_std": 0.8205097317695618,
      "rewards/reward_function": 2.033475160598755,
      "step": 1332
    },
    {
      "completion_length": 344.0357360839844,
      "epoch": 0.7625858123569794,
      "grad_norm": 0.18667615950107574,
      "kl": 0.13192704319953918,
      "learning_rate": 8.106666161910904e-07,
      "loss": 0.0053,
      "reward": 2.420025587081909,
      "reward_std": 0.9397921562194824,
      "rewards/reward_function": 2.420025587081909,
      "step": 1333
    },
    {
      "completion_length": 329.75,
      "epoch": 0.7631578947368421,
      "grad_norm": 0.19431103765964508,
      "kl": 0.14387023448944092,
      "learning_rate": 8.069894206075918e-07,
      "loss": 0.0058,
      "reward": 1.9736902713775635,
      "reward_std": 1.0937743186950684,
      "rewards/reward_function": 1.9736902713775635,
      "step": 1334
    },
    {
      "completion_length": 306.1785888671875,
      "epoch": 0.7637299771167048,
      "grad_norm": 0.19375871121883392,
      "kl": 0.1592736691236496,
      "learning_rate": 8.033189781030538e-07,
      "loss": 0.0064,
      "reward": 2.105987548828125,
      "reward_std": 1.0836613178253174,
      "rewards/reward_function": 2.105987548828125,
      "step": 1335
    },
    {
      "completion_length": 318.5714416503906,
      "epoch": 0.7643020594965675,
      "grad_norm": 0.17485934495925903,
      "kl": 0.14287753403186798,
      "learning_rate": 7.996553033181345e-07,
      "loss": 0.0057,
      "reward": 1.723618745803833,
      "reward_std": 0.951351523399353,
      "rewards/reward_function": 1.723618745803833,
      "step": 1336
    },
    {
      "completion_length": 376.0714416503906,
      "epoch": 0.7648741418764302,
      "grad_norm": 0.6658969521522522,
      "kl": 0.49094071984291077,
      "learning_rate": 7.959984108664947e-07,
      "loss": 0.0196,
      "reward": 2.0755364894866943,
      "reward_std": 0.9038960933685303,
      "rewards/reward_function": 2.0755364894866943,
      "step": 1337
    },
    {
      "completion_length": 357.46429443359375,
      "epoch": 0.7654462242562929,
      "grad_norm": 0.2629631459712982,
      "kl": 0.1701345592737198,
      "learning_rate": 7.923483153347456e-07,
      "loss": 0.0068,
      "reward": 2.459460973739624,
      "reward_std": 0.8892855644226074,
      "rewards/reward_function": 2.459460973739624,
      "step": 1338
    },
    {
      "completion_length": 341.5000305175781,
      "epoch": 0.7660183066361556,
      "grad_norm": 0.2045840471982956,
      "kl": 0.15990100800991058,
      "learning_rate": 7.887050312823827e-07,
      "loss": 0.0064,
      "reward": 1.5623319149017334,
      "reward_std": 0.6802369952201843,
      "rewards/reward_function": 1.5623319149017334,
      "step": 1339
    },
    {
      "completion_length": 349.39288330078125,
      "epoch": 0.7665903890160183,
      "grad_norm": 2.4492359161376953,
      "kl": 0.3397386372089386,
      "learning_rate": 7.850685732417346e-07,
      "loss": 0.0136,
      "reward": 2.1371874809265137,
      "reward_std": 1.0114266872406006,
      "rewards/reward_function": 2.1371874809265137,
      "step": 1340
    },
    {
      "completion_length": 314.25,
      "epoch": 0.767162471395881,
      "grad_norm": 0.8851582407951355,
      "kl": 0.18886353075504303,
      "learning_rate": 7.814389557179017e-07,
      "loss": 0.0076,
      "reward": 1.980533242225647,
      "reward_std": 0.9395735859870911,
      "rewards/reward_function": 1.980533242225647,
      "step": 1341
    },
    {
      "completion_length": 361.0357360839844,
      "epoch": 0.7677345537757437,
      "grad_norm": 0.36571750044822693,
      "kl": 0.2085670530796051,
      "learning_rate": 7.77816193188699e-07,
      "loss": 0.0083,
      "reward": 2.4247050285339355,
      "reward_std": 0.9876543283462524,
      "rewards/reward_function": 2.4247050285339355,
      "step": 1342
    },
    {
      "completion_length": 346.8571472167969,
      "epoch": 0.7683066361556065,
      "grad_norm": 0.18354453146457672,
      "kl": 0.13954606652259827,
      "learning_rate": 7.742003001045989e-07,
      "loss": 0.0056,
      "reward": 1.9081604480743408,
      "reward_std": 0.6742936372756958,
      "rewards/reward_function": 1.9081604480743408,
      "step": 1343
    },
    {
      "completion_length": 329.89288330078125,
      "epoch": 0.7688787185354691,
      "grad_norm": 0.20360955595970154,
      "kl": 0.1270325779914856,
      "learning_rate": 7.705912908886706e-07,
      "loss": 0.0051,
      "reward": 2.959796667098999,
      "reward_std": 0.6652466654777527,
      "rewards/reward_function": 2.959796667098999,
      "step": 1344
    },
    {
      "completion_length": 339.8571472167969,
      "epoch": 0.7694508009153318,
      "grad_norm": 0.3074413537979126,
      "kl": 0.17844872176647186,
      "learning_rate": 7.669891799365284e-07,
      "loss": 0.0071,
      "reward": 2.513561248779297,
      "reward_std": 1.1455559730529785,
      "rewards/reward_function": 2.513561248779297,
      "step": 1345
    },
    {
      "completion_length": 336.1071472167969,
      "epoch": 0.7700228832951945,
      "grad_norm": 0.18538568913936615,
      "kl": 0.16101714968681335,
      "learning_rate": 7.633939816162661e-07,
      "loss": 0.0064,
      "reward": 1.6005574464797974,
      "reward_std": 1.049930453300476,
      "rewards/reward_function": 1.6005574464797974,
      "step": 1346
    },
    {
      "completion_length": 324.4285888671875,
      "epoch": 0.7705949656750573,
      "grad_norm": 1.130027174949646,
      "kl": 0.17003609240055084,
      "learning_rate": 7.598057102684108e-07,
      "loss": 0.0068,
      "reward": 1.6601955890655518,
      "reward_std": 0.8309944868087769,
      "rewards/reward_function": 1.6601955890655518,
      "step": 1347
    },
    {
      "completion_length": 367.64288330078125,
      "epoch": 0.7711670480549199,
      "grad_norm": 0.2619727551937103,
      "kl": 0.1535601019859314,
      "learning_rate": 7.562243802058536e-07,
      "loss": 0.0061,
      "reward": 2.125023603439331,
      "reward_std": 1.1920344829559326,
      "rewards/reward_function": 2.125023603439331,
      "step": 1348
    },
    {
      "completion_length": 358.96429443359375,
      "epoch": 0.7717391304347826,
      "grad_norm": 0.17428965866565704,
      "kl": 0.1270972490310669,
      "learning_rate": 7.526500057138006e-07,
      "loss": 0.0051,
      "reward": 2.1765575408935547,
      "reward_std": 1.0244145393371582,
      "rewards/reward_function": 2.1765575408935547,
      "step": 1349
    },
    {
      "completion_length": 364.96429443359375,
      "epoch": 0.7723112128146453,
      "grad_norm": 1.1115334033966064,
      "kl": 0.460757315158844,
      "learning_rate": 7.490826010497146e-07,
      "loss": 0.0184,
      "reward": 2.4838242530822754,
      "reward_std": 1.1184622049331665,
      "rewards/reward_function": 2.4838242530822754,
      "step": 1350
    },
    {
      "completion_length": 302.2857360839844,
      "epoch": 0.772883295194508,
      "grad_norm": 0.3907502293586731,
      "kl": 0.22700199484825134,
      "learning_rate": 7.455221804432541e-07,
      "loss": 0.0091,
      "reward": 1.8683465719223022,
      "reward_std": 0.9948897361755371,
      "rewards/reward_function": 1.8683465719223022,
      "step": 1351
    },
    {
      "completion_length": 356.39288330078125,
      "epoch": 0.7734553775743707,
      "grad_norm": 0.16943694651126862,
      "kl": 0.11487806588411331,
      "learning_rate": 7.419687580962223e-07,
      "loss": 0.0046,
      "reward": 1.2802448272705078,
      "reward_std": 0.761513352394104,
      "rewards/reward_function": 1.2802448272705078,
      "step": 1352
    },
    {
      "completion_length": 396.0714416503906,
      "epoch": 0.7740274599542334,
      "grad_norm": 0.18037272989749908,
      "kl": 0.15069451928138733,
      "learning_rate": 7.384223481825068e-07,
      "loss": 0.006,
      "reward": 1.672765851020813,
      "reward_std": 1.4136220216751099,
      "rewards/reward_function": 1.672765851020813,
      "step": 1353
    },
    {
      "completion_length": 314.3571472167969,
      "epoch": 0.7745995423340961,
      "grad_norm": 0.8142531514167786,
      "kl": 0.21345402300357819,
      "learning_rate": 7.348829648480244e-07,
      "loss": 0.0085,
      "reward": 2.2561135292053223,
      "reward_std": 1.1758073568344116,
      "rewards/reward_function": 2.2561135292053223,
      "step": 1354
    },
    {
      "completion_length": 347.0357360839844,
      "epoch": 0.7751716247139588,
      "grad_norm": 0.28203558921813965,
      "kl": 0.25326642394065857,
      "learning_rate": 7.313506222106628e-07,
      "loss": 0.0101,
      "reward": 2.1537961959838867,
      "reward_std": 0.6210678219795227,
      "rewards/reward_function": 2.1537961959838867,
      "step": 1355
    },
    {
      "completion_length": 351.5714416503906,
      "epoch": 0.7757437070938215,
      "grad_norm": 1.4034991264343262,
      "kl": 0.16138094663619995,
      "learning_rate": 7.278253343602284e-07,
      "loss": 0.0065,
      "reward": 2.3170039653778076,
      "reward_std": 1.2863610982894897,
      "rewards/reward_function": 2.3170039653778076,
      "step": 1356
    },
    {
      "completion_length": 312.25,
      "epoch": 0.7763157894736842,
      "grad_norm": 0.2564742863178253,
      "kl": 0.1612936109304428,
      "learning_rate": 7.243071153583842e-07,
      "loss": 0.0065,
      "reward": 1.456795334815979,
      "reward_std": 0.9188453555107117,
      "rewards/reward_function": 1.456795334815979,
      "step": 1357
    },
    {
      "completion_length": 345.0714416503906,
      "epoch": 0.7768878718535469,
      "grad_norm": 3.697584629058838,
      "kl": 0.19195395708084106,
      "learning_rate": 7.207959792385999e-07,
      "loss": 0.0077,
      "reward": 1.8841935396194458,
      "reward_std": 0.9977033138275146,
      "rewards/reward_function": 1.8841935396194458,
      "step": 1358
    },
    {
      "completion_length": 420.0000305175781,
      "epoch": 0.7774599542334096,
      "grad_norm": 0.6469752788543701,
      "kl": 0.17413096129894257,
      "learning_rate": 7.172919400060912e-07,
      "loss": 0.007,
      "reward": 2.0535106658935547,
      "reward_std": 1.2417867183685303,
      "rewards/reward_function": 2.0535106658935547,
      "step": 1359
    },
    {
      "completion_length": 292.2857360839844,
      "epoch": 0.7780320366132724,
      "grad_norm": 0.35212039947509766,
      "kl": 0.17906533181667328,
      "learning_rate": 7.137950116377673e-07,
      "loss": 0.0072,
      "reward": 2.594404935836792,
      "reward_std": 1.0675710439682007,
      "rewards/reward_function": 2.594404935836792,
      "step": 1360
    },
    {
      "completion_length": 401.5714416503906,
      "epoch": 0.778604118993135,
      "grad_norm": 15.315176010131836,
      "kl": 1.354217290878296,
      "learning_rate": 7.103052080821726e-07,
      "loss": 0.0542,
      "reward": 2.231271743774414,
      "reward_std": 1.0684171915054321,
      "rewards/reward_function": 2.231271743774414,
      "step": 1361
    },
    {
      "completion_length": 342.21429443359375,
      "epoch": 0.7791762013729977,
      "grad_norm": 0.1801370233297348,
      "kl": 0.14624696969985962,
      "learning_rate": 7.068225432594311e-07,
      "loss": 0.0058,
      "reward": 2.083902359008789,
      "reward_std": 1.1590571403503418,
      "rewards/reward_function": 2.083902359008789,
      "step": 1362
    },
    {
      "completion_length": 318.5357360839844,
      "epoch": 0.7797482837528604,
      "grad_norm": 0.1699080467224121,
      "kl": 0.14237594604492188,
      "learning_rate": 7.033470310611945e-07,
      "loss": 0.0057,
      "reward": 1.4625794887542725,
      "reward_std": 0.6768223643302917,
      "rewards/reward_function": 1.4625794887542725,
      "step": 1363
    },
    {
      "completion_length": 363.8214416503906,
      "epoch": 0.7803203661327232,
      "grad_norm": 0.16880831122398376,
      "kl": 0.11273100227117538,
      "learning_rate": 6.998786853505799e-07,
      "loss": 0.0045,
      "reward": 1.1620343923568726,
      "reward_std": 0.7836564183235168,
      "rewards/reward_function": 1.1620343923568726,
      "step": 1364
    },
    {
      "completion_length": 352.46429443359375,
      "epoch": 0.7808924485125858,
      "grad_norm": 0.9311140179634094,
      "kl": 0.14812666177749634,
      "learning_rate": 6.964175199621242e-07,
      "loss": 0.0059,
      "reward": 1.8337876796722412,
      "reward_std": 1.2023462057113647,
      "rewards/reward_function": 1.8337876796722412,
      "step": 1365
    },
    {
      "completion_length": 347.7500305175781,
      "epoch": 0.7814645308924485,
      "grad_norm": 1.8918503522872925,
      "kl": 0.26873189210891724,
      "learning_rate": 6.929635487017189e-07,
      "loss": 0.0107,
      "reward": 2.6816229820251465,
      "reward_std": 0.9555817246437073,
      "rewards/reward_function": 2.6816229820251465,
      "step": 1366
    },
    {
      "completion_length": 398.9285888671875,
      "epoch": 0.7820366132723112,
      "grad_norm": 0.20412196218967438,
      "kl": 0.14843319356441498,
      "learning_rate": 6.895167853465614e-07,
      "loss": 0.0059,
      "reward": 1.8548285961151123,
      "reward_std": 1.0725560188293457,
      "rewards/reward_function": 1.8548285961151123,
      "step": 1367
    },
    {
      "completion_length": 342.1785888671875,
      "epoch": 0.782608695652174,
      "grad_norm": 3.7478671073913574,
      "kl": 0.577562689781189,
      "learning_rate": 6.86077243645099e-07,
      "loss": 0.0231,
      "reward": 1.6033226251602173,
      "reward_std": 0.7599164843559265,
      "rewards/reward_function": 1.6033226251602173,
      "step": 1368
    },
    {
      "completion_length": 361.46429443359375,
      "epoch": 0.7831807780320366,
      "grad_norm": 1.066426396369934,
      "kl": 0.3586396872997284,
      "learning_rate": 6.826449373169708e-07,
      "loss": 0.0143,
      "reward": 2.1537463665008545,
      "reward_std": 0.9988933801651001,
      "rewards/reward_function": 2.1537463665008545,
      "step": 1369
    },
    {
      "completion_length": 330.0357360839844,
      "epoch": 0.7837528604118993,
      "grad_norm": 0.1797541379928589,
      "kl": 0.1372421383857727,
      "learning_rate": 6.792198800529579e-07,
      "loss": 0.0055,
      "reward": 1.9191614389419556,
      "reward_std": 0.8965036869049072,
      "rewards/reward_function": 1.9191614389419556,
      "step": 1370
    },
    {
      "completion_length": 332.5,
      "epoch": 0.784324942791762,
      "grad_norm": 1.1980938911437988,
      "kl": 0.2542772591114044,
      "learning_rate": 6.75802085514925e-07,
      "loss": 0.0102,
      "reward": 1.7567968368530273,
      "reward_std": 0.9389827847480774,
      "rewards/reward_function": 1.7567968368530273,
      "step": 1371
    },
    {
      "completion_length": 339.4285888671875,
      "epoch": 0.7848970251716247,
      "grad_norm": 0.24388332664966583,
      "kl": 0.17237962782382965,
      "learning_rate": 6.723915673357686e-07,
      "loss": 0.0069,
      "reward": 1.9620788097381592,
      "reward_std": 0.8312029242515564,
      "rewards/reward_function": 1.9620788097381592,
      "step": 1372
    },
    {
      "completion_length": 365.8214416503906,
      "epoch": 0.7854691075514875,
      "grad_norm": 0.18092243373394012,
      "kl": 0.1511363387107849,
      "learning_rate": 6.689883391193588e-07,
      "loss": 0.006,
      "reward": 1.56046462059021,
      "reward_std": 1.015392780303955,
      "rewards/reward_function": 1.56046462059021,
      "step": 1373
    },
    {
      "completion_length": 348.21429443359375,
      "epoch": 0.7860411899313501,
      "grad_norm": 0.17201295495033264,
      "kl": 0.12774574756622314,
      "learning_rate": 6.655924144404907e-07,
      "loss": 0.0051,
      "reward": 2.1250252723693848,
      "reward_std": 0.676900327205658,
      "rewards/reward_function": 2.1250252723693848,
      "step": 1374
    },
    {
      "completion_length": 329.46429443359375,
      "epoch": 0.7866132723112128,
      "grad_norm": 0.20269162952899933,
      "kl": 0.13635967671871185,
      "learning_rate": 6.622038068448236e-07,
      "loss": 0.0055,
      "reward": 2.0240275859832764,
      "reward_std": 0.6816710829734802,
      "rewards/reward_function": 2.0240275859832764,
      "step": 1375
    },
    {
      "completion_length": 320.0714416503906,
      "epoch": 0.7871853546910755,
      "grad_norm": 0.17678996920585632,
      "kl": 0.1336514949798584,
      "learning_rate": 6.588225298488329e-07,
      "loss": 0.0053,
      "reward": 1.8857780694961548,
      "reward_std": 0.827499270439148,
      "rewards/reward_function": 1.8857780694961548,
      "step": 1376
    },
    {
      "completion_length": 359.4285888671875,
      "epoch": 0.7877574370709383,
      "grad_norm": 3.4124531745910645,
      "kl": 0.5270163416862488,
      "learning_rate": 6.554485969397534e-07,
      "loss": 0.0211,
      "reward": 2.7994544506073,
      "reward_std": 0.8944875001907349,
      "rewards/reward_function": 2.7994544506073,
      "step": 1377
    },
    {
      "completion_length": 340.6071472167969,
      "epoch": 0.7883295194508009,
      "grad_norm": 0.2214648723602295,
      "kl": 0.17586790025234222,
      "learning_rate": 6.520820215755253e-07,
      "loss": 0.007,
      "reward": 1.8125288486480713,
      "reward_std": 0.8114722371101379,
      "rewards/reward_function": 1.8125288486480713,
      "step": 1378
    },
    {
      "completion_length": 382.9285888671875,
      "epoch": 0.7889016018306636,
      "grad_norm": 0.15921996533870697,
      "kl": 0.13398510217666626,
      "learning_rate": 6.487228171847415e-07,
      "loss": 0.0054,
      "reward": 1.9511934518814087,
      "reward_std": 0.9602397084236145,
      "rewards/reward_function": 1.9511934518814087,
      "step": 1379
    },
    {
      "completion_length": 369.0714416503906,
      "epoch": 0.7894736842105263,
      "grad_norm": 0.1937718242406845,
      "kl": 0.14680111408233643,
      "learning_rate": 6.453709971665922e-07,
      "loss": 0.0059,
      "reward": 1.8048853874206543,
      "reward_std": 1.1747411489486694,
      "rewards/reward_function": 1.8048853874206543,
      "step": 1380
    },
    {
      "completion_length": 318.5357360839844,
      "epoch": 0.790045766590389,
      "grad_norm": 27.229217529296875,
      "kl": 1.1858009099960327,
      "learning_rate": 6.42026574890815e-07,
      "loss": 0.0474,
      "reward": 1.8516305685043335,
      "reward_std": 1.019425630569458,
      "rewards/reward_function": 1.8516305685043335,
      "step": 1381
    },
    {
      "completion_length": 312.0357360839844,
      "epoch": 0.7906178489702517,
      "grad_norm": 2641.3388671875,
      "kl": 66.2368392944336,
      "learning_rate": 6.38689563697637e-07,
      "loss": 2.6495,
      "reward": 2.5634946823120117,
      "reward_std": 0.6944196224212646,
      "rewards/reward_function": 2.5634946823120117,
      "step": 1382
    },
    {
      "completion_length": 355.0000305175781,
      "epoch": 0.7911899313501144,
      "grad_norm": 0.1761234998703003,
      "kl": 0.12454628944396973,
      "learning_rate": 6.353599768977251e-07,
      "loss": 0.005,
      "reward": 1.7202489376068115,
      "reward_std": 0.9333742260932922,
      "rewards/reward_function": 1.7202489376068115,
      "step": 1383
    },
    {
      "completion_length": 321.46429443359375,
      "epoch": 0.7917620137299771,
      "grad_norm": 0.20729151368141174,
      "kl": 0.15524716675281525,
      "learning_rate": 6.320378277721342e-07,
      "loss": 0.0062,
      "reward": 1.9299415349960327,
      "reward_std": 0.6717491745948792,
      "rewards/reward_function": 1.9299415349960327,
      "step": 1384
    },
    {
      "completion_length": 367.5357360839844,
      "epoch": 0.7923340961098398,
      "grad_norm": 3.4068715572357178,
      "kl": 0.6098923087120056,
      "learning_rate": 6.28723129572247e-07,
      "loss": 0.0244,
      "reward": 2.328976631164551,
      "reward_std": 1.2055268287658691,
      "rewards/reward_function": 2.328976631164551,
      "step": 1385
    },
    {
      "completion_length": 318.71429443359375,
      "epoch": 0.7929061784897025,
      "grad_norm": 0.20025071501731873,
      "kl": 0.1331786811351776,
      "learning_rate": 6.254158955197307e-07,
      "loss": 0.0053,
      "reward": 2.0508852005004883,
      "reward_std": 1.053934931755066,
      "rewards/reward_function": 2.0508852005004883,
      "step": 1386
    },
    {
      "completion_length": 339.5714416503906,
      "epoch": 0.7934782608695652,
      "grad_norm": 4.1035661697387695,
      "kl": 0.7802528142929077,
      "learning_rate": 6.221161388064747e-07,
      "loss": 0.0312,
      "reward": 1.8614964485168457,
      "reward_std": 0.8868489265441895,
      "rewards/reward_function": 1.8614964485168457,
      "step": 1387
    },
    {
      "completion_length": 354.6071472167969,
      "epoch": 0.7940503432494279,
      "grad_norm": 0.28262439370155334,
      "kl": 0.15886124968528748,
      "learning_rate": 6.188238725945472e-07,
      "loss": 0.0064,
      "reward": 2.0969302654266357,
      "reward_std": 0.6021507978439331,
      "rewards/reward_function": 2.0969302654266357,
      "step": 1388
    },
    {
      "completion_length": 355.14288330078125,
      "epoch": 0.7946224256292906,
      "grad_norm": 3.821739435195923,
      "kl": 0.21888570487499237,
      "learning_rate": 6.155391100161359e-07,
      "loss": 0.0088,
      "reward": 2.4437644481658936,
      "reward_std": 0.7811094522476196,
      "rewards/reward_function": 2.4437644481658936,
      "step": 1389
    },
    {
      "completion_length": 365.7857360839844,
      "epoch": 0.7951945080091534,
      "grad_norm": 1.7499520778656006,
      "kl": 0.20436136424541473,
      "learning_rate": 6.122618641734992e-07,
      "loss": 0.0082,
      "reward": 1.814652681350708,
      "reward_std": 0.838783860206604,
      "rewards/reward_function": 1.814652681350708,
      "step": 1390
    },
    {
      "completion_length": 350.96429443359375,
      "epoch": 0.795766590389016,
      "grad_norm": 0.331903874874115,
      "kl": 0.19169452786445618,
      "learning_rate": 6.089921481389113e-07,
      "loss": 0.0077,
      "reward": 1.736388921737671,
      "reward_std": 0.8413662314414978,
      "rewards/reward_function": 1.736388921737671,
      "step": 1391
    },
    {
      "completion_length": 328.89288330078125,
      "epoch": 0.7963386727688787,
      "grad_norm": 0.1923614889383316,
      "kl": 0.12923260033130646,
      "learning_rate": 6.057299749546125e-07,
      "loss": 0.0052,
      "reward": 1.9287254810333252,
      "reward_std": 1.1976070404052734,
      "rewards/reward_function": 1.9287254810333252,
      "step": 1392
    },
    {
      "completion_length": 332.8571472167969,
      "epoch": 0.7969107551487414,
      "grad_norm": 2.0331833362579346,
      "kl": 0.1865202933549881,
      "learning_rate": 6.024753576327569e-07,
      "loss": 0.0075,
      "reward": 2.170980930328369,
      "reward_std": 0.6674878597259521,
      "rewards/reward_function": 2.170980930328369,
      "step": 1393
    },
    {
      "completion_length": 293.0714416503906,
      "epoch": 0.7974828375286042,
      "grad_norm": 1.4381906986236572,
      "kl": 0.35542529821395874,
      "learning_rate": 5.992283091553575e-07,
      "loss": 0.0142,
      "reward": 2.115062713623047,
      "reward_std": 0.8740195035934448,
      "rewards/reward_function": 2.115062713623047,
      "step": 1394
    },
    {
      "completion_length": 376.4285888671875,
      "epoch": 0.7980549199084668,
      "grad_norm": 0.18847620487213135,
      "kl": 0.1720634549856186,
      "learning_rate": 5.959888424742391e-07,
      "loss": 0.0069,
      "reward": 2.0486323833465576,
      "reward_std": 1.0499002933502197,
      "rewards/reward_function": 2.0486323833465576,
      "step": 1395
    },
    {
      "completion_length": 372.21429443359375,
      "epoch": 0.7986270022883295,
      "grad_norm": 1.2325444221496582,
      "kl": 0.19967004656791687,
      "learning_rate": 5.927569705109828e-07,
      "loss": 0.008,
      "reward": 2.334091901779175,
      "reward_std": 1.038485050201416,
      "rewards/reward_function": 2.334091901779175,
      "step": 1396
    },
    {
      "completion_length": 368.0714416503906,
      "epoch": 0.7991990846681922,
      "grad_norm": 0.27093422412872314,
      "kl": 0.17431865632534027,
      "learning_rate": 5.895327061568776e-07,
      "loss": 0.007,
      "reward": 2.498955488204956,
      "reward_std": 1.2363183498382568,
      "rewards/reward_function": 2.498955488204956,
      "step": 1397
    },
    {
      "completion_length": 345.39288330078125,
      "epoch": 0.799771167048055,
      "grad_norm": 0.22041967511177063,
      "kl": 0.12403830140829086,
      "learning_rate": 5.863160622728642e-07,
      "loss": 0.005,
      "reward": 3.0058703422546387,
      "reward_std": 0.7497028708457947,
      "rewards/reward_function": 3.0058703422546387,
      "step": 1398
    },
    {
      "completion_length": 339.39288330078125,
      "epoch": 0.8003432494279176,
      "grad_norm": 4.908897876739502,
      "kl": 0.6441694498062134,
      "learning_rate": 5.831070516894901e-07,
      "loss": 0.0258,
      "reward": 1.6357780694961548,
      "reward_std": 0.8992767930030823,
      "rewards/reward_function": 1.6357780694961548,
      "step": 1399
    },
    {
      "completion_length": 335.75,
      "epoch": 0.8009153318077803,
      "grad_norm": 0.21723055839538574,
      "kl": 0.14932894706726074,
      "learning_rate": 5.799056872068523e-07,
      "loss": 0.006,
      "reward": 1.9358476400375366,
      "reward_std": 1.2369751930236816,
      "rewards/reward_function": 1.9358476400375366,
      "step": 1400
    },
    {
      "completion_length": 349.0000305175781,
      "epoch": 0.801487414187643,
      "grad_norm": 3.0616354942321777,
      "kl": 0.45164817571640015,
      "learning_rate": 5.767119815945504e-07,
      "loss": 0.0181,
      "reward": 1.9047977924346924,
      "reward_std": 1.1735954284667969,
      "rewards/reward_function": 1.9047977924346924,
      "step": 1401
    },
    {
      "completion_length": 323.9285888671875,
      "epoch": 0.8020594965675057,
      "grad_norm": 0.9751774668693542,
      "kl": 0.15716712176799774,
      "learning_rate": 5.735259475916361e-07,
      "loss": 0.0063,
      "reward": 2.109639883041382,
      "reward_std": 1.0525784492492676,
      "rewards/reward_function": 2.109639883041382,
      "step": 1402
    },
    {
      "completion_length": 330.5,
      "epoch": 0.8026315789473685,
      "grad_norm": 0.2590183615684509,
      "kl": 0.23454146087169647,
      "learning_rate": 5.703475979065571e-07,
      "loss": 0.0094,
      "reward": 2.4310147762298584,
      "reward_std": 0.9580467939376831,
      "rewards/reward_function": 2.4310147762298584,
      "step": 1403
    },
    {
      "completion_length": 340.5357360839844,
      "epoch": 0.8032036613272311,
      "grad_norm": 0.18704500794410706,
      "kl": 0.16088223457336426,
      "learning_rate": 5.671769452171125e-07,
      "loss": 0.0064,
      "reward": 2.0190234184265137,
      "reward_std": 1.0744845867156982,
      "rewards/reward_function": 2.0190234184265137,
      "step": 1404
    },
    {
      "completion_length": 332.64288330078125,
      "epoch": 0.8037757437070938,
      "grad_norm": 2.0605812072753906,
      "kl": 0.6649402976036072,
      "learning_rate": 5.640140021703974e-07,
      "loss": 0.0266,
      "reward": 2.8268625736236572,
      "reward_std": 0.794187605381012,
      "rewards/reward_function": 2.8268625736236572,
      "step": 1405
    },
    {
      "completion_length": 349.0357360839844,
      "epoch": 0.8043478260869565,
      "grad_norm": 4.607220649719238,
      "kl": 0.49823763966560364,
      "learning_rate": 5.608587813827568e-07,
      "loss": 0.0199,
      "reward": 1.326676368713379,
      "reward_std": 0.6451515555381775,
      "rewards/reward_function": 1.326676368713379,
      "step": 1406
    },
    {
      "completion_length": 327.1071472167969,
      "epoch": 0.8049199084668193,
      "grad_norm": 0.21370811760425568,
      "kl": 0.15941447019577026,
      "learning_rate": 5.57711295439732e-07,
      "loss": 0.0064,
      "reward": 1.6777989864349365,
      "reward_std": 0.737716555595398,
      "rewards/reward_function": 1.6777989864349365,
      "step": 1407
    },
    {
      "completion_length": 306.1071472167969,
      "epoch": 0.8054919908466819,
      "grad_norm": 0.1874859631061554,
      "kl": 0.1405525505542755,
      "learning_rate": 5.54571556896013e-07,
      "loss": 0.0056,
      "reward": 1.7848560810089111,
      "reward_std": 0.9477502107620239,
      "rewards/reward_function": 1.7848560810089111,
      "step": 1408
    },
    {
      "completion_length": 403.5714416503906,
      "epoch": 0.8060640732265446,
      "grad_norm": 0.1696355640888214,
      "kl": 0.12858912348747253,
      "learning_rate": 5.514395782753842e-07,
      "loss": 0.0051,
      "reward": 1.9698090553283691,
      "reward_std": 1.1207389831542969,
      "rewards/reward_function": 1.9698090553283691,
      "step": 1409
    },
    {
      "completion_length": 378.4285888671875,
      "epoch": 0.8066361556064073,
      "grad_norm": 0.5294426679611206,
      "kl": 0.18673191964626312,
      "learning_rate": 5.483153720706799e-07,
      "loss": 0.0075,
      "reward": 2.2969765663146973,
      "reward_std": 0.941791832447052,
      "rewards/reward_function": 2.2969765663146973,
      "step": 1410
    },
    {
      "completion_length": 342.9285888671875,
      "epoch": 0.8072082379862701,
      "grad_norm": 0.7137978076934814,
      "kl": 0.15339460968971252,
      "learning_rate": 5.451989507437311e-07,
      "loss": 0.0061,
      "reward": 2.4140090942382812,
      "reward_std": 1.206676721572876,
      "rewards/reward_function": 2.4140090942382812,
      "step": 1411
    },
    {
      "completion_length": 340.1071472167969,
      "epoch": 0.8077803203661327,
      "grad_norm": 0.18755479156970978,
      "kl": 0.12873971462249756,
      "learning_rate": 5.42090326725315e-07,
      "loss": 0.0051,
      "reward": 1.7301578521728516,
      "reward_std": 0.7801978588104248,
      "rewards/reward_function": 1.7301578521728516,
      "step": 1412
    },
    {
      "completion_length": 383.6071472167969,
      "epoch": 0.8083524027459954,
      "grad_norm": 0.6353592872619629,
      "kl": 0.2619240880012512,
      "learning_rate": 5.389895124151084e-07,
      "loss": 0.0105,
      "reward": 1.9151430130004883,
      "reward_std": 0.9097501635551453,
      "rewards/reward_function": 1.9151430130004883,
      "step": 1413
    },
    {
      "completion_length": 327.1785888671875,
      "epoch": 0.8089244851258581,
      "grad_norm": 0.34527942538261414,
      "kl": 0.14192542433738708,
      "learning_rate": 5.358965201816371e-07,
      "loss": 0.0057,
      "reward": 2.180009603500366,
      "reward_std": 1.0104219913482666,
      "rewards/reward_function": 2.180009603500366,
      "step": 1414
    },
    {
      "completion_length": 365.5357360839844,
      "epoch": 0.8094965675057209,
      "grad_norm": 0.27493613958358765,
      "kl": 0.15791741013526917,
      "learning_rate": 5.328113623622258e-07,
      "loss": 0.0063,
      "reward": 2.240363359451294,
      "reward_std": 0.9518481492996216,
      "rewards/reward_function": 2.240363359451294,
      "step": 1415
    },
    {
      "completion_length": 313.6071472167969,
      "epoch": 0.8100686498855835,
      "grad_norm": 0.19006191194057465,
      "kl": 0.16327844560146332,
      "learning_rate": 5.297340512629476e-07,
      "loss": 0.0065,
      "reward": 2.297822952270508,
      "reward_std": 0.9276653528213501,
      "rewards/reward_function": 2.297822952270508,
      "step": 1416
    },
    {
      "completion_length": 349.6071472167969,
      "epoch": 0.8106407322654462,
      "grad_norm": 0.15067018568515778,
      "kl": 0.12245438992977142,
      "learning_rate": 5.266645991585795e-07,
      "loss": 0.0049,
      "reward": 2.4609806537628174,
      "reward_std": 1.1330358982086182,
      "rewards/reward_function": 2.4609806537628174,
      "step": 1417
    },
    {
      "completion_length": 331.1071472167969,
      "epoch": 0.8112128146453089,
      "grad_norm": 1.0187569856643677,
      "kl": 0.21454958617687225,
      "learning_rate": 5.236030182925475e-07,
      "loss": 0.0086,
      "reward": 1.8447697162628174,
      "reward_std": 1.1418296098709106,
      "rewards/reward_function": 1.8447697162628174,
      "step": 1418
    },
    {
      "completion_length": 326.7857360839844,
      "epoch": 0.8117848970251716,
      "grad_norm": 0.29623928666114807,
      "kl": 0.14836548268795013,
      "learning_rate": 5.205493208768827e-07,
      "loss": 0.0059,
      "reward": 2.0400140285491943,
      "reward_std": 1.1061255931854248,
      "rewards/reward_function": 2.0400140285491943,
      "step": 1419
    },
    {
      "completion_length": 359.8214416503906,
      "epoch": 0.8123569794050344,
      "grad_norm": 0.27050018310546875,
      "kl": 0.16855821013450623,
      "learning_rate": 5.175035190921726e-07,
      "loss": 0.0067,
      "reward": 1.6054617166519165,
      "reward_std": 0.9141313433647156,
      "rewards/reward_function": 1.6054617166519165,
      "step": 1420
    },
    {
      "completion_length": 405.0714416503906,
      "epoch": 0.812929061784897,
      "grad_norm": 0.2431751936674118,
      "kl": 0.1546180546283722,
      "learning_rate": 5.144656250875061e-07,
      "loss": 0.0062,
      "reward": 2.624762535095215,
      "reward_std": 0.6725619435310364,
      "rewards/reward_function": 2.624762535095215,
      "step": 1421
    },
    {
      "completion_length": 350.9285888671875,
      "epoch": 0.8135011441647597,
      "grad_norm": 195.3209228515625,
      "kl": 3.050774574279785,
      "learning_rate": 5.11435650980433e-07,
      "loss": 0.122,
      "reward": 1.9919873476028442,
      "reward_std": 1.2814861536026,
      "rewards/reward_function": 1.9919873476028442,
      "step": 1422
    },
    {
      "completion_length": 342.4285888671875,
      "epoch": 0.8140732265446224,
      "grad_norm": 0.2839534282684326,
      "kl": 0.15005141496658325,
      "learning_rate": 5.0841360885691e-07,
      "loss": 0.006,
      "reward": 2.257258176803589,
      "reward_std": 1.146335482597351,
      "rewards/reward_function": 2.257258176803589,
      "step": 1423
    },
    {
      "completion_length": 333.21429443359375,
      "epoch": 0.8146453089244852,
      "grad_norm": 0.5349013209342957,
      "kl": 0.15031252801418304,
      "learning_rate": 5.053995107712567e-07,
      "loss": 0.006,
      "reward": 2.0218708515167236,
      "reward_std": 1.1183372735977173,
      "rewards/reward_function": 2.0218708515167236,
      "step": 1424
    },
    {
      "completion_length": 380.5714416503906,
      "epoch": 0.8152173913043478,
      "grad_norm": 0.4497327506542206,
      "kl": 0.16268706321716309,
      "learning_rate": 5.023933687461041e-07,
      "loss": 0.0065,
      "reward": 2.458691358566284,
      "reward_std": 1.139290690422058,
      "rewards/reward_function": 2.458691358566284,
      "step": 1425
    },
    {
      "completion_length": 381.14288330078125,
      "epoch": 0.8157894736842105,
      "grad_norm": 7.293765544891357,
      "kl": 0.2976752817630768,
      "learning_rate": 4.993951947723497e-07,
      "loss": 0.0119,
      "reward": 2.021026611328125,
      "reward_std": 1.059070110321045,
      "rewards/reward_function": 2.021026611328125,
      "step": 1426
    },
    {
      "completion_length": 321.14288330078125,
      "epoch": 0.8163615560640732,
      "grad_norm": 0.17501670122146606,
      "kl": 0.12569069862365723,
      "learning_rate": 4.964050008091054e-07,
      "loss": 0.005,
      "reward": 2.0267179012298584,
      "reward_std": 1.2278366088867188,
      "rewards/reward_function": 2.0267179012298584,
      "step": 1427
    },
    {
      "completion_length": 310.6071472167969,
      "epoch": 0.816933638443936,
      "grad_norm": 0.1924608200788498,
      "kl": 0.18536625802516937,
      "learning_rate": 4.934227987836546e-07,
      "loss": 0.0074,
      "reward": 1.4497196674346924,
      "reward_std": 1.074397325515747,
      "rewards/reward_function": 1.4497196674346924,
      "step": 1428
    },
    {
      "completion_length": 311.3571472167969,
      "epoch": 0.8175057208237986,
      "grad_norm": 0.19456681609153748,
      "kl": 0.1269546002149582,
      "learning_rate": 4.904486005914027e-07,
      "loss": 0.0051,
      "reward": 2.86708402633667,
      "reward_std": 0.6495133638381958,
      "rewards/reward_function": 2.86708402633667,
      "step": 1429
    },
    {
      "completion_length": 371.8571472167969,
      "epoch": 0.8180778032036613,
      "grad_norm": 16.298667907714844,
      "kl": 0.7864172458648682,
      "learning_rate": 4.874824180958271e-07,
      "loss": 0.0315,
      "reward": 2.6705803871154785,
      "reward_std": 0.8244845867156982,
      "rewards/reward_function": 2.6705803871154785,
      "step": 1430
    },
    {
      "completion_length": 385.0000305175781,
      "epoch": 0.818649885583524,
      "grad_norm": 0.2607991397380829,
      "kl": 0.14036650955677032,
      "learning_rate": 4.845242631284344e-07,
      "loss": 0.0056,
      "reward": 1.9292230606079102,
      "reward_std": 0.7941774129867554,
      "rewards/reward_function": 1.9292230606079102,
      "step": 1431
    },
    {
      "completion_length": 367.14288330078125,
      "epoch": 0.8192219679633868,
      "grad_norm": 1.7254127264022827,
      "kl": 0.20147021114826202,
      "learning_rate": 4.815741474887104e-07,
      "loss": 0.0081,
      "reward": 1.7548328638076782,
      "reward_std": 1.2670897245407104,
      "rewards/reward_function": 1.7548328638076782,
      "step": 1432
    },
    {
      "completion_length": 370.0000305175781,
      "epoch": 0.8197940503432495,
      "grad_norm": 0.9415956139564514,
      "kl": 0.3316303789615631,
      "learning_rate": 4.786320829440741e-07,
      "loss": 0.0133,
      "reward": 2.6842591762542725,
      "reward_std": 0.957444429397583,
      "rewards/reward_function": 2.6842591762542725,
      "step": 1433
    },
    {
      "completion_length": 329.0357360839844,
      "epoch": 0.8203661327231121,
      "grad_norm": 0.16224689781665802,
      "kl": 0.1338268369436264,
      "learning_rate": 4.7569808122982813e-07,
      "loss": 0.0054,
      "reward": 1.8348323106765747,
      "reward_std": 1.0276353359222412,
      "rewards/reward_function": 1.8348323106765747,
      "step": 1434
    },
    {
      "completion_length": 371.3214416503906,
      "epoch": 0.8209382151029748,
      "grad_norm": 2.2298083305358887,
      "kl": 0.42323076725006104,
      "learning_rate": 4.727721540491173e-07,
      "loss": 0.0169,
      "reward": 2.405491828918457,
      "reward_std": 1.241464614868164,
      "rewards/reward_function": 2.405491828918457,
      "step": 1435
    },
    {
      "completion_length": 318.8214416503906,
      "epoch": 0.8215102974828375,
      "grad_norm": 0.1891143023967743,
      "kl": 0.12490848451852798,
      "learning_rate": 4.6985431307287556e-07,
      "loss": 0.005,
      "reward": 2.485748767852783,
      "reward_std": 1.0970271825790405,
      "rewards/reward_function": 2.485748767852783,
      "step": 1436
    },
    {
      "completion_length": 331.9285888671875,
      "epoch": 0.8220823798627003,
      "grad_norm": 5.386889457702637,
      "kl": 0.9214995503425598,
      "learning_rate": 4.6694456993978425e-07,
      "loss": 0.0369,
      "reward": 2.212146759033203,
      "reward_std": 0.8862817883491516,
      "rewards/reward_function": 2.212146759033203,
      "step": 1437
    },
    {
      "completion_length": 317.3571472167969,
      "epoch": 0.8226544622425629,
      "grad_norm": 0.20348447561264038,
      "kl": 0.13773109018802643,
      "learning_rate": 4.640429362562263e-07,
      "loss": 0.0055,
      "reward": 1.5720654726028442,
      "reward_std": 0.6962600946426392,
      "rewards/reward_function": 1.5720654726028442,
      "step": 1438
    },
    {
      "completion_length": 375.96429443359375,
      "epoch": 0.8232265446224256,
      "grad_norm": 0.7426113486289978,
      "kl": 0.2239607274532318,
      "learning_rate": 4.61149423596233e-07,
      "loss": 0.009,
      "reward": 1.9169851541519165,
      "reward_std": 0.8660178184509277,
      "rewards/reward_function": 1.9169851541519165,
      "step": 1439
    },
    {
      "completion_length": 369.39288330078125,
      "epoch": 0.8237986270022883,
      "grad_norm": 0.30905595421791077,
      "kl": 0.15469513833522797,
      "learning_rate": 4.5826404350144597e-07,
      "loss": 0.0062,
      "reward": 2.0114150047302246,
      "reward_std": 1.133552074432373,
      "rewards/reward_function": 2.0114150047302246,
      "step": 1440
    },
    {
      "completion_length": 314.4285888671875,
      "epoch": 0.8243707093821511,
      "grad_norm": 0.18118704855442047,
      "kl": 0.13749240338802338,
      "learning_rate": 4.5538680748106576e-07,
      "loss": 0.0055,
      "reward": 1.847055435180664,
      "reward_std": 0.6376089453697205,
      "rewards/reward_function": 1.847055435180664,
      "step": 1441
    },
    {
      "completion_length": 342.64288330078125,
      "epoch": 0.8249427917620137,
      "grad_norm": 6.001440048217773,
      "kl": 0.719327986240387,
      "learning_rate": 4.525177270118089e-07,
      "loss": 0.0288,
      "reward": 1.590598702430725,
      "reward_std": 0.7346889972686768,
      "rewards/reward_function": 1.590598702430725,
      "step": 1442
    },
    {
      "completion_length": 337.0357360839844,
      "epoch": 0.8255148741418764,
      "grad_norm": 0.18746139109134674,
      "kl": 0.13921408355236053,
      "learning_rate": 4.4965681353786094e-07,
      "loss": 0.0056,
      "reward": 1.722407341003418,
      "reward_std": 0.898395836353302,
      "rewards/reward_function": 1.722407341003418,
      "step": 1443
    },
    {
      "completion_length": 312.64288330078125,
      "epoch": 0.8260869565217391,
      "grad_norm": 0.19125182926654816,
      "kl": 0.1283683031797409,
      "learning_rate": 4.468040784708305e-07,
      "loss": 0.0051,
      "reward": 2.330160617828369,
      "reward_std": 0.7709058523178101,
      "rewards/reward_function": 2.330160617828369,
      "step": 1444
    },
    {
      "completion_length": 342.2857360839844,
      "epoch": 0.8266590389016019,
      "grad_norm": 796.9683837890625,
      "kl": 2.695296049118042,
      "learning_rate": 4.4395953318970564e-07,
      "loss": 0.1078,
      "reward": 1.4038784503936768,
      "reward_std": 0.9138849973678589,
      "rewards/reward_function": 1.4038784503936768,
      "step": 1445
    },
    {
      "completion_length": 378.21429443359375,
      "epoch": 0.8272311212814645,
      "grad_norm": 6.14599609375,
      "kl": 0.5843194723129272,
      "learning_rate": 4.411231890408041e-07,
      "loss": 0.0234,
      "reward": 2.4367570877075195,
      "reward_std": 1.2064775228500366,
      "rewards/reward_function": 2.4367570877075195,
      "step": 1446
    },
    {
      "completion_length": 373.46429443359375,
      "epoch": 0.8278032036613272,
      "grad_norm": 1.303322672843933,
      "kl": 0.23940660059452057,
      "learning_rate": 4.3829505733773397e-07,
      "loss": 0.0096,
      "reward": 2.4751391410827637,
      "reward_std": 0.8208857774734497,
      "rewards/reward_function": 2.4751391410827637,
      "step": 1447
    },
    {
      "completion_length": 320.5714416503906,
      "epoch": 0.8283752860411899,
      "grad_norm": 0.19205820560455322,
      "kl": 0.16358593106269836,
      "learning_rate": 4.354751493613432e-07,
      "loss": 0.0065,
      "reward": 2.5623714923858643,
      "reward_std": 1.0782825946807861,
      "rewards/reward_function": 2.5623714923858643,
      "step": 1448
    },
    {
      "completion_length": 322.8214416503906,
      "epoch": 0.8289473684210527,
      "grad_norm": 0.26951757073402405,
      "kl": 0.15536978840827942,
      "learning_rate": 4.326634763596785e-07,
      "loss": 0.0062,
      "reward": 2.269127130508423,
      "reward_std": 0.8872981071472168,
      "rewards/reward_function": 2.269127130508423,
      "step": 1449
    },
    {
      "completion_length": 352.46429443359375,
      "epoch": 0.8295194508009154,
      "grad_norm": 8.434430122375488,
      "kl": 0.8448156118392944,
      "learning_rate": 4.2986004954793823e-07,
      "loss": 0.0338,
      "reward": 2.6338393688201904,
      "reward_std": 1.0191859006881714,
      "rewards/reward_function": 2.6338393688201904,
      "step": 1450
    },
    {
      "completion_length": 349.21429443359375,
      "epoch": 0.830091533180778,
      "grad_norm": 0.6324264407157898,
      "kl": 0.1502688229084015,
      "learning_rate": 4.2706488010842957e-07,
      "loss": 0.006,
      "reward": 1.7609425783157349,
      "reward_std": 0.8715909123420715,
      "rewards/reward_function": 1.7609425783157349,
      "step": 1451
    },
    {
      "completion_length": 377.6071472167969,
      "epoch": 0.8306636155606407,
      "grad_norm": 5.895503520965576,
      "kl": 0.4670054018497467,
      "learning_rate": 4.242779791905202e-07,
      "loss": 0.0187,
      "reward": 1.6852335929870605,
      "reward_std": 1.2596409320831299,
      "rewards/reward_function": 1.6852335929870605,
      "step": 1452
    },
    {
      "completion_length": 400.3214416503906,
      "epoch": 0.8312356979405034,
      "grad_norm": 5.375813007354736,
      "kl": 0.8417397737503052,
      "learning_rate": 4.214993579105986e-07,
      "loss": 0.0337,
      "reward": 1.8650842905044556,
      "reward_std": 1.114862084388733,
      "rewards/reward_function": 1.8650842905044556,
      "step": 1453
    },
    {
      "completion_length": 369.0000305175781,
      "epoch": 0.8318077803203662,
      "grad_norm": 0.3320905268192291,
      "kl": 0.12648127973079681,
      "learning_rate": 4.1872902735202695e-07,
      "loss": 0.0051,
      "reward": 2.4202651977539062,
      "reward_std": 1.2915420532226562,
      "rewards/reward_function": 2.4202651977539062,
      "step": 1454
    },
    {
      "completion_length": 349.5000305175781,
      "epoch": 0.8323798627002288,
      "grad_norm": 0.17249469459056854,
      "kl": 0.11766600608825684,
      "learning_rate": 4.15966998565096e-07,
      "loss": 0.0047,
      "reward": 1.8294163942337036,
      "reward_std": 0.4925500452518463,
      "rewards/reward_function": 1.8294163942337036,
      "step": 1455
    },
    {
      "completion_length": 389.46429443359375,
      "epoch": 0.8329519450800915,
      "grad_norm": 1.405072808265686,
      "kl": 0.2111685872077942,
      "learning_rate": 4.1321328256698435e-07,
      "loss": 0.0084,
      "reward": 2.2511627674102783,
      "reward_std": 0.8904911875724792,
      "rewards/reward_function": 2.2511627674102783,
      "step": 1456
    },
    {
      "completion_length": 306.5357360839844,
      "epoch": 0.8335240274599542,
      "grad_norm": 4.864382266998291,
      "kl": 1.0993497371673584,
      "learning_rate": 4.104678903417114e-07,
      "loss": 0.044,
      "reward": 1.7169008255004883,
      "reward_std": 1.1182262897491455,
      "rewards/reward_function": 1.7169008255004883,
      "step": 1457
    },
    {
      "completion_length": 369.0357360839844,
      "epoch": 0.834096109839817,
      "grad_norm": 6.67081356048584,
      "kl": 0.32501059770584106,
      "learning_rate": 4.0773083284009587e-07,
      "loss": 0.013,
      "reward": 1.7760096788406372,
      "reward_std": 0.9615750312805176,
      "rewards/reward_function": 1.7760096788406372,
      "step": 1458
    },
    {
      "completion_length": 310.25,
      "epoch": 0.8346681922196796,
      "grad_norm": 0.18418560922145844,
      "kl": 0.12310075759887695,
      "learning_rate": 4.050021209797084e-07,
      "loss": 0.0049,
      "reward": 2.3356943130493164,
      "reward_std": 0.6851566433906555,
      "rewards/reward_function": 2.3356943130493164,
      "step": 1459
    },
    {
      "completion_length": 374.21429443359375,
      "epoch": 0.8352402745995423,
      "grad_norm": 0.18964990973472595,
      "kl": 0.18182091414928436,
      "learning_rate": 4.022817656448338e-07,
      "loss": 0.0073,
      "reward": 1.9337406158447266,
      "reward_std": 0.817672610282898,
      "rewards/reward_function": 1.9337406158447266,
      "step": 1460
    },
    {
      "completion_length": 386.8571472167969,
      "epoch": 0.835812356979405,
      "grad_norm": 2.9496452808380127,
      "kl": 0.2518266439437866,
      "learning_rate": 3.99569777686421e-07,
      "loss": 0.0101,
      "reward": 1.4270379543304443,
      "reward_std": 0.875464916229248,
      "rewards/reward_function": 1.4270379543304443,
      "step": 1461
    },
    {
      "completion_length": 355.4285888671875,
      "epoch": 0.8363844393592678,
      "grad_norm": 0.2935781478881836,
      "kl": 0.1724141389131546,
      "learning_rate": 3.9686616792204677e-07,
      "loss": 0.0069,
      "reward": 2.4691078662872314,
      "reward_std": 1.0747913122177124,
      "rewards/reward_function": 2.4691078662872314,
      "step": 1462
    },
    {
      "completion_length": 357.7857360839844,
      "epoch": 0.8369565217391305,
      "grad_norm": 50.379173278808594,
      "kl": 2.0868215560913086,
      "learning_rate": 3.941709471358671e-07,
      "loss": 0.0835,
      "reward": 1.4173035621643066,
      "reward_std": 0.6262977719306946,
      "rewards/reward_function": 1.4173035621643066,
      "step": 1463
    },
    {
      "completion_length": 385.39288330078125,
      "epoch": 0.8375286041189931,
      "grad_norm": 0.20338037610054016,
      "kl": 0.1633460819721222,
      "learning_rate": 3.914841260785746e-07,
      "loss": 0.0065,
      "reward": 2.5186328887939453,
      "reward_std": 0.8310827612876892,
      "rewards/reward_function": 2.5186328887939453,
      "step": 1464
    },
    {
      "completion_length": 339.8571472167969,
      "epoch": 0.8381006864988558,
      "grad_norm": 0.22087977826595306,
      "kl": 0.1631796658039093,
      "learning_rate": 3.8880571546736e-07,
      "loss": 0.0065,
      "reward": 1.8193871974945068,
      "reward_std": 1.1553272008895874,
      "rewards/reward_function": 1.8193871974945068,
      "step": 1465
    },
    {
      "completion_length": 435.7857360839844,
      "epoch": 0.8386727688787186,
      "grad_norm": 0.5147433280944824,
      "kl": 0.19549652934074402,
      "learning_rate": 3.8613572598586333e-07,
      "loss": 0.0078,
      "reward": 1.4958434104919434,
      "reward_std": 0.9718704223632812,
      "rewards/reward_function": 1.4958434104919434,
      "step": 1466
    },
    {
      "completion_length": 338.75,
      "epoch": 0.8392448512585813,
      "grad_norm": 2.491565704345703,
      "kl": 0.43243589997291565,
      "learning_rate": 3.8347416828413727e-07,
      "loss": 0.0173,
      "reward": 2.167450428009033,
      "reward_std": 0.9556452035903931,
      "rewards/reward_function": 2.167450428009033,
      "step": 1467
    },
    {
      "completion_length": 329.96429443359375,
      "epoch": 0.8398169336384439,
      "grad_norm": 0.353941410779953,
      "kl": 0.15673059225082397,
      "learning_rate": 3.8082105297860024e-07,
      "loss": 0.0063,
      "reward": 2.1957919597625732,
      "reward_std": 1.3913146257400513,
      "rewards/reward_function": 2.1957919597625732,
      "step": 1468
    },
    {
      "completion_length": 345.7857360839844,
      "epoch": 0.8403890160183066,
      "grad_norm": 0.5966039896011353,
      "kl": 0.17263220250606537,
      "learning_rate": 3.781763906519964e-07,
      "loss": 0.0069,
      "reward": 2.627110719680786,
      "reward_std": 0.9355580806732178,
      "rewards/reward_function": 2.627110719680786,
      "step": 1469
    },
    {
      "completion_length": 346.6785888671875,
      "epoch": 0.8409610983981693,
      "grad_norm": 0.19574742019176483,
      "kl": 0.1693505495786667,
      "learning_rate": 3.755401918533516e-07,
      "loss": 0.0068,
      "reward": 2.3578155040740967,
      "reward_std": 1.2962230443954468,
      "rewards/reward_function": 2.3578155040740967,
      "step": 1470
    },
    {
      "completion_length": 376.4285888671875,
      "epoch": 0.8415331807780321,
      "grad_norm": 3.9605765342712402,
      "kl": 0.16199283301830292,
      "learning_rate": 3.729124670979334e-07,
      "loss": 0.0065,
      "reward": 2.433136463165283,
      "reward_std": 0.7834250330924988,
      "rewards/reward_function": 2.433136463165283,
      "step": 1471
    },
    {
      "completion_length": 328.6071472167969,
      "epoch": 0.8421052631578947,
      "grad_norm": 4.1895270347595215,
      "kl": 0.16999617218971252,
      "learning_rate": 3.7029322686720823e-07,
      "loss": 0.0068,
      "reward": 2.529766082763672,
      "reward_std": 0.9608149528503418,
      "rewards/reward_function": 2.529766082763672,
      "step": 1472
    },
    {
      "completion_length": 298.71429443359375,
      "epoch": 0.8426773455377574,
      "grad_norm": 0.21105647087097168,
      "kl": 0.13161656260490417,
      "learning_rate": 3.6768248160879786e-07,
      "loss": 0.0053,
      "reward": 2.0764403343200684,
      "reward_std": 0.9813300967216492,
      "rewards/reward_function": 2.0764403343200684,
      "step": 1473
    },
    {
      "completion_length": 320.5714416503906,
      "epoch": 0.8432494279176201,
      "grad_norm": 1.4531828165054321,
      "kl": 0.34209373593330383,
      "learning_rate": 3.650802417364413e-07,
      "loss": 0.0137,
      "reward": 1.9977179765701294,
      "reward_std": 0.9317797422409058,
      "rewards/reward_function": 1.9977179765701294,
      "step": 1474
    },
    {
      "completion_length": 369.2500305175781,
      "epoch": 0.8438215102974829,
      "grad_norm": 0.19835075736045837,
      "kl": 0.15535473823547363,
      "learning_rate": 3.6248651762995e-07,
      "loss": 0.0062,
      "reward": 2.057137966156006,
      "reward_std": 1.245540976524353,
      "rewards/reward_function": 2.057137966156006,
      "step": 1475
    },
    {
      "completion_length": 373.2857360839844,
      "epoch": 0.8443935926773455,
      "grad_norm": 4.145951271057129,
      "kl": 0.3053969740867615,
      "learning_rate": 3.5990131963516916e-07,
      "loss": 0.0122,
      "reward": 2.3853988647460938,
      "reward_std": 1.0548079013824463,
      "rewards/reward_function": 2.3853988647460938,
      "step": 1476
    },
    {
      "completion_length": 359.0357360839844,
      "epoch": 0.8449656750572082,
      "grad_norm": 0.1741158366203308,
      "kl": 0.13750866055488586,
      "learning_rate": 3.573246580639325e-07,
      "loss": 0.0055,
      "reward": 2.1364898681640625,
      "reward_std": 1.1207377910614014,
      "rewards/reward_function": 2.1364898681640625,
      "step": 1477
    },
    {
      "completion_length": 313.0714416503906,
      "epoch": 0.8455377574370709,
      "grad_norm": 0.806139349937439,
      "kl": 0.33796775341033936,
      "learning_rate": 3.547565431940267e-07,
      "loss": 0.0135,
      "reward": 1.5936248302459717,
      "reward_std": 0.8841392397880554,
      "rewards/reward_function": 1.5936248302459717,
      "step": 1478
    },
    {
      "completion_length": 362.5000305175781,
      "epoch": 0.8461098398169337,
      "grad_norm": 1.8940387964248657,
      "kl": 0.27472972869873047,
      "learning_rate": 3.52196985269144e-07,
      "loss": 0.011,
      "reward": 2.0369107723236084,
      "reward_std": 0.9645910859107971,
      "rewards/reward_function": 2.0369107723236084,
      "step": 1479
    },
    {
      "completion_length": 361.0714416503906,
      "epoch": 0.8466819221967964,
      "grad_norm": 323.1961669921875,
      "kl": 1.751446008682251,
      "learning_rate": 3.496459944988484e-07,
      "loss": 0.0701,
      "reward": 2.413499593734741,
      "reward_std": 0.8150543570518494,
      "rewards/reward_function": 2.413499593734741,
      "step": 1480
    },
    {
      "completion_length": 314.3214416503906,
      "epoch": 0.847254004576659,
      "grad_norm": 0.9114041328430176,
      "kl": 0.17322345077991486,
      "learning_rate": 3.471035810585291e-07,
      "loss": 0.0069,
      "reward": 2.4582366943359375,
      "reward_std": 0.9683695435523987,
      "rewards/reward_function": 2.4582366943359375,
      "step": 1481
    },
    {
      "completion_length": 367.2857360839844,
      "epoch": 0.8478260869565217,
      "grad_norm": 0.18436577916145325,
      "kl": 0.16056504845619202,
      "learning_rate": 3.445697550893615e-07,
      "loss": 0.0064,
      "reward": 1.5614807605743408,
      "reward_std": 0.7262765765190125,
      "rewards/reward_function": 1.5614807605743408,
      "step": 1482
    },
    {
      "completion_length": 369.21429443359375,
      "epoch": 0.8483981693363845,
      "grad_norm": 0.47044509649276733,
      "kl": 0.1596624255180359,
      "learning_rate": 3.4204452669826915e-07,
      "loss": 0.0064,
      "reward": 2.444218158721924,
      "reward_std": 1.0357286930084229,
      "rewards/reward_function": 2.444218158721924,
      "step": 1483
    },
    {
      "completion_length": 342.7500305175781,
      "epoch": 0.8489702517162472,
      "grad_norm": 0.212871715426445,
      "kl": 0.14318525791168213,
      "learning_rate": 3.3952790595787986e-07,
      "loss": 0.0057,
      "reward": 2.1133780479431152,
      "reward_std": 1.1242070198059082,
      "rewards/reward_function": 2.1133780479431152,
      "step": 1484
    },
    {
      "completion_length": 320.8214416503906,
      "epoch": 0.8495423340961098,
      "grad_norm": 0.20239533483982086,
      "kl": 0.12685512006282806,
      "learning_rate": 3.37019902906488e-07,
      "loss": 0.0051,
      "reward": 1.7311807870864868,
      "reward_std": 0.8676111102104187,
      "rewards/reward_function": 1.7311807870864868,
      "step": 1485
    },
    {
      "completion_length": 349.3571472167969,
      "epoch": 0.8501144164759725,
      "grad_norm": 0.18494689464569092,
      "kl": 0.17264769971370697,
      "learning_rate": 3.34520527548014e-07,
      "loss": 0.0069,
      "reward": 2.2940456867218018,
      "reward_std": 1.0172919034957886,
      "rewards/reward_function": 2.2940456867218018,
      "step": 1486
    },
    {
      "completion_length": 410.5357360839844,
      "epoch": 0.8506864988558352,
      "grad_norm": 0.29048895835876465,
      "kl": 0.1637188047170639,
      "learning_rate": 3.320297898519645e-07,
      "loss": 0.0065,
      "reward": 1.872045874595642,
      "reward_std": 0.6250460743904114,
      "rewards/reward_function": 1.872045874595642,
      "step": 1487
    },
    {
      "completion_length": 411.71429443359375,
      "epoch": 0.851258581235698,
      "grad_norm": 5.836248397827148,
      "kl": 0.5008272528648376,
      "learning_rate": 3.295476997533906e-07,
      "loss": 0.02,
      "reward": 1.9439891576766968,
      "reward_std": 0.9422363042831421,
      "rewards/reward_function": 1.9439891576766968,
      "step": 1488
    },
    {
      "completion_length": 345.4285888671875,
      "epoch": 0.8518306636155606,
      "grad_norm": 94.52761840820312,
      "kl": 1.2313975095748901,
      "learning_rate": 3.2707426715285096e-07,
      "loss": 0.0493,
      "reward": 2.00411319732666,
      "reward_std": 0.663395345211029,
      "rewards/reward_function": 2.00411319732666,
      "step": 1489
    },
    {
      "completion_length": 401.7500305175781,
      "epoch": 0.8524027459954233,
      "grad_norm": 0.14217205345630646,
      "kl": 0.12079623341560364,
      "learning_rate": 3.2460950191637226e-07,
      "loss": 0.0048,
      "reward": 2.080117702484131,
      "reward_std": 0.961166262626648,
      "rewards/reward_function": 2.080117702484131,
      "step": 1490
    },
    {
      "completion_length": 323.25,
      "epoch": 0.852974828375286,
      "grad_norm": 2.5820248126983643,
      "kl": 0.23964793980121613,
      "learning_rate": 3.221534138754062e-07,
      "loss": 0.0096,
      "reward": 2.366797924041748,
      "reward_std": 0.7986151576042175,
      "rewards/reward_function": 2.366797924041748,
      "step": 1491
    },
    {
      "completion_length": 358.14288330078125,
      "epoch": 0.8535469107551488,
      "grad_norm": 0.19644352793693542,
      "kl": 0.12206330895423889,
      "learning_rate": 3.197060128267951e-07,
      "loss": 0.0049,
      "reward": 2.223844051361084,
      "reward_std": 0.9858920574188232,
      "rewards/reward_function": 2.223844051361084,
      "step": 1492
    },
    {
      "completion_length": 345.9285888671875,
      "epoch": 0.8541189931350115,
      "grad_norm": 0.6903737187385559,
      "kl": 0.23671004176139832,
      "learning_rate": 3.1726730853272963e-07,
      "loss": 0.0095,
      "reward": 2.508556365966797,
      "reward_std": 0.5229029655456543,
      "rewards/reward_function": 2.508556365966797,
      "step": 1493
    },
    {
      "completion_length": 388.6785888671875,
      "epoch": 0.8546910755148741,
      "grad_norm": 0.184109166264534,
      "kl": 0.13550494611263275,
      "learning_rate": 3.148373107207123e-07,
      "loss": 0.0054,
      "reward": 2.756850242614746,
      "reward_std": 1.044445276260376,
      "rewards/reward_function": 2.756850242614746,
      "step": 1494
    },
    {
      "completion_length": 421.5357360839844,
      "epoch": 0.8552631578947368,
      "grad_norm": 0.17963574826717377,
      "kl": 0.13240477442741394,
      "learning_rate": 3.1241602908351404e-07,
      "loss": 0.0053,
      "reward": 1.9862985610961914,
      "reward_std": 1.0649855136871338,
      "rewards/reward_function": 1.9862985610961914,
      "step": 1495
    },
    {
      "completion_length": 349.21429443359375,
      "epoch": 0.8558352402745996,
      "grad_norm": 0.6400407552719116,
      "kl": 0.18273286521434784,
      "learning_rate": 3.100034732791421e-07,
      "loss": 0.0073,
      "reward": 1.3066765069961548,
      "reward_std": 0.9073527455329895,
      "rewards/reward_function": 1.3066765069961548,
      "step": 1496
    },
    {
      "completion_length": 374.96429443359375,
      "epoch": 0.8564073226544623,
      "grad_norm": 2.6338489055633545,
      "kl": 0.8302949070930481,
      "learning_rate": 3.075996529307962e-07,
      "loss": 0.0332,
      "reward": 1.8699560165405273,
      "reward_std": 0.920386552810669,
      "rewards/reward_function": 1.8699560165405273,
      "step": 1497
    },
    {
      "completion_length": 307.46429443359375,
      "epoch": 0.8569794050343249,
      "grad_norm": 0.36770981550216675,
      "kl": 0.16287939250469208,
      "learning_rate": 3.052045776268328e-07,
      "loss": 0.0065,
      "reward": 2.8362276554107666,
      "reward_std": 0.6093575954437256,
      "rewards/reward_function": 2.8362276554107666,
      "step": 1498
    },
    {
      "completion_length": 336.3571472167969,
      "epoch": 0.8575514874141876,
      "grad_norm": 0.25482600927352905,
      "kl": 0.1376713216304779,
      "learning_rate": 3.0281825692072665e-07,
      "loss": 0.0055,
      "reward": 2.256852865219116,
      "reward_std": 1.0693310499191284,
      "rewards/reward_function": 2.256852865219116,
      "step": 1499
    },
    {
      "completion_length": 333.7857360839844,
      "epoch": 0.8581235697940504,
      "grad_norm": 0.7854841947555542,
      "kl": 0.19771452248096466,
      "learning_rate": 3.0044070033103e-07,
      "loss": 0.0079,
      "reward": 2.379192352294922,
      "reward_std": 1.1138428449630737,
      "rewards/reward_function": 2.379192352294922,
      "step": 1500
    },
    {
      "completion_length": 294.7857360839844,
      "epoch": 0.8586956521739131,
      "grad_norm": 0.25198763608932495,
      "kl": 0.1281736195087433,
      "learning_rate": 2.980719173413396e-07,
      "loss": 0.0051,
      "reward": 1.4967557191848755,
      "reward_std": 0.8060566186904907,
      "rewards/reward_function": 1.4967557191848755,
      "step": 1501
    },
    {
      "completion_length": 367.64288330078125,
      "epoch": 0.8592677345537757,
      "grad_norm": 0.18107230961322784,
      "kl": 0.13589708507061005,
      "learning_rate": 2.9571191740025373e-07,
      "loss": 0.0054,
      "reward": 2.2637650966644287,
      "reward_std": 0.8674246072769165,
      "rewards/reward_function": 2.2637650966644287,
      "step": 1502
    },
    {
      "completion_length": 344.39288330078125,
      "epoch": 0.8598398169336384,
      "grad_norm": 0.7773903012275696,
      "kl": 0.18235668540000916,
      "learning_rate": 2.9336070992133844e-07,
      "loss": 0.0073,
      "reward": 2.1941609382629395,
      "reward_std": 1.0541398525238037,
      "rewards/reward_function": 2.1941609382629395,
      "step": 1503
    },
    {
      "completion_length": 318.6785888671875,
      "epoch": 0.8604118993135011,
      "grad_norm": 0.31673818826675415,
      "kl": 0.16058987379074097,
      "learning_rate": 2.910183042830875e-07,
      "loss": 0.0064,
      "reward": 1.7389718294143677,
      "reward_std": 0.9150226712226868,
      "rewards/reward_function": 1.7389718294143677,
      "step": 1504
    },
    {
      "completion_length": 363.0000305175781,
      "epoch": 0.8609839816933639,
      "grad_norm": 0.16817735135555267,
      "kl": 0.13181333243846893,
      "learning_rate": 2.886847098288867e-07,
      "loss": 0.0053,
      "reward": 2.127561330795288,
      "reward_std": 0.9760417938232422,
      "rewards/reward_function": 2.127561330795288,
      "step": 1505
    },
    {
      "completion_length": 320.1071472167969,
      "epoch": 0.8615560640732265,
      "grad_norm": 0.23835867643356323,
      "kl": 0.1397227644920349,
      "learning_rate": 2.8635993586697555e-07,
      "loss": 0.0056,
      "reward": 2.5166015625,
      "reward_std": 0.8392670750617981,
      "rewards/reward_function": 2.5166015625,
      "step": 1506
    },
    {
      "completion_length": 358.2500305175781,
      "epoch": 0.8621281464530892,
      "grad_norm": 5.139972686767578,
      "kl": 0.6105756759643555,
      "learning_rate": 2.840439916704091e-07,
      "loss": 0.0244,
      "reward": 1.7909263372421265,
      "reward_std": 1.0271446704864502,
      "rewards/reward_function": 1.7909263372421265,
      "step": 1507
    },
    {
      "completion_length": 358.1785888671875,
      "epoch": 0.8627002288329519,
      "grad_norm": 2.572617292404175,
      "kl": 0.7386428117752075,
      "learning_rate": 2.817368864770245e-07,
      "loss": 0.0295,
      "reward": 1.7424309253692627,
      "reward_std": 1.0884315967559814,
      "rewards/reward_function": 1.7424309253692627,
      "step": 1508
    },
    {
      "completion_length": 345.2857360839844,
      "epoch": 0.8632723112128147,
      "grad_norm": 3.1940224170684814,
      "kl": 0.4492894113063812,
      "learning_rate": 2.794386294894e-07,
      "loss": 0.018,
      "reward": 1.4709277153015137,
      "reward_std": 0.6381848454475403,
      "rewards/reward_function": 1.4709277153015137,
      "step": 1509
    },
    {
      "completion_length": 374.14288330078125,
      "epoch": 0.8638443935926774,
      "grad_norm": 1.17685866355896,
      "kl": 0.19142641127109528,
      "learning_rate": 2.7714922987482074e-07,
      "loss": 0.0077,
      "reward": 2.2331931591033936,
      "reward_std": 0.840381920337677,
      "rewards/reward_function": 2.2331931591033936,
      "step": 1510
    },
    {
      "completion_length": 353.5714416503906,
      "epoch": 0.86441647597254,
      "grad_norm": 0.3028343915939331,
      "kl": 0.1599695235490799,
      "learning_rate": 2.7486869676524197e-07,
      "loss": 0.0064,
      "reward": 1.7883936166763306,
      "reward_std": 1.3907660245895386,
      "rewards/reward_function": 1.7883936166763306,
      "step": 1511
    },
    {
      "completion_length": 314.6785888671875,
      "epoch": 0.8649885583524027,
      "grad_norm": 2.145860195159912,
      "kl": 0.3058929145336151,
      "learning_rate": 2.7259703925725243e-07,
      "loss": 0.0122,
      "reward": 1.9084930419921875,
      "reward_std": 0.8103724122047424,
      "rewards/reward_function": 1.9084930419921875,
      "step": 1512
    },
    {
      "completion_length": 392.89288330078125,
      "epoch": 0.8655606407322655,
      "grad_norm": 0.8444435596466064,
      "kl": 0.26265108585357666,
      "learning_rate": 2.7033426641203677e-07,
      "loss": 0.0105,
      "reward": 2.232625961303711,
      "reward_std": 0.6259433627128601,
      "rewards/reward_function": 2.232625961303711,
      "step": 1513
    },
    {
      "completion_length": 349.5357360839844,
      "epoch": 0.8661327231121282,
      "grad_norm": 0.21188592910766602,
      "kl": 0.1381276398897171,
      "learning_rate": 2.6808038725534086e-07,
      "loss": 0.0055,
      "reward": 2.219186544418335,
      "reward_std": 0.8160838484764099,
      "rewards/reward_function": 2.219186544418335,
      "step": 1514
    },
    {
      "completion_length": 344.96429443359375,
      "epoch": 0.8667048054919908,
      "grad_norm": 6.651116847991943,
      "kl": 0.460727721452713,
      "learning_rate": 2.6583541077743636e-07,
      "loss": 0.0184,
      "reward": 2.319561719894409,
      "reward_std": 0.8698969483375549,
      "rewards/reward_function": 2.319561719894409,
      "step": 1515
    },
    {
      "completion_length": 351.3214416503906,
      "epoch": 0.8672768878718535,
      "grad_norm": 0.22034892439842224,
      "kl": 0.1495547592639923,
      "learning_rate": 2.6359934593308293e-07,
      "loss": 0.006,
      "reward": 1.790543556213379,
      "reward_std": 1.1141293048858643,
      "rewards/reward_function": 1.790543556213379,
      "step": 1516
    },
    {
      "completion_length": 387.64288330078125,
      "epoch": 0.8678489702517163,
      "grad_norm": 20.7731876373291,
      "kl": 0.6095807552337646,
      "learning_rate": 2.613722016414944e-07,
      "loss": 0.0244,
      "reward": 2.1316750049591064,
      "reward_std": 0.9321194291114807,
      "rewards/reward_function": 2.1316750049591064,
      "step": 1517
    },
    {
      "completion_length": 313.14288330078125,
      "epoch": 0.868421052631579,
      "grad_norm": 1.046893835067749,
      "kl": 0.20343086123466492,
      "learning_rate": 2.5915398678630105e-07,
      "loss": 0.0081,
      "reward": 1.9101849794387817,
      "reward_std": 0.9447150230407715,
      "rewards/reward_function": 1.9101849794387817,
      "step": 1518
    },
    {
      "completion_length": 369.5000305175781,
      "epoch": 0.8689931350114416,
      "grad_norm": 0.23739811778068542,
      "kl": 0.14186573028564453,
      "learning_rate": 2.5694471021551674e-07,
      "loss": 0.0057,
      "reward": 1.365831971168518,
      "reward_std": 0.8128988146781921,
      "rewards/reward_function": 1.365831971168518,
      "step": 1519
    },
    {
      "completion_length": 323.96429443359375,
      "epoch": 0.8695652173913043,
      "grad_norm": 0.2234698385000229,
      "kl": 0.1394919604063034,
      "learning_rate": 2.547443807415009e-07,
      "loss": 0.0056,
      "reward": 1.4440855979919434,
      "reward_std": 0.7174498438835144,
      "rewards/reward_function": 1.4440855979919434,
      "step": 1520
    },
    {
      "completion_length": 350.96429443359375,
      "epoch": 0.870137299771167,
      "grad_norm": 0.5736585259437561,
      "kl": 0.226461261510849,
      "learning_rate": 2.5255300714092603e-07,
      "loss": 0.0091,
      "reward": 1.9235565662384033,
      "reward_std": 1.0191974639892578,
      "rewards/reward_function": 1.9235565662384033,
      "step": 1521
    },
    {
      "completion_length": 331.6071472167969,
      "epoch": 0.8707093821510298,
      "grad_norm": 1.8768917322158813,
      "kl": 0.2788146436214447,
      "learning_rate": 2.503705981547411e-07,
      "loss": 0.0112,
      "reward": 2.814617872238159,
      "reward_std": 0.7734389901161194,
      "rewards/reward_function": 2.814617872238159,
      "step": 1522
    },
    {
      "completion_length": 386.2857360839844,
      "epoch": 0.8712814645308925,
      "grad_norm": 2.4470365047454834,
      "kl": 0.3559344708919525,
      "learning_rate": 2.4819716248813713e-07,
      "loss": 0.0142,
      "reward": 1.3279390335083008,
      "reward_std": 0.7948994040489197,
      "rewards/reward_function": 1.3279390335083008,
      "step": 1523
    },
    {
      "completion_length": 329.6071472167969,
      "epoch": 0.8718535469107551,
      "grad_norm": 0.19637900590896606,
      "kl": 0.1395793855190277,
      "learning_rate": 2.4603270881051245e-07,
      "loss": 0.0056,
      "reward": 1.8809740543365479,
      "reward_std": 0.6630791425704956,
      "rewards/reward_function": 1.8809740543365479,
      "step": 1524
    },
    {
      "completion_length": 356.3214416503906,
      "epoch": 0.8724256292906178,
      "grad_norm": 0.20394673943519592,
      "kl": 0.13902704417705536,
      "learning_rate": 2.4387724575543697e-07,
      "loss": 0.0056,
      "reward": 2.634458065032959,
      "reward_std": 0.8395692706108093,
      "rewards/reward_function": 2.634458065032959,
      "step": 1525
    },
    {
      "completion_length": 336.3571472167969,
      "epoch": 0.8729977116704806,
      "grad_norm": 0.21804238855838776,
      "kl": 0.16305747628211975,
      "learning_rate": 2.417307819206202e-07,
      "loss": 0.0065,
      "reward": 2.2145113945007324,
      "reward_std": 1.291898250579834,
      "rewards/reward_function": 2.2145113945007324,
      "step": 1526
    },
    {
      "completion_length": 369.0714416503906,
      "epoch": 0.8735697940503433,
      "grad_norm": 187.90011596679688,
      "kl": 2.7105517387390137,
      "learning_rate": 2.3959332586787454e-07,
      "loss": 0.1084,
      "reward": 2.2487123012542725,
      "reward_std": 1.1734495162963867,
      "rewards/reward_function": 2.2487123012542725,
      "step": 1527
    },
    {
      "completion_length": 298.6071472167969,
      "epoch": 0.8741418764302059,
      "grad_norm": 0.3113255202770233,
      "kl": 0.1460842788219452,
      "learning_rate": 2.37464886123083e-07,
      "loss": 0.0058,
      "reward": 2.0520834922790527,
      "reward_std": 0.6409804821014404,
      "rewards/reward_function": 2.0520834922790527,
      "step": 1528
    },
    {
      "completion_length": 307.0714416503906,
      "epoch": 0.8747139588100686,
      "grad_norm": 0.1568872332572937,
      "kl": 0.10797043144702911,
      "learning_rate": 2.3534547117616407e-07,
      "loss": 0.0043,
      "reward": 2.6870102882385254,
      "reward_std": 0.9936245679855347,
      "rewards/reward_function": 2.6870102882385254,
      "step": 1529
    },
    {
      "completion_length": 317.71429443359375,
      "epoch": 0.8752860411899314,
      "grad_norm": 0.20696650445461273,
      "kl": 0.13790972530841827,
      "learning_rate": 2.3323508948103813e-07,
      "loss": 0.0055,
      "reward": 2.578908681869507,
      "reward_std": 0.606139600276947,
      "rewards/reward_function": 2.578908681869507,
      "step": 1530
    },
    {
      "completion_length": 398.7857360839844,
      "epoch": 0.8758581235697941,
      "grad_norm": 78.46916961669922,
      "kl": 1.8981940746307373,
      "learning_rate": 2.3113374945559292e-07,
      "loss": 0.0759,
      "reward": 1.9127963781356812,
      "reward_std": 1.0493125915527344,
      "rewards/reward_function": 1.9127963781356812,
      "step": 1531
    },
    {
      "completion_length": 333.5,
      "epoch": 0.8764302059496567,
      "grad_norm": 0.7175906300544739,
      "kl": 0.15401002764701843,
      "learning_rate": 2.2904145948165201e-07,
      "loss": 0.0062,
      "reward": 2.149339199066162,
      "reward_std": 0.5549109578132629,
      "rewards/reward_function": 2.149339199066162,
      "step": 1532
    },
    {
      "completion_length": 412.71429443359375,
      "epoch": 0.8770022883295194,
      "grad_norm": 1.5480432510375977,
      "kl": 0.3425666391849518,
      "learning_rate": 2.269582279049401e-07,
      "loss": 0.0137,
      "reward": 2.2542009353637695,
      "reward_std": 0.9375922679901123,
      "rewards/reward_function": 2.2542009353637695,
      "step": 1533
    },
    {
      "completion_length": 343.4285888671875,
      "epoch": 0.8775743707093822,
      "grad_norm": 0.8965217471122742,
      "kl": 0.23876316845417023,
      "learning_rate": 2.2488406303504766e-07,
      "loss": 0.0096,
      "reward": 2.602328300476074,
      "reward_std": 1.0387763977050781,
      "rewards/reward_function": 2.602328300476074,
      "step": 1534
    },
    {
      "completion_length": 327.89288330078125,
      "epoch": 0.8781464530892449,
      "grad_norm": 1.329047679901123,
      "kl": 0.25646695494651794,
      "learning_rate": 2.228189731454039e-07,
      "loss": 0.0103,
      "reward": 2.014115571975708,
      "reward_std": 0.8758649826049805,
      "rewards/reward_function": 2.014115571975708,
      "step": 1535
    },
    {
      "completion_length": 331.5357360839844,
      "epoch": 0.8787185354691075,
      "grad_norm": 1.0532811880111694,
      "kl": 0.22979716956615448,
      "learning_rate": 2.2076296647323525e-07,
      "loss": 0.0092,
      "reward": 1.6439591646194458,
      "reward_std": 0.7823508381843567,
      "rewards/reward_function": 1.6439591646194458,
      "step": 1536
    },
    {
      "completion_length": 316.7857360839844,
      "epoch": 0.8792906178489702,
      "grad_norm": 0.223018616437912,
      "kl": 0.1537347435951233,
      "learning_rate": 2.1871605121954076e-07,
      "loss": 0.0061,
      "reward": 1.65119206905365,
      "reward_std": 0.8255411386489868,
      "rewards/reward_function": 1.65119206905365,
      "step": 1537
    },
    {
      "completion_length": 284.8571472167969,
      "epoch": 0.879862700228833,
      "grad_norm": 0.17379704117774963,
      "kl": 0.13299015164375305,
      "learning_rate": 2.1667823554905264e-07,
      "loss": 0.0053,
      "reward": 2.032548189163208,
      "reward_std": 1.0193051099777222,
      "rewards/reward_function": 2.032548189163208,
      "step": 1538
    },
    {
      "completion_length": 338.25,
      "epoch": 0.8804347826086957,
      "grad_norm": 0.23066069185733795,
      "kl": 0.12587040662765503,
      "learning_rate": 2.1464952759020857e-07,
      "loss": 0.005,
      "reward": 2.3585596084594727,
      "reward_std": 1.1565574407577515,
      "rewards/reward_function": 2.3585596084594727,
      "step": 1539
    },
    {
      "completion_length": 313.7857360839844,
      "epoch": 0.8810068649885584,
      "grad_norm": 2.3874385356903076,
      "kl": 0.21264900267124176,
      "learning_rate": 2.1262993543511717e-07,
      "loss": 0.0085,
      "reward": 2.313469648361206,
      "reward_std": 1.216913104057312,
      "rewards/reward_function": 2.313469648361206,
      "step": 1540
    }
  ],
  "logging_steps": 1,
  "max_steps": 1748,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 140,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 28,
  "trial_name": null,
  "trial_params": null
}