{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5154639175257731,
  "eval_steps": 500,
  "global_step": 600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 143.8359375,
      "epoch": 0.000859106529209622,
      "grad_norm": 3.728695289308407,
      "kl": 0.0,
      "learning_rate": 9.995704467353951e-07,
      "loss": -0.0,
      "reward": 1.225318193435669,
      "reward_std": 0.5892010927200317,
      "rewards/format_reward_gen": 0.609375,
      "rewards/llm_reward": 0.6159431040287018,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.78125,
      "epoch": 0.001718213058419244,
      "grad_norm": 2.510726404973143,
      "kl": 0.0007686614990234375,
      "learning_rate": 9.991408934707903e-07,
      "loss": 0.0,
      "reward": 1.148492693901062,
      "reward_std": 0.5800909698009491,
      "rewards/format_reward_gen": 0.5390625,
      "rewards/llm_reward": 0.6094301640987396,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.578125,
      "epoch": 0.002577319587628866,
      "grad_norm": 5.237708678465742,
      "kl": 0.001880645751953125,
      "learning_rate": 9.987113402061855e-07,
      "loss": 0.0001,
      "reward": 1.2333476543426514,
      "reward_std": 0.5131954550743103,
      "rewards/format_reward_gen": 0.7109375,
      "rewards/llm_reward": 0.5224101841449738,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.265625,
      "epoch": 0.003436426116838488,
      "grad_norm": 4.430606101451457,
      "kl": 0.0068817138671875,
      "learning_rate": 9.982817869415807e-07,
      "loss": 0.0003,
      "reward": 1.3201585412025452,
      "reward_std": 0.5908865928649902,
      "rewards/format_reward_gen": 0.6640625,
      "rewards/llm_reward": 0.6560960710048676,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.34375,
      "epoch": 0.00429553264604811,
      "grad_norm": 2.966410650688808,
      "kl": 0.00145721435546875,
      "learning_rate": 9.97852233676976e-07,
      "loss": 0.0001,
      "reward": 1.5005085468292236,
      "reward_std": 0.446560800075531,
      "rewards/format_reward_gen": 0.8046875,
      "rewards/llm_reward": 0.6958210468292236,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.0234375,
      "epoch": 0.005154639175257732,
      "grad_norm": 18.73054467782952,
      "kl": 0.00228118896484375,
      "learning_rate": 9.97422680412371e-07,
      "loss": 0.0001,
      "reward": 1.379498541355133,
      "reward_std": 0.3884614408016205,
      "rewards/format_reward_gen": 0.796875,
      "rewards/llm_reward": 0.5826235115528107,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.1796875,
      "epoch": 0.006013745704467354,
      "grad_norm": 2.1996526299389556,
      "kl": 0.00290679931640625,
      "learning_rate": 9.969931271477663e-07,
      "loss": 0.0001,
      "reward": 1.4183476567268372,
      "reward_std": 0.46953827142715454,
      "rewards/format_reward_gen": 0.828125,
      "rewards/llm_reward": 0.5902226716279984,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 159.2265625,
      "epoch": 0.006872852233676976,
      "grad_norm": 1.5763458045998808,
      "kl": 0.00386810302734375,
      "learning_rate": 9.965635738831615e-07,
      "loss": 0.0002,
      "reward": 1.4628726840019226,
      "reward_std": 0.3434259593486786,
      "rewards/format_reward_gen": 0.859375,
      "rewards/llm_reward": 0.6034976243972778,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.8203125,
      "epoch": 0.007731958762886598,
      "grad_norm": 1.8925562854231095,
      "kl": 0.003387451171875,
      "learning_rate": 9.961340206185566e-07,
      "loss": 0.0001,
      "reward": 1.466668725013733,
      "reward_std": 0.3619833141565323,
      "rewards/format_reward_gen": 0.828125,
      "rewards/llm_reward": 0.6385437846183777,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.6640625,
      "epoch": 0.00859106529209622,
      "grad_norm": 2.30348378416368,
      "kl": 0.008270263671875,
      "learning_rate": 9.957044673539518e-07,
      "loss": 0.0003,
      "reward": 1.4460369348526,
      "reward_std": 0.2754833847284317,
      "rewards/format_reward_gen": 0.8828125,
      "rewards/llm_reward": 0.5632244646549225,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 152.671875,
      "epoch": 0.009450171821305841,
      "grad_norm": 4.5902996335202495,
      "kl": 0.006683349609375,
      "learning_rate": 9.95274914089347e-07,
      "loss": 0.0003,
      "reward": 1.6509097814559937,
      "reward_std": 0.25286635756492615,
      "rewards/format_reward_gen": 0.953125,
      "rewards/llm_reward": 0.6977846920490265,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.9140625,
      "epoch": 0.010309278350515464,
      "grad_norm": 4.341349059182411,
      "kl": 0.0069732666015625,
      "learning_rate": 9.948453608247422e-07,
      "loss": 0.0003,
      "reward": 1.63209068775177,
      "reward_std": 0.20565086603164673,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.6711532175540924,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.453125,
      "epoch": 0.011168384879725086,
      "grad_norm": 5.165117571002937,
      "kl": 0.0079345703125,
      "learning_rate": 9.944158075601374e-07,
      "loss": 0.0003,
      "reward": 1.7142389416694641,
      "reward_std": 0.219488687813282,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7376764714717865,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.25,
      "epoch": 0.012027491408934709,
      "grad_norm": 1.529830865974652,
      "kl": 0.017974853515625,
      "learning_rate": 9.939862542955326e-07,
      "loss": 0.0007,
      "reward": 1.6100916266441345,
      "reward_std": 0.15942473709583282,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.6491541266441345,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.9765625,
      "epoch": 0.01288659793814433,
      "grad_norm": 1.959441191085793,
      "kl": 0.01171875,
      "learning_rate": 9.935567010309278e-07,
      "loss": 0.0005,
      "reward": 1.6476787328720093,
      "reward_std": 0.21745194494724274,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.6867412328720093,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.8515625,
      "epoch": 0.013745704467353952,
      "grad_norm": 1.8012904974133117,
      "kl": 0.01544189453125,
      "learning_rate": 9.93127147766323e-07,
      "loss": 0.0006,
      "reward": 1.648992121219635,
      "reward_std": 0.22647833824157715,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.6802421808242798,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.6796875,
      "epoch": 0.014604810996563574,
      "grad_norm": 2.0371515238907456,
      "kl": 0.02178955078125,
      "learning_rate": 9.926975945017182e-07,
      "loss": 0.0009,
      "reward": 1.7388362884521484,
      "reward_std": 0.23847993463277817,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7544613182544708,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.171875,
      "epoch": 0.015463917525773196,
      "grad_norm": 2.29872644703533,
      "kl": 0.02154541015625,
      "learning_rate": 9.922680412371133e-07,
      "loss": 0.0009,
      "reward": 1.698328673839569,
      "reward_std": 0.21018245071172714,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7139536440372467,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.8203125,
      "epoch": 0.01632302405498282,
      "grad_norm": 1.642185067375333,
      "kl": 0.026123046875,
      "learning_rate": 9.918384879725085e-07,
      "loss": 0.001,
      "reward": 1.5711023807525635,
      "reward_std": 0.24714628607034683,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.5945399105548859,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.0859375,
      "epoch": 0.01718213058419244,
      "grad_norm": 1.8301018336586894,
      "kl": 0.04449462890625,
      "learning_rate": 9.914089347079037e-07,
      "loss": 0.0018,
      "reward": 1.7036264538764954,
      "reward_std": 0.2401985377073288,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7348764538764954,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 105.5859375,
      "epoch": 0.01804123711340206,
      "grad_norm": 2.4502501155105696,
      "kl": 0.02874755859375,
      "learning_rate": 9.90979381443299e-07,
      "loss": 0.0011,
      "reward": 1.5346225500106812,
      "reward_std": 0.18206386268138885,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.5502475053071976,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.828125,
      "epoch": 0.018900343642611683,
      "grad_norm": 4.579990678131132,
      "kl": 0.020751953125,
      "learning_rate": 9.90549828178694e-07,
      "loss": 0.0008,
      "reward": 1.5973477363586426,
      "reward_std": 0.21275992691516876,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.605160266160965,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.1171875,
      "epoch": 0.019759450171821305,
      "grad_norm": 1.9133262074405595,
      "kl": 0.025390625,
      "learning_rate": 9.901202749140893e-07,
      "loss": 0.001,
      "reward": 1.6978419423103333,
      "reward_std": 0.1712438389658928,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7056544423103333,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.3515625,
      "epoch": 0.020618556701030927,
      "grad_norm": 1.6795653310334677,
      "kl": 0.0228271484375,
      "learning_rate": 9.896907216494845e-07,
      "loss": 0.0009,
      "reward": 1.5728907585144043,
      "reward_std": 0.23567190766334534,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.5885157585144043,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.4609375,
      "epoch": 0.02147766323024055,
      "grad_norm": 1.8945583882920514,
      "kl": 0.02520751953125,
      "learning_rate": 9.892611683848797e-07,
      "loss": 0.001,
      "reward": 1.6186752319335938,
      "reward_std": 0.2236367017030716,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6343001425266266,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.0,
      "epoch": 0.022336769759450172,
      "grad_norm": 1.6471025834878852,
      "kl": 0.02325439453125,
      "learning_rate": 9.888316151202748e-07,
      "loss": 0.0009,
      "reward": 1.7836337089538574,
      "reward_std": 0.18849463760852814,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.791446179151535,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.0546875,
      "epoch": 0.023195876288659795,
      "grad_norm": 1.8224790908918138,
      "kl": 0.01788330078125,
      "learning_rate": 9.8840206185567e-07,
      "loss": 0.0007,
      "reward": 1.6910547018051147,
      "reward_std": 0.177225723862648,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6988671720027924,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.984375,
      "epoch": 0.024054982817869417,
      "grad_norm": 1.7847962406057651,
      "kl": 0.0181884765625,
      "learning_rate": 9.879725085910652e-07,
      "loss": 0.0007,
      "reward": 1.7889049053192139,
      "reward_std": 0.17536725103855133,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7967174351215363,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.0234375,
      "epoch": 0.02491408934707904,
      "grad_norm": 3.1801733940089547,
      "kl": 0.02703857421875,
      "learning_rate": 9.875429553264604e-07,
      "loss": 0.0011,
      "reward": 1.7557846307754517,
      "reward_std": 0.17591644823551178,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7714096307754517,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 109.265625,
      "epoch": 0.02577319587628866,
      "grad_norm": 3.160227437825882,
      "kl": 0.0194091796875,
      "learning_rate": 9.871134020618556e-07,
      "loss": 0.0008,
      "reward": 1.6949102878570557,
      "reward_std": 0.1834886148571968,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.710535317659378,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 112.265625,
      "epoch": 0.02663230240549828,
      "grad_norm": 1.4868195880486625,
      "kl": 0.0224609375,
      "learning_rate": 9.866838487972508e-07,
      "loss": 0.0009,
      "reward": 1.6780639290809631,
      "reward_std": 0.16649843007326126,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7015013694763184,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 105.984375,
      "epoch": 0.027491408934707903,
      "grad_norm": 22.472420255137777,
      "kl": 0.017974853515625,
      "learning_rate": 9.86254295532646e-07,
      "loss": 0.0007,
      "reward": 1.64022696018219,
      "reward_std": 0.19799882918596268,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6558520197868347,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.46875,
      "epoch": 0.028350515463917526,
      "grad_norm": 1.8825542488707594,
      "kl": 0.027587890625,
      "learning_rate": 9.858247422680412e-07,
      "loss": 0.0011,
      "reward": 1.686434805393219,
      "reward_std": 0.1929420754313469,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.694247305393219,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.7109375,
      "epoch": 0.029209621993127148,
      "grad_norm": 1.318783449188221,
      "kl": 0.011474609375,
      "learning_rate": 9.853951890034363e-07,
      "loss": 0.0005,
      "reward": 1.7323167324066162,
      "reward_std": 0.14837764203548431,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7323167026042938,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.7421875,
      "epoch": 0.03006872852233677,
      "grad_norm": 1.6922286395862565,
      "kl": 0.016632080078125,
      "learning_rate": 9.849656357388315e-07,
      "loss": 0.0007,
      "reward": 1.6450969576835632,
      "reward_std": 0.18919343501329422,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6450969874858856,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.8828125,
      "epoch": 0.030927835051546393,
      "grad_norm": 1.8376143817871085,
      "kl": 0.017822265625,
      "learning_rate": 9.845360824742267e-07,
      "loss": 0.0007,
      "reward": 1.6301739811897278,
      "reward_std": 0.20516958832740784,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6379865407943726,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 98.4921875,
      "epoch": 0.03178694158075601,
      "grad_norm": 2.8326207376956565,
      "kl": 0.009918212890625,
      "learning_rate": 9.84106529209622e-07,
      "loss": 0.0004,
      "reward": 1.709650695323944,
      "reward_std": 0.1534641981124878,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7096506357192993,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 101.6796875,
      "epoch": 0.03264604810996564,
      "grad_norm": 2.0347285959045536,
      "kl": 0.009307861328125,
      "learning_rate": 9.83676975945017e-07,
      "loss": 0.0004,
      "reward": 1.6739385724067688,
      "reward_std": 0.13153361529111862,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6739385724067688,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 112.5625,
      "epoch": 0.03350515463917526,
      "grad_norm": 1.9197526254739072,
      "kl": 0.01287841796875,
      "learning_rate": 9.832474226804123e-07,
      "loss": 0.0005,
      "reward": 1.7046000361442566,
      "reward_std": 0.19367831200361252,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7124125361442566,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 108.1328125,
      "epoch": 0.03436426116838488,
      "grad_norm": 3.1475346486430174,
      "kl": 0.011077880859375,
      "learning_rate": 9.828178694158075e-07,
      "loss": 0.0004,
      "reward": 1.714475929737091,
      "reward_std": 0.11251556500792503,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7144758999347687,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.6484375,
      "epoch": 0.0352233676975945,
      "grad_norm": 1.9386038530957954,
      "kl": 0.013763427734375,
      "learning_rate": 9.823883161512027e-07,
      "loss": 0.0006,
      "reward": 1.7007797360420227,
      "reward_std": 0.22316401451826096,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7085922360420227,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.2734375,
      "epoch": 0.03608247422680412,
      "grad_norm": 1.6026084575263893,
      "kl": 0.009185791015625,
      "learning_rate": 9.819587628865979e-07,
      "loss": 0.0004,
      "reward": 1.7506839632987976,
      "reward_std": 0.16962886601686478,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.75068399310112,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 109.265625,
      "epoch": 0.036941580756013746,
      "grad_norm": 1.4019849035616754,
      "kl": 0.015960693359375,
      "learning_rate": 9.81529209621993e-07,
      "loss": 0.0006,
      "reward": 1.6929492950439453,
      "reward_std": 0.15880103036761284,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6929492652416229,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 108.78125,
      "epoch": 0.037800687285223365,
      "grad_norm": 1.874751262052608,
      "kl": 0.01318359375,
      "learning_rate": 9.810996563573882e-07,
      "loss": 0.0005,
      "reward": 1.7645366191864014,
      "reward_std": 0.1825566589832306,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7723491191864014,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.1171875,
      "epoch": 0.03865979381443299,
      "grad_norm": 1.628943635799199,
      "kl": 0.012939453125,
      "learning_rate": 9.806701030927834e-07,
      "loss": 0.0005,
      "reward": 1.7157459259033203,
      "reward_std": 0.13008157908916473,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7157459259033203,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 112.3203125,
      "epoch": 0.03951890034364261,
      "grad_norm": 1.3333890922786833,
      "kl": 0.02374267578125,
      "learning_rate": 9.802405498281786e-07,
      "loss": 0.001,
      "reward": 1.7577217817306519,
      "reward_std": 0.1191495880484581,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7577218413352966,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.3359375,
      "epoch": 0.040378006872852236,
      "grad_norm": 2.7773278425143313,
      "kl": 0.014434814453125,
      "learning_rate": 9.798109965635738e-07,
      "loss": 0.0006,
      "reward": 1.6698015332221985,
      "reward_std": 0.18330103904008865,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6776140332221985,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.4296875,
      "epoch": 0.041237113402061855,
      "grad_norm": 3.1125459908194886,
      "kl": 0.0120849609375,
      "learning_rate": 9.79381443298969e-07,
      "loss": 0.0005,
      "reward": 1.7181047797203064,
      "reward_std": 0.2058340311050415,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.718104749917984,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.65625,
      "epoch": 0.04209621993127148,
      "grad_norm": 1.7917182356261732,
      "kl": 0.01312255859375,
      "learning_rate": 9.789518900343642e-07,
      "loss": 0.0005,
      "reward": 1.6312512159347534,
      "reward_std": 0.1998831108212471,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.631251186132431,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.765625,
      "epoch": 0.0429553264604811,
      "grad_norm": 2.715958501901883,
      "kl": 0.01019287109375,
      "learning_rate": 9.785223367697594e-07,
      "loss": 0.0004,
      "reward": 1.758800745010376,
      "reward_std": 0.17326835542917252,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.758800745010376,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.609375,
      "epoch": 0.04381443298969072,
      "grad_norm": 1.6233811186388696,
      "kl": 0.0115966796875,
      "learning_rate": 9.780927835051545e-07,
      "loss": 0.0005,
      "reward": 1.693307340145111,
      "reward_std": 0.16266920417547226,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6933073401451111,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 108.203125,
      "epoch": 0.044673539518900345,
      "grad_norm": 1.655879971578571,
      "kl": 0.024871826171875,
      "learning_rate": 9.776632302405497e-07,
      "loss": 0.001,
      "reward": 1.6892904043197632,
      "reward_std": 0.17997007817029953,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7049154043197632,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.65625,
      "epoch": 0.04553264604810996,
      "grad_norm": 1.6681145967268827,
      "kl": 0.010528564453125,
      "learning_rate": 9.77233676975945e-07,
      "loss": 0.0004,
      "reward": 1.7307260036468506,
      "reward_std": 0.18761465698480606,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7307261228561401,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.5078125,
      "epoch": 0.04639175257731959,
      "grad_norm": 1.9783878004259334,
      "kl": 0.00994873046875,
      "learning_rate": 9.768041237113401e-07,
      "loss": 0.0004,
      "reward": 1.6929954290390015,
      "reward_std": 0.15189384669065475,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6929953396320343,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.3828125,
      "epoch": 0.04725085910652921,
      "grad_norm": 1.6703845988223984,
      "kl": 0.014129638671875,
      "learning_rate": 9.763745704467353e-07,
      "loss": 0.0006,
      "reward": 1.616851270198822,
      "reward_std": 0.18548668175935745,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6168512403964996,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.6015625,
      "epoch": 0.048109965635738834,
      "grad_norm": 1.5488291217502541,
      "kl": 0.013580322265625,
      "learning_rate": 9.759450171821305e-07,
      "loss": 0.0005,
      "reward": 1.7046599388122559,
      "reward_std": 0.22010766714811325,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7124724388122559,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.9453125,
      "epoch": 0.04896907216494845,
      "grad_norm": 1.4634966134639982,
      "kl": 0.0120849609375,
      "learning_rate": 9.755154639175257e-07,
      "loss": 0.0005,
      "reward": 1.6978161334991455,
      "reward_std": 0.17123103514313698,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7134410738945007,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.203125,
      "epoch": 0.04982817869415808,
      "grad_norm": 2.773139620924608,
      "kl": 0.01214599609375,
      "learning_rate": 9.750859106529209e-07,
      "loss": 0.0005,
      "reward": 1.7033132314682007,
      "reward_std": 0.17858785390853882,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7033132314682007,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.6484375,
      "epoch": 0.0506872852233677,
      "grad_norm": 2.112316846879123,
      "kl": 0.015411376953125,
      "learning_rate": 9.74656357388316e-07,
      "loss": 0.0006,
      "reward": 1.7434183359146118,
      "reward_std": 0.1504114419221878,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7434183359146118,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.8203125,
      "epoch": 0.05154639175257732,
      "grad_norm": 4.733899218910252,
      "kl": 0.0123291015625,
      "learning_rate": 9.742268041237112e-07,
      "loss": 0.0005,
      "reward": 1.7861030101776123,
      "reward_std": 0.21234874427318573,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7939155101776123,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.7734375,
      "epoch": 0.05240549828178694,
      "grad_norm": 2.410724757522067,
      "kl": 0.01287841796875,
      "learning_rate": 9.737972508591064e-07,
      "loss": 0.0005,
      "reward": 1.7895718216896057,
      "reward_std": 0.1751849427819252,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7973843216896057,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.0,
      "epoch": 0.05326460481099656,
      "grad_norm": 1.3683631722816978,
      "kl": 0.0125732421875,
      "learning_rate": 9.733676975945016e-07,
      "loss": 0.0005,
      "reward": 1.6282992959022522,
      "reward_std": 0.1704944670200348,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6439243257045746,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.2421875,
      "epoch": 0.05412371134020619,
      "grad_norm": 1.6450659007412214,
      "kl": 0.01019287109375,
      "learning_rate": 9.729381443298968e-07,
      "loss": 0.0004,
      "reward": 1.6509077548980713,
      "reward_std": 0.17651135474443436,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6509077548980713,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.171875,
      "epoch": 0.054982817869415807,
      "grad_norm": 1.6294625649527108,
      "kl": 0.01202392578125,
      "learning_rate": 9.72508591065292e-07,
      "loss": 0.0005,
      "reward": 1.7486688494682312,
      "reward_std": 0.2049519494175911,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7564813494682312,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 112.4375,
      "epoch": 0.05584192439862543,
      "grad_norm": 1.3207489104187833,
      "kl": 0.014923095703125,
      "learning_rate": 9.720790378006872e-07,
      "loss": 0.0006,
      "reward": 1.686493158340454,
      "reward_std": 0.1424517035484314,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7021180987358093,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.4453125,
      "epoch": 0.05670103092783505,
      "grad_norm": 1.8194250939249632,
      "kl": 0.01617431640625,
      "learning_rate": 9.716494845360824e-07,
      "loss": 0.0006,
      "reward": 1.7264857292175293,
      "reward_std": 0.16944430023431778,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7342982888221741,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.1953125,
      "epoch": 0.05756013745704467,
      "grad_norm": 1.8454925104090358,
      "kl": 0.013275146484375,
      "learning_rate": 9.712199312714776e-07,
      "loss": 0.0005,
      "reward": 1.7975256443023682,
      "reward_std": 0.18923871964216232,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7975256443023682,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.2265625,
      "epoch": 0.058419243986254296,
      "grad_norm": 1.8448295930855816,
      "kl": 0.012298583984375,
      "learning_rate": 9.707903780068727e-07,
      "loss": 0.0005,
      "reward": 1.7086012363433838,
      "reward_std": 0.14006467163562775,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7086012661457062,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 111.421875,
      "epoch": 0.059278350515463915,
      "grad_norm": 1.5791228430586928,
      "kl": 0.0155029296875,
      "learning_rate": 9.70360824742268e-07,
      "loss": 0.0006,
      "reward": 1.799054503440857,
      "reward_std": 0.14379648119211197,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8068670034408569,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.359375,
      "epoch": 0.06013745704467354,
      "grad_norm": 1.3943612865559436,
      "kl": 0.010162353515625,
      "learning_rate": 9.699312714776631e-07,
      "loss": 0.0004,
      "reward": 1.8229413032531738,
      "reward_std": 0.15740595757961273,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8229413330554962,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.6953125,
      "epoch": 0.06099656357388316,
      "grad_norm": 2.8156538308984262,
      "kl": 0.009796142578125,
      "learning_rate": 9.695017182130583e-07,
      "loss": 0.0004,
      "reward": 1.7003414630889893,
      "reward_std": 0.19474071264266968,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7159664928913116,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.2265625,
      "epoch": 0.061855670103092786,
      "grad_norm": 1.866123797516234,
      "kl": 0.0084228515625,
      "learning_rate": 9.690721649484535e-07,
      "loss": 0.0003,
      "reward": 1.7515292763710022,
      "reward_std": 0.1583714485168457,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7515292763710022,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.3671875,
      "epoch": 0.0627147766323024,
      "grad_norm": 1.394074930738141,
      "kl": 0.011199951171875,
      "learning_rate": 9.686426116838487e-07,
      "loss": 0.0004,
      "reward": 1.679770588874817,
      "reward_std": 0.21282349526882172,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7110206186771393,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.3984375,
      "epoch": 0.06357388316151202,
      "grad_norm": 4.5617894751936285,
      "kl": 0.019927978515625,
      "learning_rate": 9.682130584192439e-07,
      "loss": 0.0008,
      "reward": 1.6840202808380127,
      "reward_std": 0.18173471838235855,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6918327808380127,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.40625,
      "epoch": 0.06443298969072164,
      "grad_norm": 20.644470378771178,
      "kl": 0.0096435546875,
      "learning_rate": 9.67783505154639e-07,
      "loss": 0.0004,
      "reward": 1.6653663516044617,
      "reward_std": 0.15374305844306946,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6653663218021393,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.75,
      "epoch": 0.06529209621993128,
      "grad_norm": 1.3520802500043023,
      "kl": 0.011871337890625,
      "learning_rate": 9.673539518900342e-07,
      "loss": 0.0005,
      "reward": 1.7905267477035522,
      "reward_std": 0.11402217298746109,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.790526807308197,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0703125,
      "epoch": 0.0661512027491409,
      "grad_norm": 2.1828273386945662,
      "kl": 0.008880615234375,
      "learning_rate": 9.669243986254294e-07,
      "loss": 0.0004,
      "reward": 1.6396149396896362,
      "reward_std": 0.22704153507947922,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6552398800849915,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.5625,
      "epoch": 0.06701030927835051,
      "grad_norm": 1.9960711680675047,
      "kl": 0.01043701171875,
      "learning_rate": 9.664948453608246e-07,
      "loss": 0.0004,
      "reward": 1.717953085899353,
      "reward_std": 0.1279629021883011,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7179530560970306,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.375,
      "epoch": 0.06786941580756013,
      "grad_norm": 1.573275119243184,
      "kl": 0.009918212890625,
      "learning_rate": 9.660652920962198e-07,
      "loss": 0.0004,
      "reward": 1.7117525339126587,
      "reward_std": 0.17472369223833084,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7117525339126587,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.6953125,
      "epoch": 0.06872852233676977,
      "grad_norm": 5.858171715596188,
      "kl": 0.0074462890625,
      "learning_rate": 9.65635738831615e-07,
      "loss": 0.0003,
      "reward": 1.7692703604698181,
      "reward_std": 0.18383784592151642,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7692703902721405,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.8203125,
      "epoch": 0.06958762886597938,
      "grad_norm": 2.153346346698614,
      "kl": 0.0079345703125,
      "learning_rate": 9.652061855670102e-07,
      "loss": 0.0003,
      "reward": 1.7922507524490356,
      "reward_std": 0.16792288422584534,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7922507226467133,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.2890625,
      "epoch": 0.070446735395189,
      "grad_norm": 1.5730677509411617,
      "kl": 0.009002685546875,
      "learning_rate": 9.647766323024054e-07,
      "loss": 0.0004,
      "reward": 1.7323782444000244,
      "reward_std": 0.17609868198633194,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7480032444000244,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.578125,
      "epoch": 0.07130584192439862,
      "grad_norm": 1.374124074188527,
      "kl": 0.011138916015625,
      "learning_rate": 9.643470790378006e-07,
      "loss": 0.0004,
      "reward": 1.6585241556167603,
      "reward_std": 0.19784274697303772,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.6819616854190826,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.9296875,
      "epoch": 0.07216494845360824,
      "grad_norm": 2.383137509998003,
      "kl": 0.01141357421875,
      "learning_rate": 9.639175257731957e-07,
      "loss": 0.0005,
      "reward": 1.6670758724212646,
      "reward_std": 0.19278159737586975,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6670758724212646,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.578125,
      "epoch": 0.07302405498281787,
      "grad_norm": 1.7362778804071193,
      "kl": 0.011749267578125,
      "learning_rate": 9.63487972508591e-07,
      "loss": 0.0005,
      "reward": 1.7598541378974915,
      "reward_std": 0.1449267938733101,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7754791080951691,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 164.609375,
      "epoch": 0.07388316151202749,
      "grad_norm": 2.8650428143418325,
      "kl": 0.012176513671875,
      "learning_rate": 9.630584192439863e-07,
      "loss": 0.0005,
      "reward": 1.6261486411094666,
      "reward_std": 0.29106324911117554,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.6573987007141113,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.90625,
      "epoch": 0.07474226804123711,
      "grad_norm": 1.9633221241492278,
      "kl": 0.0074920654296875,
      "learning_rate": 9.626288659793815e-07,
      "loss": 0.0003,
      "reward": 1.7269136905670166,
      "reward_std": 0.1622983068227768,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7347261309623718,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.75,
      "epoch": 0.07560137457044673,
      "grad_norm": 2.0681449704464954,
      "kl": 0.0077056884765625,
      "learning_rate": 9.621993127147767e-07,
      "loss": 0.0003,
      "reward": 1.7325759530067444,
      "reward_std": 0.14999449253082275,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7325759530067444,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.3984375,
      "epoch": 0.07646048109965636,
      "grad_norm": 1.1844639755913142,
      "kl": 0.007354736328125,
      "learning_rate": 9.61769759450172e-07,
      "loss": 0.0003,
      "reward": 1.7702937126159668,
      "reward_std": 0.12665041163563728,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7702936828136444,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.25,
      "epoch": 0.07731958762886598,
      "grad_norm": 1.3413729169105462,
      "kl": 0.0083465576171875,
      "learning_rate": 9.61340206185567e-07,
      "loss": 0.0003,
      "reward": 1.6900493502616882,
      "reward_std": 0.08954241871833801,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6978618502616882,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.421875,
      "epoch": 0.0781786941580756,
      "grad_norm": 1.9584992131927317,
      "kl": 0.015350341796875,
      "learning_rate": 9.609106529209623e-07,
      "loss": 0.0006,
      "reward": 1.6389594078063965,
      "reward_std": 0.231736421585083,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6545844078063965,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.6796875,
      "epoch": 0.07903780068728522,
      "grad_norm": 1.4808391695134289,
      "kl": 0.015228271484375,
      "learning_rate": 9.604810996563575e-07,
      "loss": 0.0006,
      "reward": 1.7157284617424011,
      "reward_std": 0.14664900302886963,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7313534915447235,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.8515625,
      "epoch": 0.07989690721649484,
      "grad_norm": 1.3560060082947887,
      "kl": 0.011688232421875,
      "learning_rate": 9.600515463917527e-07,
      "loss": 0.0005,
      "reward": 1.6663364171981812,
      "reward_std": 0.17353109270334244,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.6897739470005035,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.78125,
      "epoch": 0.08075601374570447,
      "grad_norm": 1.4417733286684913,
      "kl": 0.009521484375,
      "learning_rate": 9.596219931271478e-07,
      "loss": 0.0004,
      "reward": 1.695834457874298,
      "reward_std": 0.15896284580230713,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7270845174789429,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.7421875,
      "epoch": 0.08161512027491409,
      "grad_norm": 1.81422275990836,
      "kl": 0.008056640625,
      "learning_rate": 9.59192439862543e-07,
      "loss": 0.0003,
      "reward": 1.7134593725204468,
      "reward_std": 0.16323717311024666,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7134594321250916,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.21875,
      "epoch": 0.08247422680412371,
      "grad_norm": 1.2278195202315274,
      "kl": 0.015380859375,
      "learning_rate": 9.587628865979382e-07,
      "loss": 0.0006,
      "reward": 1.833047330379486,
      "reward_std": 0.15521731600165367,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8486724197864532,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.7890625,
      "epoch": 0.08333333333333333,
      "grad_norm": 1.7371338217464498,
      "kl": 0.012603759765625,
      "learning_rate": 9.583333333333334e-07,
      "loss": 0.0005,
      "reward": 1.6824183464050293,
      "reward_std": 0.1946360394358635,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6824184060096741,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.09375,
      "epoch": 0.08419243986254296,
      "grad_norm": 2.9141007581916285,
      "kl": 0.01483154296875,
      "learning_rate": 9.579037800687286e-07,
      "loss": 0.0006,
      "reward": 1.7687426805496216,
      "reward_std": 0.16979511827230453,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7687426805496216,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.703125,
      "epoch": 0.08505154639175258,
      "grad_norm": 3.157985367093318,
      "kl": 0.012939453125,
      "learning_rate": 9.574742268041238e-07,
      "loss": 0.0005,
      "reward": 1.8471919298171997,
      "reward_std": 0.08835843577980995,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8471919894218445,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.96875,
      "epoch": 0.0859106529209622,
      "grad_norm": 1.791601113718697,
      "kl": 0.01171875,
      "learning_rate": 9.57044673539519e-07,
      "loss": 0.0005,
      "reward": 1.7132195234298706,
      "reward_std": 0.1518014371395111,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7132195234298706,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.640625,
      "epoch": 0.08676975945017182,
      "grad_norm": 1.6750678433197985,
      "kl": 0.01409912109375,
      "learning_rate": 9.566151202749142e-07,
      "loss": 0.0006,
      "reward": 1.6660110354423523,
      "reward_std": 0.16518359631299973,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6660110354423523,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.453125,
      "epoch": 0.08762886597938144,
      "grad_norm": 2.100280147210653,
      "kl": 0.014434814453125,
      "learning_rate": 9.561855670103093e-07,
      "loss": 0.0006,
      "reward": 1.7369045615196228,
      "reward_std": 0.11925114691257477,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.736904501914978,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.5703125,
      "epoch": 0.08848797250859107,
      "grad_norm": 2.48977370571022,
      "kl": 0.012603759765625,
      "learning_rate": 9.557560137457045e-07,
      "loss": 0.0005,
      "reward": 1.7730196118354797,
      "reward_std": 0.18081708252429962,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7808321118354797,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.546875,
      "epoch": 0.08934707903780069,
      "grad_norm": 2.083788782587846,
      "kl": 0.0164794921875,
      "learning_rate": 9.553264604810997e-07,
      "loss": 0.0007,
      "reward": 1.668508231639862,
      "reward_std": 0.1277586668729782,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6763207614421844,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.1953125,
      "epoch": 0.09020618556701031,
      "grad_norm": 3.157040349398891,
      "kl": 0.015838623046875,
      "learning_rate": 9.54896907216495e-07,
      "loss": 0.0006,
      "reward": 1.792285680770874,
      "reward_std": 0.1403278186917305,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.800098180770874,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.53125,
      "epoch": 0.09106529209621993,
      "grad_norm": 1.9043694578355221,
      "kl": 0.01544189453125,
      "learning_rate": 9.5446735395189e-07,
      "loss": 0.0006,
      "reward": 1.7426919341087341,
      "reward_std": 0.17454702407121658,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7426919043064117,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.0234375,
      "epoch": 0.09192439862542956,
      "grad_norm": 1.1762483491043985,
      "kl": 0.013885498046875,
      "learning_rate": 9.540378006872853e-07,
      "loss": 0.0006,
      "reward": 1.76499605178833,
      "reward_std": 0.12454288452863693,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7649960517883301,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.375,
      "epoch": 0.09278350515463918,
      "grad_norm": 2.4203700264289822,
      "kl": 0.014434814453125,
      "learning_rate": 9.536082474226805e-07,
      "loss": 0.0006,
      "reward": 1.7665973901748657,
      "reward_std": 0.16096310317516327,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7900348901748657,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.6796875,
      "epoch": 0.0936426116838488,
      "grad_norm": 1.1516404581818234,
      "kl": 0.013824462890625,
      "learning_rate": 9.531786941580757e-07,
      "loss": 0.0006,
      "reward": 1.7484302520751953,
      "reward_std": 0.11297959834337234,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7562427520751953,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.765625,
      "epoch": 0.09450171821305842,
      "grad_norm": 1.8664322686617876,
      "kl": 0.013092041015625,
      "learning_rate": 9.527491408934707e-07,
      "loss": 0.0005,
      "reward": 1.7427197694778442,
      "reward_std": 0.1969207152724266,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7505322694778442,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.4140625,
      "epoch": 0.09536082474226804,
      "grad_norm": 2.8553637446748055,
      "kl": 0.01678466796875,
      "learning_rate": 9.523195876288659e-07,
      "loss": 0.0007,
      "reward": 1.6789370775222778,
      "reward_std": 0.14484084397554398,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6867496371269226,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.9765625,
      "epoch": 0.09621993127147767,
      "grad_norm": 2.8547235701713527,
      "kl": 0.01629638671875,
      "learning_rate": 9.518900343642611e-07,
      "loss": 0.0007,
      "reward": 1.7494203448295593,
      "reward_std": 0.17836876958608627,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7572328150272369,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.1328125,
      "epoch": 0.09707903780068729,
      "grad_norm": 1.1274603410680148,
      "kl": 0.014923095703125,
      "learning_rate": 9.514604810996563e-07,
      "loss": 0.0006,
      "reward": 1.775137186050415,
      "reward_std": 0.12154169753193855,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7829497158527374,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.5625,
      "epoch": 0.0979381443298969,
      "grad_norm": 2.4835835333144347,
      "kl": 0.025482177734375,
      "learning_rate": 9.510309278350515e-07,
      "loss": 0.001,
      "reward": 1.6830406785011292,
      "reward_std": 0.07851138710975647,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6986656486988068,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.2578125,
      "epoch": 0.09879725085910653,
      "grad_norm": 1.457424888302412,
      "kl": 0.01715087890625,
      "learning_rate": 9.506013745704467e-07,
      "loss": 0.0007,
      "reward": 1.8288711309432983,
      "reward_std": 0.1552474945783615,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8366836309432983,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.15625,
      "epoch": 0.09965635738831616,
      "grad_norm": 5.261633631939683,
      "kl": 0.012969970703125,
      "learning_rate": 9.501718213058419e-07,
      "loss": 0.0005,
      "reward": 1.7097431421279907,
      "reward_std": 0.1742759346961975,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7097431421279907,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.25,
      "epoch": 0.10051546391752578,
      "grad_norm": 2.0866973884427784,
      "kl": 0.01885986328125,
      "learning_rate": 9.497422680412371e-07,
      "loss": 0.0008,
      "reward": 1.7714307308197021,
      "reward_std": 0.2078239470720291,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7714307308197021,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.0546875,
      "epoch": 0.1013745704467354,
      "grad_norm": 1.9401421255992048,
      "kl": 0.0369873046875,
      "learning_rate": 9.493127147766322e-07,
      "loss": 0.0015,
      "reward": 1.7130677700042725,
      "reward_std": 0.19828316569328308,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7208802998065948,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.390625,
      "epoch": 0.10223367697594501,
      "grad_norm": 2.5355319382822827,
      "kl": 0.0181884765625,
      "learning_rate": 9.488831615120274e-07,
      "loss": 0.0007,
      "reward": 1.737857460975647,
      "reward_std": 0.1541828289628029,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7378574907779694,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.7265625,
      "epoch": 0.10309278350515463,
      "grad_norm": 1.4772557885572284,
      "kl": 0.02142333984375,
      "learning_rate": 9.484536082474226e-07,
      "loss": 0.0009,
      "reward": 1.7606690526008606,
      "reward_std": 0.18398111313581467,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7606690227985382,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.7578125,
      "epoch": 0.10395189003436427,
      "grad_norm": 2.4929887339743244,
      "kl": 0.01837158203125,
      "learning_rate": 9.480240549828178e-07,
      "loss": 0.0007,
      "reward": 1.727208137512207,
      "reward_std": 0.13875007256865501,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.727208137512207,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.640625,
      "epoch": 0.10481099656357389,
      "grad_norm": 3.0006792162829257,
      "kl": 0.02630615234375,
      "learning_rate": 9.47594501718213e-07,
      "loss": 0.001,
      "reward": 1.8001123070716858,
      "reward_std": 0.20054399222135544,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8157372772693634,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 113.046875,
      "epoch": 0.1056701030927835,
      "grad_norm": 2.6748232943569827,
      "kl": 0.0206298828125,
      "learning_rate": 9.471649484536082e-07,
      "loss": 0.0008,
      "reward": 1.7969177961349487,
      "reward_std": 0.10830671712756157,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8047303557395935,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.953125,
      "epoch": 0.10652920962199312,
      "grad_norm": 1.4841628651180314,
      "kl": 0.024658203125,
      "learning_rate": 9.467353951890034e-07,
      "loss": 0.001,
      "reward": 1.7056427001953125,
      "reward_std": 0.1875780150294304,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7368926405906677,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.203125,
      "epoch": 0.10738831615120274,
      "grad_norm": 1.874212030480472,
      "kl": 0.023193359375,
      "learning_rate": 9.463058419243986e-07,
      "loss": 0.0009,
      "reward": 1.6158097386360168,
      "reward_std": 0.1327304244041443,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6236222088336945,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.75,
      "epoch": 0.10824742268041238,
      "grad_norm": 1.0408020027656946,
      "kl": 0.014923095703125,
      "learning_rate": 9.458762886597938e-07,
      "loss": 0.0006,
      "reward": 1.7203240394592285,
      "reward_std": 0.1105212289839983,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7281366288661957,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.359375,
      "epoch": 0.109106529209622,
      "grad_norm": 1.2726802820456735,
      "kl": 0.0220947265625,
      "learning_rate": 9.454467353951889e-07,
      "loss": 0.0009,
      "reward": 1.6998094320297241,
      "reward_std": 0.16664542257785797,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7076218724250793,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.765625,
      "epoch": 0.10996563573883161,
      "grad_norm": 1.6961988401912567,
      "kl": 0.015594482421875,
      "learning_rate": 9.450171821305841e-07,
      "loss": 0.0006,
      "reward": 1.6439663171768188,
      "reward_std": 0.16561322659254074,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6517787873744965,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.390625,
      "epoch": 0.11082474226804123,
      "grad_norm": 2.510071105917193,
      "kl": 0.015350341796875,
      "learning_rate": 9.445876288659793e-07,
      "loss": 0.0006,
      "reward": 1.66585111618042,
      "reward_std": 0.1521969810128212,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6736635565757751,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.484375,
      "epoch": 0.11168384879725086,
      "grad_norm": 5.638902477820217,
      "kl": 0.01513671875,
      "learning_rate": 9.441580756013745e-07,
      "loss": 0.0006,
      "reward": 1.7457653284072876,
      "reward_std": 0.15448395907878876,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7457653880119324,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.1484375,
      "epoch": 0.11254295532646048,
      "grad_norm": 1.513851686329793,
      "kl": 0.015289306640625,
      "learning_rate": 9.437285223367697e-07,
      "loss": 0.0006,
      "reward": 1.7635972499847412,
      "reward_std": 0.17401929199695587,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7714097797870636,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.3359375,
      "epoch": 0.1134020618556701,
      "grad_norm": 1.322125994104397,
      "kl": 0.012359619140625,
      "learning_rate": 9.432989690721649e-07,
      "loss": 0.0005,
      "reward": 1.823841154575348,
      "reward_std": 0.14871416985988617,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8238411545753479,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0,
      "epoch": 0.11426116838487972,
      "grad_norm": 1.355698173704868,
      "kl": 0.023406982421875,
      "learning_rate": 9.428694158075601e-07,
      "loss": 0.0009,
      "reward": 1.766915202140808,
      "reward_std": 0.14532212167978287,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7825402021408081,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.4140625,
      "epoch": 0.11512027491408934,
      "grad_norm": 1.4739462448440241,
      "kl": 0.0181884765625,
      "learning_rate": 9.424398625429553e-07,
      "loss": 0.0007,
      "reward": 1.7099063992500305,
      "reward_std": 0.1308385580778122,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7099063992500305,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.9296875,
      "epoch": 0.11597938144329897,
      "grad_norm": 1.3274707295914516,
      "kl": 0.014129638671875,
      "learning_rate": 9.420103092783504e-07,
      "loss": 0.0006,
      "reward": 1.7741219997406006,
      "reward_std": 0.16581101715564728,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7819344997406006,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.140625,
      "epoch": 0.11683848797250859,
      "grad_norm": 1.5843650427627012,
      "kl": 0.01092529296875,
      "learning_rate": 9.415807560137456e-07,
      "loss": 0.0004,
      "reward": 1.6576457023620605,
      "reward_std": 0.1768977865576744,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6654582023620605,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.8984375,
      "epoch": 0.11769759450171821,
      "grad_norm": 1.670051625687803,
      "kl": 0.012542724609375,
      "learning_rate": 9.411512027491408e-07,
      "loss": 0.0005,
      "reward": 1.7060455083847046,
      "reward_std": 0.14821957796812057,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.706045538187027,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.953125,
      "epoch": 0.11855670103092783,
      "grad_norm": 1.64228564707855,
      "kl": 0.0146484375,
      "learning_rate": 9.40721649484536e-07,
      "loss": 0.0006,
      "reward": 1.698437750339508,
      "reward_std": 0.1564936824142933,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6984376907348633,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.15625,
      "epoch": 0.11941580756013746,
      "grad_norm": 1.376863304210578,
      "kl": 0.010498046875,
      "learning_rate": 9.402920962199312e-07,
      "loss": 0.0004,
      "reward": 1.8016321063041687,
      "reward_std": 0.15034572780132294,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8172571659088135,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.8359375,
      "epoch": 0.12027491408934708,
      "grad_norm": 6.099834295001363,
      "kl": 0.013580322265625,
      "learning_rate": 9.398625429553264e-07,
      "loss": 0.0005,
      "reward": 1.6821348667144775,
      "reward_std": 0.13406795635819435,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6821348965167999,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.953125,
      "epoch": 0.1211340206185567,
      "grad_norm": 1.6651173493841875,
      "kl": 0.01617431640625,
      "learning_rate": 9.394329896907216e-07,
      "loss": 0.0006,
      "reward": 1.7098830938339233,
      "reward_std": 0.17125768959522247,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7333205342292786,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.2265625,
      "epoch": 0.12199312714776632,
      "grad_norm": 2.528179883736634,
      "kl": 0.013946533203125,
      "learning_rate": 9.390034364261168e-07,
      "loss": 0.0006,
      "reward": 1.7531933784484863,
      "reward_std": 0.15759651362895966,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7610058188438416,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.453125,
      "epoch": 0.12285223367697594,
      "grad_norm": 3.1296650163459088,
      "kl": 0.01806640625,
      "learning_rate": 9.38573883161512e-07,
      "loss": 0.0007,
      "reward": 1.8122310042381287,
      "reward_std": 0.17129109799861908,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8122310042381287,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.421875,
      "epoch": 0.12371134020618557,
      "grad_norm": 2.002669092661699,
      "kl": 0.01312255859375,
      "learning_rate": 9.381443298969071e-07,
      "loss": 0.0005,
      "reward": 1.7977581024169922,
      "reward_std": 0.145246010273695,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8055705428123474,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.7421875,
      "epoch": 0.12457044673539519,
      "grad_norm": 1.2100201264770116,
      "kl": 0.0140380859375,
      "learning_rate": 9.377147766323023e-07,
      "loss": 0.0006,
      "reward": 1.829803466796875,
      "reward_std": 0.1415480300784111,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.837615966796875,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.3203125,
      "epoch": 0.1254295532646048,
      "grad_norm": 2.180495745625869,
      "kl": 0.01171875,
      "learning_rate": 9.372852233676975e-07,
      "loss": 0.0005,
      "reward": 1.7289317846298218,
      "reward_std": 0.18080250918865204,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7445567846298218,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.84375,
      "epoch": 0.12628865979381443,
      "grad_norm": 2.9127570085332892,
      "kl": 0.01275634765625,
      "learning_rate": 9.368556701030927e-07,
      "loss": 0.0005,
      "reward": 1.8339774012565613,
      "reward_std": 0.09063693135976791,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8339774012565613,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.8125,
      "epoch": 0.12714776632302405,
      "grad_norm": 1.5988443873795521,
      "kl": 0.015777587890625,
      "learning_rate": 9.364261168384879e-07,
      "loss": 0.0006,
      "reward": 1.7010595202445984,
      "reward_std": 0.21928820759058,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7010595202445984,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.015625,
      "epoch": 0.12800687285223367,
      "grad_norm": 2.3035156258187697,
      "kl": 0.016021728515625,
      "learning_rate": 9.359965635738831e-07,
      "loss": 0.0006,
      "reward": 1.7444458603858948,
      "reward_std": 0.21774866431951523,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7444458901882172,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.453125,
      "epoch": 0.12886597938144329,
      "grad_norm": 1.335338188075339,
      "kl": 0.014312744140625,
      "learning_rate": 9.355670103092783e-07,
      "loss": 0.0006,
      "reward": 1.8212101459503174,
      "reward_std": 0.08616838604211807,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8212102353572845,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.2734375,
      "epoch": 0.12972508591065293,
      "grad_norm": 1.4631690042303354,
      "kl": 0.01123046875,
      "learning_rate": 9.351374570446736e-07,
      "loss": 0.0004,
      "reward": 1.7321330308914185,
      "reward_std": 0.1448444351553917,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7321330606937408,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.3515625,
      "epoch": 0.13058419243986255,
      "grad_norm": 0.9273287057028194,
      "kl": 0.009765625,
      "learning_rate": 9.347079037800687e-07,
      "loss": 0.0004,
      "reward": 1.7624321579933167,
      "reward_std": 0.09383735246956348,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7624321281909943,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.8515625,
      "epoch": 0.13144329896907217,
      "grad_norm": 1.47382353081262,
      "kl": 0.0152587890625,
      "learning_rate": 9.342783505154639e-07,
      "loss": 0.0006,
      "reward": 1.6556521654129028,
      "reward_std": 0.15647713094949722,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6634646654129028,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.5859375,
      "epoch": 0.1323024054982818,
      "grad_norm": 7.187735004337097,
      "kl": 0.01318359375,
      "learning_rate": 9.338487972508591e-07,
      "loss": 0.0005,
      "reward": 1.781952142715454,
      "reward_std": 0.20670166611671448,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7897646427154541,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 117.8828125,
      "epoch": 0.1331615120274914,
      "grad_norm": 1.5089418750844499,
      "kl": 0.016754150390625,
      "learning_rate": 9.334192439862543e-07,
      "loss": 0.0007,
      "reward": 1.763621985912323,
      "reward_std": 0.18807729333639145,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7792469561100006,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.15625,
      "epoch": 0.13402061855670103,
      "grad_norm": 14.193518545363872,
      "kl": 0.012908935546875,
      "learning_rate": 9.329896907216495e-07,
      "loss": 0.0005,
      "reward": 1.8192681670188904,
      "reward_std": 0.11732286959886551,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8192681968212128,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.5859375,
      "epoch": 0.13487972508591065,
      "grad_norm": 1.5467385079039138,
      "kl": 0.012451171875,
      "learning_rate": 9.325601374570447e-07,
      "loss": 0.0005,
      "reward": 1.710074007511139,
      "reward_std": 0.2205553948879242,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7178865075111389,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.3671875,
      "epoch": 0.13573883161512026,
      "grad_norm": 1.4389192395608355,
      "kl": 0.0135498046875,
      "learning_rate": 9.321305841924399e-07,
      "loss": 0.0005,
      "reward": 1.6988090872764587,
      "reward_std": 0.10531151667237282,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6988090872764587,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.109375,
      "epoch": 0.13659793814432988,
      "grad_norm": 1.6050785903093132,
      "kl": 0.016448974609375,
      "learning_rate": 9.317010309278351e-07,
      "loss": 0.0007,
      "reward": 1.7135010957717896,
      "reward_std": 0.21779370307922363,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7213136553764343,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.8828125,
      "epoch": 0.13745704467353953,
      "grad_norm": 1.4854987369310133,
      "kl": 0.016021728515625,
      "learning_rate": 9.312714776632303e-07,
      "loss": 0.0006,
      "reward": 1.7385917901992798,
      "reward_std": 0.15268366783857346,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7464043200016022,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.8203125,
      "epoch": 0.13831615120274915,
      "grad_norm": 3.524052752645729,
      "kl": 0.0146484375,
      "learning_rate": 9.308419243986254e-07,
      "loss": 0.0006,
      "reward": 1.7990399599075317,
      "reward_std": 0.15285367518663406,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.799039900302887,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.203125,
      "epoch": 0.13917525773195877,
      "grad_norm": 2.86620135877599,
      "kl": 0.014190673828125,
      "learning_rate": 9.304123711340206e-07,
      "loss": 0.0006,
      "reward": 1.6504857540130615,
      "reward_std": 0.1360258013010025,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6504857540130615,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.4609375,
      "epoch": 0.1400343642611684,
      "grad_norm": 1.4169335821055606,
      "kl": 0.014739990234375,
      "learning_rate": 9.299828178694158e-07,
      "loss": 0.0006,
      "reward": 1.7095910906791687,
      "reward_std": 0.08373668044805527,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7174035906791687,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.8515625,
      "epoch": 0.140893470790378,
      "grad_norm": 1.3575110989008008,
      "kl": 0.01556396484375,
      "learning_rate": 9.29553264604811e-07,
      "loss": 0.0006,
      "reward": 1.7448028326034546,
      "reward_std": 0.14756760746240616,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7448029220104218,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.1171875,
      "epoch": 0.14175257731958762,
      "grad_norm": 1.8530325477228802,
      "kl": 0.01324462890625,
      "learning_rate": 9.291237113402062e-07,
      "loss": 0.0005,
      "reward": 1.742842435836792,
      "reward_std": 0.16235359013080597,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7428424954414368,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.75,
      "epoch": 0.14261168384879724,
      "grad_norm": 1.465950210564534,
      "kl": 0.01416015625,
      "learning_rate": 9.286941580756014e-07,
      "loss": 0.0006,
      "reward": 1.813442349433899,
      "reward_std": 0.1405864767730236,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8134423494338989,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.2265625,
      "epoch": 0.14347079037800686,
      "grad_norm": 1.7221592259263123,
      "kl": 0.015533447265625,
      "learning_rate": 9.282646048109966e-07,
      "loss": 0.0006,
      "reward": 1.6789227724075317,
      "reward_std": 0.198103629052639,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6867353320121765,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.0,
      "epoch": 0.14432989690721648,
      "grad_norm": 2.1528629212609407,
      "kl": 0.013214111328125,
      "learning_rate": 9.278350515463918e-07,
      "loss": 0.0005,
      "reward": 1.7893621921539307,
      "reward_std": 0.13996141962707043,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7893621623516083,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.7265625,
      "epoch": 0.14518900343642613,
      "grad_norm": 1.3004655134925518,
      "kl": 0.0184326171875,
      "learning_rate": 9.274054982817869e-07,
      "loss": 0.0007,
      "reward": 1.7796956300735474,
      "reward_std": 0.14173657447099686,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7875081300735474,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.7578125,
      "epoch": 0.14604810996563575,
      "grad_norm": 1.4101337237690597,
      "kl": 0.01959228515625,
      "learning_rate": 9.269759450171821e-07,
      "loss": 0.0008,
      "reward": 1.8010075688362122,
      "reward_std": 0.1661326214671135,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8010075390338898,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.7265625,
      "epoch": 0.14690721649484537,
      "grad_norm": 1.4640575230347574,
      "kl": 0.02215576171875,
      "learning_rate": 9.265463917525773e-07,
      "loss": 0.0009,
      "reward": 1.6987681984901428,
      "reward_std": 0.21677575260400772,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6987681686878204,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1171875,
      "epoch": 0.14776632302405499,
      "grad_norm": 1.6395435895829094,
      "kl": 0.017822265625,
      "learning_rate": 9.261168384879725e-07,
      "loss": 0.0007,
      "reward": 1.7570144534111023,
      "reward_std": 0.14532551914453506,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7570143938064575,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.1796875,
      "epoch": 0.1486254295532646,
      "grad_norm": 1.696240174354143,
      "kl": 0.0164794921875,
      "learning_rate": 9.256872852233677e-07,
      "loss": 0.0007,
      "reward": 1.8731423616409302,
      "reward_std": 0.14208180457353592,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8731423616409302,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.1875,
      "epoch": 0.14948453608247422,
      "grad_norm": 1.3562918586715518,
      "kl": 0.01910400390625,
      "learning_rate": 9.252577319587629e-07,
      "loss": 0.0008,
      "reward": 1.7502126693725586,
      "reward_std": 0.1445324867963791,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.758025199174881,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.34375,
      "epoch": 0.15034364261168384,
      "grad_norm": 1.8285091498658437,
      "kl": 0.01953125,
      "learning_rate": 9.248281786941581e-07,
      "loss": 0.0008,
      "reward": 1.7030513286590576,
      "reward_std": 0.20504559576511383,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7108637988567352,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.4375,
      "epoch": 0.15120274914089346,
      "grad_norm": 2.1424066584845747,
      "kl": 0.0216064453125,
      "learning_rate": 9.243986254295533e-07,
      "loss": 0.0009,
      "reward": 1.6664610505104065,
      "reward_std": 0.07924733310937881,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6664610505104065,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.1484375,
      "epoch": 0.15206185567010308,
      "grad_norm": 1.8341695343755187,
      "kl": 0.021728515625,
      "learning_rate": 9.239690721649484e-07,
      "loss": 0.0009,
      "reward": 1.701620638370514,
      "reward_std": 0.135139562189579,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7016206681728363,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.5703125,
      "epoch": 0.15292096219931273,
      "grad_norm": 1.7560417473294632,
      "kl": 0.02288818359375,
      "learning_rate": 9.235395189003436e-07,
      "loss": 0.0009,
      "reward": 1.6936078667640686,
      "reward_std": 0.20969898253679276,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.717045396566391,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.453125,
      "epoch": 0.15378006872852235,
      "grad_norm": 1.66239183509703,
      "kl": 0.02362060546875,
      "learning_rate": 9.231099656357388e-07,
      "loss": 0.0009,
      "reward": 1.72810560464859,
      "reward_std": 0.174056738615036,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7515431046485901,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.2265625,
      "epoch": 0.15463917525773196,
      "grad_norm": 1.3638203436692424,
      "kl": 0.0224609375,
      "learning_rate": 9.22680412371134e-07,
      "loss": 0.0009,
      "reward": 1.7772685289382935,
      "reward_std": 0.09648095071315765,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7850809693336487,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.546875,
      "epoch": 0.15549828178694158,
      "grad_norm": 1.2134969783708147,
      "kl": 0.02392578125,
      "learning_rate": 9.222508591065292e-07,
      "loss": 0.001,
      "reward": 1.826979637145996,
      "reward_std": 0.1237468272447586,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8347920775413513,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.7890625,
      "epoch": 0.1563573883161512,
      "grad_norm": 1.3458969430241254,
      "kl": 0.02227783203125,
      "learning_rate": 9.218213058419243e-07,
      "loss": 0.0009,
      "reward": 1.7129201889038086,
      "reward_std": 0.18879132717847824,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7285451889038086,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.3359375,
      "epoch": 0.15721649484536082,
      "grad_norm": 1.750103363285581,
      "kl": 0.06610107421875,
      "learning_rate": 9.213917525773195e-07,
      "loss": 0.0026,
      "reward": 1.6476903557777405,
      "reward_std": 0.265672467648983,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.67894047498703,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0078125,
      "epoch": 0.15807560137457044,
      "grad_norm": 1.820111828776988,
      "kl": 0.02685546875,
      "learning_rate": 9.209621993127147e-07,
      "loss": 0.0011,
      "reward": 1.774660885334015,
      "reward_std": 0.15566430985927582,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7824733853340149,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.703125,
      "epoch": 0.15893470790378006,
      "grad_norm": 1.6209110696010172,
      "kl": 0.0299072265625,
      "learning_rate": 9.205326460481098e-07,
      "loss": 0.0012,
      "reward": 1.7323817014694214,
      "reward_std": 0.20174291729927063,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7636317014694214,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.859375,
      "epoch": 0.15979381443298968,
      "grad_norm": 1.1877258799941435,
      "kl": 0.02227783203125,
      "learning_rate": 9.20103092783505e-07,
      "loss": 0.0009,
      "reward": 1.7020468711853027,
      "reward_std": 0.14912591874599457,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7098594009876251,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.25,
      "epoch": 0.16065292096219932,
      "grad_norm": 1.7483290111833802,
      "kl": 0.02362060546875,
      "learning_rate": 9.196735395189002e-07,
      "loss": 0.0009,
      "reward": 1.7843592166900635,
      "reward_std": 0.18126945197582245,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7999841570854187,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.625,
      "epoch": 0.16151202749140894,
      "grad_norm": 1.6918407108780025,
      "kl": 0.02398681640625,
      "learning_rate": 9.192439862542954e-07,
      "loss": 0.001,
      "reward": 1.70003342628479,
      "reward_std": 0.15035928413271904,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7156584560871124,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 162.4609375,
      "epoch": 0.16237113402061856,
      "grad_norm": 1.3068095870355032,
      "kl": 0.0291748046875,
      "learning_rate": 9.188144329896906e-07,
      "loss": 0.0012,
      "reward": 1.6980275511741638,
      "reward_std": 0.14009612798690796,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7058400511741638,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.5625,
      "epoch": 0.16323024054982818,
      "grad_norm": 5.9784870187151045,
      "kl": 0.2835693359375,
      "learning_rate": 9.183848797250858e-07,
      "loss": 0.0113,
      "reward": 1.5892316102981567,
      "reward_std": 0.09760742634534836,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.604856550693512,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.7734375,
      "epoch": 0.1640893470790378,
      "grad_norm": 1.2813901114072874,
      "kl": 0.0228271484375,
      "learning_rate": 9.17955326460481e-07,
      "loss": 0.0009,
      "reward": 1.6868805289268494,
      "reward_std": 0.18260416388511658,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7025054693222046,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.046875,
      "epoch": 0.16494845360824742,
      "grad_norm": 6.416179300212472,
      "kl": 0.02294921875,
      "learning_rate": 9.175257731958762e-07,
      "loss": 0.0009,
      "reward": 1.6928731799125671,
      "reward_std": 0.2074473798274994,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7241232395172119,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 157.703125,
      "epoch": 0.16580756013745704,
      "grad_norm": 2.6639805313439546,
      "kl": 0.0198974609375,
      "learning_rate": 9.170962199312713e-07,
      "loss": 0.0008,
      "reward": 1.7725132703781128,
      "reward_std": 0.16904744878411293,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7881382405757904,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.21875,
      "epoch": 0.16666666666666666,
      "grad_norm": 2.6627936333789113,
      "kl": 0.0616455078125,
      "learning_rate": 9.166666666666665e-07,
      "loss": 0.0025,
      "reward": 1.726097822189331,
      "reward_std": 0.17287509143352509,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7260977625846863,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.6171875,
      "epoch": 0.16752577319587628,
      "grad_norm": 2.3427841091043904,
      "kl": 0.0228271484375,
      "learning_rate": 9.162371134020618e-07,
      "loss": 0.0009,
      "reward": 1.6643958687782288,
      "reward_std": 0.16956821829080582,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6722084581851959,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.9765625,
      "epoch": 0.16838487972508592,
      "grad_norm": 1.2879832569815648,
      "kl": 0.02276611328125,
      "learning_rate": 9.15807560137457e-07,
      "loss": 0.0009,
      "reward": 1.7251017093658447,
      "reward_std": 0.19944577664136887,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.7641642093658447,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.2890625,
      "epoch": 0.16924398625429554,
      "grad_norm": 1.302637581591733,
      "kl": 0.0181884765625,
      "learning_rate": 9.153780068728522e-07,
      "loss": 0.0007,
      "reward": 1.7328863739967346,
      "reward_std": 0.170260988175869,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7485113739967346,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.390625,
      "epoch": 0.17010309278350516,
      "grad_norm": 1.185620411258173,
      "kl": 0.0184326171875,
      "learning_rate": 9.149484536082474e-07,
      "loss": 0.0007,
      "reward": 1.7054132223129272,
      "reward_std": 0.14447502046823502,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7054132521152496,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.4296875,
      "epoch": 0.17096219931271478,
      "grad_norm": 1.7985827377480441,
      "kl": 0.02508544921875,
      "learning_rate": 9.145189003436426e-07,
      "loss": 0.001,
      "reward": 1.7419326901435852,
      "reward_std": 0.10603855177760124,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7497451901435852,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.734375,
      "epoch": 0.1718213058419244,
      "grad_norm": 1.4591627868559685,
      "kl": 0.01800537109375,
      "learning_rate": 9.140893470790378e-07,
      "loss": 0.0007,
      "reward": 1.7669562697410583,
      "reward_std": 0.16979996114969254,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7747687101364136,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.765625,
      "epoch": 0.17268041237113402,
      "grad_norm": 1.7341999103239758,
      "kl": 0.02862548828125,
      "learning_rate": 9.13659793814433e-07,
      "loss": 0.0011,
      "reward": 1.6440722942352295,
      "reward_std": 0.13084383308887482,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6518846750259399,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.2734375,
      "epoch": 0.17353951890034364,
      "grad_norm": 1.6909406413369592,
      "kl": 0.01934814453125,
      "learning_rate": 9.132302405498281e-07,
      "loss": 0.0008,
      "reward": 1.8257556557655334,
      "reward_std": 0.1285765990614891,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8257556557655334,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.515625,
      "epoch": 0.17439862542955326,
      "grad_norm": 0.9710832816221044,
      "kl": 0.01922607421875,
      "learning_rate": 9.128006872852233e-07,
      "loss": 0.0008,
      "reward": 1.7562236189842224,
      "reward_std": 0.1861056238412857,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7796610593795776,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.4609375,
      "epoch": 0.17525773195876287,
      "grad_norm": 1.5930942457393673,
      "kl": 0.01959228515625,
      "learning_rate": 9.123711340206185e-07,
      "loss": 0.0008,
      "reward": 1.7354264855384827,
      "reward_std": 0.14093422889709473,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.743239015340805,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.8828125,
      "epoch": 0.17611683848797252,
      "grad_norm": 1.1786198725658004,
      "kl": 0.02020263671875,
      "learning_rate": 9.119415807560137e-07,
      "loss": 0.0008,
      "reward": 1.8254988193511963,
      "reward_std": 0.13964856415987015,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8254987895488739,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.5,
      "epoch": 0.17697594501718214,
      "grad_norm": 1.561736876348039,
      "kl": 0.03765869140625,
      "learning_rate": 9.115120274914089e-07,
      "loss": 0.0015,
      "reward": 1.7375341653823853,
      "reward_std": 0.16468166559934616,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7531591653823853,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.421875,
      "epoch": 0.17783505154639176,
      "grad_norm": 1.673938587392466,
      "kl": 0.02197265625,
      "learning_rate": 9.110824742268041e-07,
      "loss": 0.0009,
      "reward": 1.828542172908783,
      "reward_std": 0.12264525145292282,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.844167172908783,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.1171875,
      "epoch": 0.17869415807560138,
      "grad_norm": 1.7407490513436423,
      "kl": 0.02032470703125,
      "learning_rate": 9.106529209621993e-07,
      "loss": 0.0008,
      "reward": 1.7809680104255676,
      "reward_std": 0.13108700513839722,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7809680104255676,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.921875,
      "epoch": 0.179553264604811,
      "grad_norm": 1.2574262300280894,
      "kl": 0.028564453125,
      "learning_rate": 9.102233676975945e-07,
      "loss": 0.0011,
      "reward": 1.7650489807128906,
      "reward_std": 0.11492755264043808,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7728614807128906,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.7578125,
      "epoch": 0.18041237113402062,
      "grad_norm": 1.803569017181721,
      "kl": 0.01959228515625,
      "learning_rate": 9.097938144329897e-07,
      "loss": 0.0008,
      "reward": 1.685543417930603,
      "reward_std": 0.13148176670074463,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.693355917930603,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.7265625,
      "epoch": 0.18127147766323023,
      "grad_norm": 1.8750450940771943,
      "kl": 0.02978515625,
      "learning_rate": 9.093642611683848e-07,
      "loss": 0.0012,
      "reward": 1.723173975944519,
      "reward_std": 0.21225081384181976,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7309865355491638,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.6640625,
      "epoch": 0.18213058419243985,
      "grad_norm": 1.8697489802004137,
      "kl": 0.0194091796875,
      "learning_rate": 9.0893470790378e-07,
      "loss": 0.0008,
      "reward": 1.773992896080017,
      "reward_std": 0.10730047896504402,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7818053960800171,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.6328125,
      "epoch": 0.18298969072164947,
      "grad_norm": 1.1778709789513797,
      "kl": 0.0174560546875,
      "learning_rate": 9.085051546391752e-07,
      "loss": 0.0007,
      "reward": 1.7802048325538635,
      "reward_std": 0.1343718022108078,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7802048325538635,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.171875,
      "epoch": 0.18384879725085912,
      "grad_norm": 1.6555684252295757,
      "kl": 0.0272216796875,
      "learning_rate": 9.080756013745704e-07,
      "loss": 0.0011,
      "reward": 1.6995335221290588,
      "reward_std": 0.26308566331863403,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7229710221290588,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.921875,
      "epoch": 0.18470790378006874,
      "grad_norm": 1.2357582827778086,
      "kl": 0.0206298828125,
      "learning_rate": 9.076460481099656e-07,
      "loss": 0.0008,
      "reward": 1.6951950788497925,
      "reward_std": 0.1771727241575718,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7186326384544373,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.15625,
      "epoch": 0.18556701030927836,
      "grad_norm": 1.7499046552420514,
      "kl": 0.02349853515625,
      "learning_rate": 9.072164948453608e-07,
      "loss": 0.0009,
      "reward": 1.826527178287506,
      "reward_std": 0.10365994274616241,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8343397080898285,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.9609375,
      "epoch": 0.18642611683848798,
      "grad_norm": 1.8005132756117428,
      "kl": 0.0301513671875,
      "learning_rate": 9.06786941580756e-07,
      "loss": 0.0012,
      "reward": 1.746228814125061,
      "reward_std": 0.20855771750211716,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7540413737297058,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.3359375,
      "epoch": 0.1872852233676976,
      "grad_norm": 1.9030503926874742,
      "kl": 0.0220947265625,
      "learning_rate": 9.063573883161512e-07,
      "loss": 0.0009,
      "reward": 1.7833629846572876,
      "reward_std": 0.12160376086831093,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7833629846572876,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.546875,
      "epoch": 0.18814432989690721,
      "grad_norm": 2.346663613038113,
      "kl": 0.02880859375,
      "learning_rate": 9.059278350515463e-07,
      "loss": 0.0012,
      "reward": 1.810929536819458,
      "reward_std": 0.12845928221940994,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8187420070171356,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.4140625,
      "epoch": 0.18900343642611683,
      "grad_norm": 1.951232732702119,
      "kl": 0.0296630859375,
      "learning_rate": 9.054982817869415e-07,
      "loss": 0.0012,
      "reward": 1.7572931051254272,
      "reward_std": 0.2043527588248253,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7807306349277496,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.3671875,
      "epoch": 0.18986254295532645,
      "grad_norm": 1.4773283463402496,
      "kl": 0.02398681640625,
      "learning_rate": 9.050687285223367e-07,
      "loss": 0.001,
      "reward": 1.8425282835960388,
      "reward_std": 0.09052041545510292,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8503407537937164,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.546875,
      "epoch": 0.19072164948453607,
      "grad_norm": 1.6430837871831834,
      "kl": 0.0224609375,
      "learning_rate": 9.046391752577319e-07,
      "loss": 0.0009,
      "reward": 1.6133168935775757,
      "reward_std": 0.1693805679678917,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.6367543637752533,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.3359375,
      "epoch": 0.19158075601374572,
      "grad_norm": 1.5498241580052614,
      "kl": 0.028076171875,
      "learning_rate": 9.042096219931271e-07,
      "loss": 0.0011,
      "reward": 1.832928478717804,
      "reward_std": 0.10157027468085289,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8407409191131592,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.3828125,
      "epoch": 0.19243986254295534,
      "grad_norm": 1.427575606765542,
      "kl": 0.02471923828125,
      "learning_rate": 9.037800687285223e-07,
      "loss": 0.001,
      "reward": 1.7951682806015015,
      "reward_std": 0.11901043727993965,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7951683104038239,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.5234375,
      "epoch": 0.19329896907216496,
      "grad_norm": 1.745841693534615,
      "kl": 0.05950927734375,
      "learning_rate": 9.033505154639175e-07,
      "loss": 0.0024,
      "reward": 1.7870480418205261,
      "reward_std": 0.1087818406522274,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8104856014251709,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.015625,
      "epoch": 0.19415807560137457,
      "grad_norm": 1.0774096537668199,
      "kl": 0.022216796875,
      "learning_rate": 9.029209621993127e-07,
      "loss": 0.0009,
      "reward": 1.8171106576919556,
      "reward_std": 0.10428809374570847,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.824923187494278,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.5234375,
      "epoch": 0.1950171821305842,
      "grad_norm": 2.0901239612312854,
      "kl": 0.02642822265625,
      "learning_rate": 9.024914089347078e-07,
      "loss": 0.0011,
      "reward": 1.7465097308158875,
      "reward_std": 0.14316842332482338,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7465097308158875,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.8125,
      "epoch": 0.1958762886597938,
      "grad_norm": 1.3420343738640794,
      "kl": 0.02850341796875,
      "learning_rate": 9.02061855670103e-07,
      "loss": 0.0011,
      "reward": 1.7858530282974243,
      "reward_std": 0.10892975702881813,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7936655879020691,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.6484375,
      "epoch": 0.19673539518900343,
      "grad_norm": 1.2697191544586062,
      "kl": 0.03155517578125,
      "learning_rate": 9.016323024054982e-07,
      "loss": 0.0013,
      "reward": 1.7829349040985107,
      "reward_std": 0.17472659796476364,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7985599339008331,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.53125,
      "epoch": 0.19759450171821305,
      "grad_norm": 1.2098132776705879,
      "kl": 0.02154541015625,
      "learning_rate": 9.012027491408934e-07,
      "loss": 0.0009,
      "reward": 1.8274175524711609,
      "reward_std": 0.1526181548833847,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8430425822734833,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.6015625,
      "epoch": 0.19845360824742267,
      "grad_norm": 1.3150045930691368,
      "kl": 0.0223388671875,
      "learning_rate": 9.007731958762886e-07,
      "loss": 0.0009,
      "reward": 1.7271833419799805,
      "reward_std": 0.1835765838623047,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7428083121776581,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.7578125,
      "epoch": 0.19931271477663232,
      "grad_norm": 1.1993647390638915,
      "kl": 0.031494140625,
      "learning_rate": 9.003436426116838e-07,
      "loss": 0.0013,
      "reward": 1.7216781377792358,
      "reward_std": 0.14081217721104622,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7373031079769135,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.8984375,
      "epoch": 0.20017182130584193,
      "grad_norm": 0.731243698465931,
      "kl": 0.02880859375,
      "learning_rate": 8.99914089347079e-07,
      "loss": 0.0012,
      "reward": 1.7585216164588928,
      "reward_std": 0.07797488383948803,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7663341164588928,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.171875,
      "epoch": 0.20103092783505155,
      "grad_norm": 1.4749482594585495,
      "kl": 0.021240234375,
      "learning_rate": 8.994845360824742e-07,
      "loss": 0.0009,
      "reward": 1.7960580587387085,
      "reward_std": 0.15863198041915894,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8116830587387085,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.21875,
      "epoch": 0.20189003436426117,
      "grad_norm": 1.773680287142342,
      "kl": 0.0177001953125,
      "learning_rate": 8.990549828178694e-07,
      "loss": 0.0007,
      "reward": 1.7810790538787842,
      "reward_std": 0.13028161227703094,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7810790240764618,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.6328125,
      "epoch": 0.2027491408934708,
      "grad_norm": 8.259511454922954,
      "kl": 0.0208740234375,
      "learning_rate": 8.986254295532645e-07,
      "loss": 0.0008,
      "reward": 1.8064086437225342,
      "reward_std": 0.12397770583629608,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8064086735248566,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.7890625,
      "epoch": 0.2036082474226804,
      "grad_norm": 1.308723092707521,
      "kl": 0.02813720703125,
      "learning_rate": 8.981958762886598e-07,
      "loss": 0.0011,
      "reward": 1.7656047344207764,
      "reward_std": 0.11740110442042351,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7656047642230988,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.9296875,
      "epoch": 0.20446735395189003,
      "grad_norm": 1.4015170970241377,
      "kl": 0.02276611328125,
      "learning_rate": 8.97766323024055e-07,
      "loss": 0.0009,
      "reward": 1.7170850038528442,
      "reward_std": 0.08975991420447826,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7170850038528442,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.921875,
      "epoch": 0.20532646048109965,
      "grad_norm": 1.7165031378836524,
      "kl": 0.031982421875,
      "learning_rate": 8.973367697594502e-07,
      "loss": 0.0013,
      "reward": 1.714181363582611,
      "reward_std": 0.1387975737452507,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7298063337802887,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.7734375,
      "epoch": 0.20618556701030927,
      "grad_norm": 1.1986523220206222,
      "kl": 0.021484375,
      "learning_rate": 8.969072164948454e-07,
      "loss": 0.0009,
      "reward": 1.70902019739151,
      "reward_std": 0.13189618289470673,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7090202271938324,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.0859375,
      "epoch": 0.20704467353951891,
      "grad_norm": 1.4338358835989133,
      "kl": 0.029052734375,
      "learning_rate": 8.964776632302406e-07,
      "loss": 0.0012,
      "reward": 1.7179991602897644,
      "reward_std": 0.16914209723472595,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7492491602897644,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.640625,
      "epoch": 0.20790378006872853,
      "grad_norm": 0.8698039683331652,
      "kl": 0.03466796875,
      "learning_rate": 8.960481099656358e-07,
      "loss": 0.0014,
      "reward": 1.817783772945404,
      "reward_std": 0.08906146325170994,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.825596272945404,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.46875,
      "epoch": 0.20876288659793815,
      "grad_norm": 3.260349200890088,
      "kl": 0.02471923828125,
      "learning_rate": 8.95618556701031e-07,
      "loss": 0.001,
      "reward": 1.698820948600769,
      "reward_std": 0.1781657487154007,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7066334784030914,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.1796875,
      "epoch": 0.20962199312714777,
      "grad_norm": 1.0734369880505024,
      "kl": 0.02301025390625,
      "learning_rate": 8.951890034364261e-07,
      "loss": 0.0009,
      "reward": 1.7488206624984741,
      "reward_std": 0.1078120581805706,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7566331923007965,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.4609375,
      "epoch": 0.2104810996563574,
      "grad_norm": 1.230920345198539,
      "kl": 0.03179931640625,
      "learning_rate": 8.947594501718213e-07,
      "loss": 0.0013,
      "reward": 1.7286466360092163,
      "reward_std": 0.16450222581624985,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7598966360092163,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.265625,
      "epoch": 0.211340206185567,
      "grad_norm": 4.523859706146446,
      "kl": 0.0262451171875,
      "learning_rate": 8.943298969072165e-07,
      "loss": 0.0011,
      "reward": 1.763322114944458,
      "reward_std": 0.1752321869134903,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7867595553398132,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.8359375,
      "epoch": 0.21219931271477663,
      "grad_norm": 1.363046353001964,
      "kl": 0.02996826171875,
      "learning_rate": 8.939003436426117e-07,
      "loss": 0.0012,
      "reward": 1.734560787677765,
      "reward_std": 0.17826475948095322,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7423732876777649,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.1015625,
      "epoch": 0.21305841924398625,
      "grad_norm": 1.2016846191079753,
      "kl": 0.02099609375,
      "learning_rate": 8.934707903780069e-07,
      "loss": 0.0008,
      "reward": 1.8029004335403442,
      "reward_std": 0.15353290364146233,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8107129037380219,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.953125,
      "epoch": 0.21391752577319587,
      "grad_norm": 1.6472740449517276,
      "kl": 0.03009033203125,
      "learning_rate": 8.930412371134021e-07,
      "loss": 0.0012,
      "reward": 1.728321135044098,
      "reward_std": 0.17561784386634827,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7439461648464203,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.2578125,
      "epoch": 0.21477663230240548,
      "grad_norm": 1.3208424429966092,
      "kl": 0.0274658203125,
      "learning_rate": 8.926116838487973e-07,
      "loss": 0.0011,
      "reward": 1.7464920282363892,
      "reward_std": 0.1454971358180046,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7543044984340668,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.8984375,
      "epoch": 0.21563573883161513,
      "grad_norm": 1.7163264390417992,
      "kl": 0.02838134765625,
      "learning_rate": 8.921821305841925e-07,
      "loss": 0.0011,
      "reward": 1.8135504722595215,
      "reward_std": 0.14255821704864502,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8135504722595215,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.46875,
      "epoch": 0.21649484536082475,
      "grad_norm": 1.4660984975320737,
      "kl": 0.0269775390625,
      "learning_rate": 8.917525773195877e-07,
      "loss": 0.0011,
      "reward": 1.6587898135185242,
      "reward_std": 0.16928455978631973,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6666022539138794,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.2890625,
      "epoch": 0.21735395189003437,
      "grad_norm": 1.4026200377012226,
      "kl": 0.02838134765625,
      "learning_rate": 8.913230240549828e-07,
      "loss": 0.0011,
      "reward": 1.7479596138000488,
      "reward_std": 0.09517625346779823,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7479596734046936,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.734375,
      "epoch": 0.218213058419244,
      "grad_norm": 1.8900236317096133,
      "kl": 0.03192138671875,
      "learning_rate": 8.90893470790378e-07,
      "loss": 0.0013,
      "reward": 1.7791557312011719,
      "reward_std": 0.08186223357915878,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7791557610034943,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.0234375,
      "epoch": 0.2190721649484536,
      "grad_norm": 1.348548755398183,
      "kl": 0.035400390625,
      "learning_rate": 8.904639175257731e-07,
      "loss": 0.0014,
      "reward": 1.762001395225525,
      "reward_std": 0.10934092476963997,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7698139548301697,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 157.3359375,
      "epoch": 0.21993127147766323,
      "grad_norm": 1.4276465496185335,
      "kl": 0.0269775390625,
      "learning_rate": 8.900343642611683e-07,
      "loss": 0.0011,
      "reward": 1.675103783607483,
      "reward_std": 0.21139457076787949,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7063537836074829,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.515625,
      "epoch": 0.22079037800687284,
      "grad_norm": 1.133588870905801,
      "kl": 0.03448486328125,
      "learning_rate": 8.896048109965635e-07,
      "loss": 0.0014,
      "reward": 1.71183180809021,
      "reward_std": 0.10940879955887794,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7118318676948547,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.203125,
      "epoch": 0.22164948453608246,
      "grad_norm": 0.8368892953167869,
      "kl": 0.0330810546875,
      "learning_rate": 8.891752577319587e-07,
      "loss": 0.0013,
      "reward": 1.8508310914039612,
      "reward_std": 0.07327214255928993,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8508311212062836,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 144.3359375,
      "epoch": 0.22250859106529208,
      "grad_norm": 1.006775443305009,
      "kl": 0.0328369140625,
      "learning_rate": 8.887457044673539e-07,
      "loss": 0.0013,
      "reward": 1.8622272610664368,
      "reward_std": 0.1266963928937912,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8778522610664368,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.421875,
      "epoch": 0.22336769759450173,
      "grad_norm": 0.9631829361222369,
      "kl": 0.0396728515625,
      "learning_rate": 8.88316151202749e-07,
      "loss": 0.0016,
      "reward": 1.8064301013946533,
      "reward_std": 0.11555318906903267,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8142426609992981,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.375,
      "epoch": 0.22422680412371135,
      "grad_norm": 1.5467830446043906,
      "kl": 0.03759765625,
      "learning_rate": 8.878865979381442e-07,
      "loss": 0.0015,
      "reward": 1.7076144218444824,
      "reward_std": 0.16983914375305176,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7154269218444824,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.078125,
      "epoch": 0.22508591065292097,
      "grad_norm": 1.385570288599824,
      "kl": 0.0396728515625,
      "learning_rate": 8.874570446735394e-07,
      "loss": 0.0016,
      "reward": 1.727626919746399,
      "reward_std": 0.15141097083687782,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7432518899440765,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 144.109375,
      "epoch": 0.2259450171821306,
      "grad_norm": 1.0781029500650152,
      "kl": 0.03558349609375,
      "learning_rate": 8.870274914089346e-07,
      "loss": 0.0014,
      "reward": 1.8266533613204956,
      "reward_std": 0.13985048979520798,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8344658315181732,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.7109375,
      "epoch": 0.2268041237113402,
      "grad_norm": 1.5051081251639746,
      "kl": 0.04119873046875,
      "learning_rate": 8.865979381443298e-07,
      "loss": 0.0016,
      "reward": 1.9263587594032288,
      "reward_std": 0.06610674690455198,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.9263588190078735,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.953125,
      "epoch": 0.22766323024054982,
      "grad_norm": 1.2568559492969402,
      "kl": 0.043212890625,
      "learning_rate": 8.86168384879725e-07,
      "loss": 0.0017,
      "reward": 1.7441568970680237,
      "reward_std": 0.052992574870586395,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7441569268703461,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.8203125,
      "epoch": 0.22852233676975944,
      "grad_norm": 12.957475206253665,
      "kl": 0.03289794921875,
      "learning_rate": 8.857388316151202e-07,
      "loss": 0.0013,
      "reward": 1.7724300026893616,
      "reward_std": 0.12845268845558167,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.780242532491684,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.46875,
      "epoch": 0.22938144329896906,
      "grad_norm": 4.112320049307154,
      "kl": 0.030517578125,
      "learning_rate": 8.853092783505154e-07,
      "loss": 0.0012,
      "reward": 1.7406333684921265,
      "reward_std": 0.16561638191342354,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7484458982944489,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.0859375,
      "epoch": 0.23024054982817868,
      "grad_norm": 1.3042211470111829,
      "kl": 0.039306640625,
      "learning_rate": 8.848797250859106e-07,
      "loss": 0.0016,
      "reward": 1.7143108248710632,
      "reward_std": 0.13581188768148422,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7377482950687408,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.8828125,
      "epoch": 0.23109965635738833,
      "grad_norm": 1.928604268090887,
      "kl": 0.037353515625,
      "learning_rate": 8.844501718213057e-07,
      "loss": 0.0015,
      "reward": 1.8002280592918396,
      "reward_std": 0.15524645149707794,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8080405294895172,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.171875,
      "epoch": 0.23195876288659795,
      "grad_norm": 5.27894758100115,
      "kl": 0.02862548828125,
      "learning_rate": 8.840206185567009e-07,
      "loss": 0.0011,
      "reward": 1.8452274799346924,
      "reward_std": 0.15420031547546387,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8530399203300476,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.671875,
      "epoch": 0.23281786941580757,
      "grad_norm": 1.306834060633908,
      "kl": 0.0325927734375,
      "learning_rate": 8.835910652920961e-07,
      "loss": 0.0013,
      "reward": 1.6935831308364868,
      "reward_std": 0.154945969581604,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7092081606388092,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.8125,
      "epoch": 0.23367697594501718,
      "grad_norm": 1.2910579810108835,
      "kl": 0.03216552734375,
      "learning_rate": 8.831615120274913e-07,
      "loss": 0.0013,
      "reward": 1.785846471786499,
      "reward_std": 0.08592062070965767,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7936590611934662,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.4765625,
      "epoch": 0.2345360824742268,
      "grad_norm": 1.6016973929619667,
      "kl": 0.0333251953125,
      "learning_rate": 8.827319587628865e-07,
      "loss": 0.0013,
      "reward": 1.6971846222877502,
      "reward_std": 0.1201891340315342,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7049971222877502,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.0703125,
      "epoch": 0.23539518900343642,
      "grad_norm": 1.357010665541966,
      "kl": 0.03564453125,
      "learning_rate": 8.823024054982817e-07,
      "loss": 0.0014,
      "reward": 1.7180689573287964,
      "reward_std": 0.19224438071250916,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7415064871311188,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0859375,
      "epoch": 0.23625429553264604,
      "grad_norm": 1.1159550961761515,
      "kl": 0.03515625,
      "learning_rate": 8.818728522336769e-07,
      "loss": 0.0014,
      "reward": 1.7949897646903992,
      "reward_std": 0.14928951114416122,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8184272944927216,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.6328125,
      "epoch": 0.23711340206185566,
      "grad_norm": 3.5515611265357485,
      "kl": 0.02630615234375,
      "learning_rate": 8.814432989690721e-07,
      "loss": 0.0011,
      "reward": 1.7773825526237488,
      "reward_std": 0.10580384358763695,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7773825824260712,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.4140625,
      "epoch": 0.23797250859106528,
      "grad_norm": 1.5580462509961617,
      "kl": 0.03564453125,
      "learning_rate": 8.810137457044672e-07,
      "loss": 0.0014,
      "reward": 1.7888085842132568,
      "reward_std": 0.21199411898851395,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8122460544109344,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.171875,
      "epoch": 0.23883161512027493,
      "grad_norm": 1.0543303085586277,
      "kl": 0.02984619140625,
      "learning_rate": 8.805841924398624e-07,
      "loss": 0.0012,
      "reward": 1.8188990950584412,
      "reward_std": 0.13441307097673416,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8423365950584412,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.8984375,
      "epoch": 0.23969072164948454,
      "grad_norm": 1.5265695805128905,
      "kl": 0.02557373046875,
      "learning_rate": 8.801546391752576e-07,
      "loss": 0.001,
      "reward": 1.7924699783325195,
      "reward_std": 0.16526872664690018,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8002825081348419,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.8671875,
      "epoch": 0.24054982817869416,
      "grad_norm": 1.1812679545358764,
      "kl": 0.0225830078125,
      "learning_rate": 8.797250859106528e-07,
      "loss": 0.0009,
      "reward": 1.827331304550171,
      "reward_std": 0.10759979486465454,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8273313045501709,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.28125,
      "epoch": 0.24140893470790378,
      "grad_norm": 1.5991468981516737,
      "kl": 0.0291748046875,
      "learning_rate": 8.792955326460481e-07,
      "loss": 0.0012,
      "reward": 1.7398195266723633,
      "reward_std": 0.19154010713100433,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7476319670677185,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.546875,
      "epoch": 0.2422680412371134,
      "grad_norm": 0.9116875920398242,
      "kl": 0.0299072265625,
      "learning_rate": 8.788659793814433e-07,
      "loss": 0.0012,
      "reward": 1.7467403411865234,
      "reward_std": 0.08945905789732933,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7545528709888458,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.46875,
      "epoch": 0.24312714776632302,
      "grad_norm": 0.8380935360171634,
      "kl": 0.0316162109375,
      "learning_rate": 8.784364261168385e-07,
      "loss": 0.0013,
      "reward": 1.8638163208961487,
      "reward_std": 0.09008487407118082,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8638162612915039,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.296875,
      "epoch": 0.24398625429553264,
      "grad_norm": 0.9161141195578109,
      "kl": 0.027587890625,
      "learning_rate": 8.780068728522337e-07,
      "loss": 0.0011,
      "reward": 1.8011575937271118,
      "reward_std": 0.0535787851549685,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8011575937271118,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.3203125,
      "epoch": 0.24484536082474226,
      "grad_norm": 1.9166929431895137,
      "kl": 0.024658203125,
      "learning_rate": 8.775773195876289e-07,
      "loss": 0.001,
      "reward": 1.7418934106826782,
      "reward_std": 0.14604221284389496,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7497059106826782,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.2890625,
      "epoch": 0.24570446735395188,
      "grad_norm": 1.1252522693717517,
      "kl": 0.02996826171875,
      "learning_rate": 8.77147766323024e-07,
      "loss": 0.0012,
      "reward": 1.7983113527297974,
      "reward_std": 0.09610910713672638,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.806123822927475,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.921875,
      "epoch": 0.24656357388316152,
      "grad_norm": 1.3367048117093618,
      "kl": 0.034912109375,
      "learning_rate": 8.767182130584192e-07,
      "loss": 0.0014,
      "reward": 1.7578275799751282,
      "reward_std": 0.14203480631113052,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.765640139579773,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.625,
      "epoch": 0.24742268041237114,
      "grad_norm": 0.724502338816319,
      "kl": 0.03143310546875,
      "learning_rate": 8.762886597938144e-07,
      "loss": 0.0013,
      "reward": 1.7635814547538757,
      "reward_std": 0.0678582051768899,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7635815143585205,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 163.6953125,
      "epoch": 0.24828178694158076,
      "grad_norm": 1.1733316115040695,
      "kl": 0.0283203125,
      "learning_rate": 8.758591065292096e-07,
      "loss": 0.0011,
      "reward": 1.7535958290100098,
      "reward_std": 0.16154304146766663,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7614083290100098,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.8046875,
      "epoch": 0.24914089347079038,
      "grad_norm": 1.4167971747096428,
      "kl": 0.0279541015625,
      "learning_rate": 8.754295532646048e-07,
      "loss": 0.0011,
      "reward": 1.795049250125885,
      "reward_std": 0.07856442406773567,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7950493097305298,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.734375,
      "epoch": 0.25,
      "grad_norm": 3.1009628872255646,
      "kl": 0.03045654296875,
      "learning_rate": 8.75e-07,
      "loss": 0.0012,
      "reward": 1.8120030760765076,
      "reward_std": 0.16107841953635216,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8276280760765076,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.7265625,
      "epoch": 0.2508591065292096,
      "grad_norm": 1.1395121387903864,
      "kl": 0.02764892578125,
      "learning_rate": 8.745704467353952e-07,
      "loss": 0.0011,
      "reward": 1.7605109214782715,
      "reward_std": 0.093373142182827,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7683233916759491,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.34375,
      "epoch": 0.25171821305841924,
      "grad_norm": 1.0703218519168713,
      "kl": 0.03216552734375,
      "learning_rate": 8.741408934707904e-07,
      "loss": 0.0013,
      "reward": 1.880930781364441,
      "reward_std": 0.11160749942064285,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8887432813644409,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1484375,
      "epoch": 0.25257731958762886,
      "grad_norm": 3.1378356819250217,
      "kl": 0.0274658203125,
      "learning_rate": 8.737113402061856e-07,
      "loss": 0.0011,
      "reward": 1.8660714626312256,
      "reward_std": 0.1355828121304512,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8738839626312256,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.90625,
      "epoch": 0.2534364261168385,
      "grad_norm": 0.9752935666515755,
      "kl": 0.02984619140625,
      "learning_rate": 8.732817869415807e-07,
      "loss": 0.0012,
      "reward": 1.8270512223243713,
      "reward_std": 0.09455467015504837,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8348636627197266,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.4140625,
      "epoch": 0.2542955326460481,
      "grad_norm": 2.5028271019973256,
      "kl": 0.02728271484375,
      "learning_rate": 8.728522336769759e-07,
      "loss": 0.0011,
      "reward": 1.81145840883255,
      "reward_std": 0.1066664457321167,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8114584386348724,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.6171875,
      "epoch": 0.2551546391752577,
      "grad_norm": 1.9066759357991514,
      "kl": 0.03387451171875,
      "learning_rate": 8.724226804123711e-07,
      "loss": 0.0014,
      "reward": 1.7507621049880981,
      "reward_std": 0.1560019999742508,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7585746049880981,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.296875,
      "epoch": 0.25601374570446733,
      "grad_norm": 1.659392268817117,
      "kl": 0.03125,
      "learning_rate": 8.719931271477663e-07,
      "loss": 0.0012,
      "reward": 1.8714821338653564,
      "reward_std": 0.10832902416586876,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8792945742607117,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.0234375,
      "epoch": 0.25687285223367695,
      "grad_norm": 1.109700712202529,
      "kl": 0.02984619140625,
      "learning_rate": 8.715635738831615e-07,
      "loss": 0.0012,
      "reward": 1.8865798711776733,
      "reward_std": 0.08204978704452515,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8865799009799957,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.015625,
      "epoch": 0.25773195876288657,
      "grad_norm": 1.0487633124419409,
      "kl": 0.0244140625,
      "learning_rate": 8.711340206185567e-07,
      "loss": 0.001,
      "reward": 1.809591829776764,
      "reward_std": 0.15175417065620422,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8330293297767639,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.25,
      "epoch": 0.25859106529209624,
      "grad_norm": 1.5750530363933386,
      "kl": 0.04327392578125,
      "learning_rate": 8.707044673539519e-07,
      "loss": 0.0017,
      "reward": 1.792675495147705,
      "reward_std": 0.08446579799056053,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8004880249500275,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.0546875,
      "epoch": 0.25945017182130586,
      "grad_norm": 1.9949049568284047,
      "kl": 0.026123046875,
      "learning_rate": 8.702749140893471e-07,
      "loss": 0.001,
      "reward": 1.8067399859428406,
      "reward_std": 0.14312193542718887,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8145524859428406,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.796875,
      "epoch": 0.2603092783505155,
      "grad_norm": 1.7326491889574818,
      "kl": 0.02447509765625,
      "learning_rate": 8.698453608247422e-07,
      "loss": 0.001,
      "reward": 1.8493663668632507,
      "reward_std": 0.06654036790132523,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8493663370609283,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.4296875,
      "epoch": 0.2611683848797251,
      "grad_norm": 1.9340178331586866,
      "kl": 0.0367431640625,
      "learning_rate": 8.694158075601374e-07,
      "loss": 0.0015,
      "reward": 1.7990782260894775,
      "reward_std": 0.14300024509429932,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7990781962871552,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.3046875,
      "epoch": 0.2620274914089347,
      "grad_norm": 1.6168654993721436,
      "kl": 0.0328369140625,
      "learning_rate": 8.689862542955326e-07,
      "loss": 0.0013,
      "reward": 1.7844675183296204,
      "reward_std": 0.10683033242821693,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.784467488527298,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.4921875,
      "epoch": 0.26288659793814434,
      "grad_norm": 1.3435959118517327,
      "kl": 0.0362548828125,
      "learning_rate": 8.685567010309278e-07,
      "loss": 0.0015,
      "reward": 1.7007364630699158,
      "reward_std": 0.10049042850732803,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7007364630699158,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.8359375,
      "epoch": 0.26374570446735396,
      "grad_norm": 1.7041209629618606,
      "kl": 0.02471923828125,
      "learning_rate": 8.68127147766323e-07,
      "loss": 0.001,
      "reward": 1.7297524213790894,
      "reward_std": 0.11169169098138809,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7297524213790894,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1171875,
      "epoch": 0.2646048109965636,
      "grad_norm": 1.0611383574670885,
      "kl": 0.0352783203125,
      "learning_rate": 8.676975945017182e-07,
      "loss": 0.0014,
      "reward": 1.765690803527832,
      "reward_std": 0.07691102847456932,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7656907737255096,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.5,
      "epoch": 0.2654639175257732,
      "grad_norm": 1.810412989768572,
      "kl": 0.061279296875,
      "learning_rate": 8.672680412371134e-07,
      "loss": 0.0024,
      "reward": 1.6709439158439636,
      "reward_std": 0.11246037483215332,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6709439158439636,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.1796875,
      "epoch": 0.2663230240549828,
      "grad_norm": 3.183250415302431,
      "kl": 0.03106689453125,
      "learning_rate": 8.668384879725086e-07,
      "loss": 0.0012,
      "reward": 1.7835421562194824,
      "reward_std": 0.08028195053339005,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7835421562194824,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.3515625,
      "epoch": 0.26718213058419243,
      "grad_norm": 1.290985809671743,
      "kl": 0.033203125,
      "learning_rate": 8.664089347079037e-07,
      "loss": 0.0013,
      "reward": 1.7408164143562317,
      "reward_std": 0.15223591029644012,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7564414143562317,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.75,
      "epoch": 0.26804123711340205,
      "grad_norm": 1.8077145483203556,
      "kl": 0.040283203125,
      "learning_rate": 8.659793814432989e-07,
      "loss": 0.0016,
      "reward": 1.7184379696846008,
      "reward_std": 0.12002924084663391,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7340629696846008,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.3828125,
      "epoch": 0.26890034364261167,
      "grad_norm": 2.7429724901343433,
      "kl": 0.048583984375,
      "learning_rate": 8.655498281786941e-07,
      "loss": 0.0019,
      "reward": 1.731776773929596,
      "reward_std": 0.1657547503709793,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.747401773929596,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.0234375,
      "epoch": 0.2697594501718213,
      "grad_norm": 0.9847082773950215,
      "kl": 0.0380859375,
      "learning_rate": 8.651202749140893e-07,
      "loss": 0.0015,
      "reward": 1.701866626739502,
      "reward_std": 0.12148784846067429,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7018666565418243,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 156.46875,
      "epoch": 0.2706185567010309,
      "grad_norm": 1.5399880460978463,
      "kl": 0.03057861328125,
      "learning_rate": 8.646907216494845e-07,
      "loss": 0.0012,
      "reward": 1.6783122420310974,
      "reward_std": 0.1540703848004341,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.686124712228775,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.9921875,
      "epoch": 0.27147766323024053,
      "grad_norm": 1.5313679692857116,
      "kl": 0.0394287109375,
      "learning_rate": 8.642611683848797e-07,
      "loss": 0.0016,
      "reward": 1.7149747014045715,
      "reward_std": 0.2113272026181221,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7462246716022491,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.3984375,
      "epoch": 0.27233676975945015,
      "grad_norm": 1.3181550613878923,
      "kl": 0.0369873046875,
      "learning_rate": 8.638316151202749e-07,
      "loss": 0.0015,
      "reward": 1.8070083856582642,
      "reward_std": 0.18914027512073517,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8226334154605865,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.4453125,
      "epoch": 0.27319587628865977,
      "grad_norm": 1.1473725643237571,
      "kl": 0.035400390625,
      "learning_rate": 8.634020618556701e-07,
      "loss": 0.0014,
      "reward": 1.8190871477127075,
      "reward_std": 0.15289827063679695,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.8503372073173523,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 157.6015625,
      "epoch": 0.27405498281786944,
      "grad_norm": 0.7143585607947084,
      "kl": 0.03167724609375,
      "learning_rate": 8.629725085910653e-07,
      "loss": 0.0013,
      "reward": 1.8210635781288147,
      "reward_std": 0.07932163029909134,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8210635781288147,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.28125,
      "epoch": 0.27491408934707906,
      "grad_norm": 1.3665635736893489,
      "kl": 0.02557373046875,
      "learning_rate": 8.625429553264604e-07,
      "loss": 0.001,
      "reward": 1.7617181539535522,
      "reward_std": 0.1237938292324543,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7617180943489075,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.1953125,
      "epoch": 0.2757731958762887,
      "grad_norm": 1.1585944459835054,
      "kl": 0.0296630859375,
      "learning_rate": 8.621134020618556e-07,
      "loss": 0.0012,
      "reward": 1.7693681120872498,
      "reward_std": 0.15534278005361557,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7771806418895721,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.359375,
      "epoch": 0.2766323024054983,
      "grad_norm": 1.3885088694278251,
      "kl": 0.03216552734375,
      "learning_rate": 8.616838487972508e-07,
      "loss": 0.0013,
      "reward": 1.6785261631011963,
      "reward_std": 0.11299961805343628,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6785261631011963,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.0234375,
      "epoch": 0.2774914089347079,
      "grad_norm": 1.0963555730545884,
      "kl": 0.035888671875,
      "learning_rate": 8.612542955326461e-07,
      "loss": 0.0014,
      "reward": 1.724756121635437,
      "reward_std": 0.15419768542051315,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7325686514377594,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.734375,
      "epoch": 0.27835051546391754,
      "grad_norm": 1.9036632085091891,
      "kl": 0.029296875,
      "learning_rate": 8.608247422680413e-07,
      "loss": 0.0012,
      "reward": 1.7815952897071838,
      "reward_std": 0.07885266095399857,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7815952897071838,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.078125,
      "epoch": 0.27920962199312716,
      "grad_norm": 0.8583001841176572,
      "kl": 0.03070068359375,
      "learning_rate": 8.603951890034365e-07,
      "loss": 0.0012,
      "reward": 1.805925726890564,
      "reward_std": 0.08024111017584801,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8059256374835968,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.4921875,
      "epoch": 0.2800687285223368,
      "grad_norm": 1.6115188524753052,
      "kl": 0.0347900390625,
      "learning_rate": 8.599656357388317e-07,
      "loss": 0.0014,
      "reward": 1.6898165941238403,
      "reward_std": 0.17968852818012238,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7132540941238403,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.09375,
      "epoch": 0.2809278350515464,
      "grad_norm": 3.0925571712058755,
      "kl": 0.0787353515625,
      "learning_rate": 8.595360824742269e-07,
      "loss": 0.0032,
      "reward": 1.7974568605422974,
      "reward_std": 0.1517610400915146,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8208943903446198,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 171.8125,
      "epoch": 0.281786941580756,
      "grad_norm": 1.0479920039978226,
      "kl": 0.02923583984375,
      "learning_rate": 8.591065292096219e-07,
      "loss": 0.0012,
      "reward": 1.8526938557624817,
      "reward_std": 0.07072590291500092,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8526938557624817,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.2734375,
      "epoch": 0.28264604810996563,
      "grad_norm": 2.741217901691856,
      "kl": 0.03240966796875,
      "learning_rate": 8.586769759450171e-07,
      "loss": 0.0013,
      "reward": 1.7304821014404297,
      "reward_std": 0.10214803740382195,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7461071014404297,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 161.578125,
      "epoch": 0.28350515463917525,
      "grad_norm": 1.1171189004763298,
      "kl": 0.0263671875,
      "learning_rate": 8.582474226804123e-07,
      "loss": 0.0011,
      "reward": 1.7979252934455872,
      "reward_std": 0.12719742208719254,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8057378232479095,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 151.4765625,
      "epoch": 0.28436426116838487,
      "grad_norm": 2.031628516822189,
      "kl": 0.05181884765625,
      "learning_rate": 8.578178694158075e-07,
      "loss": 0.0021,
      "reward": 1.6497213244438171,
      "reward_std": 0.1195969358086586,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.6731588840484619,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.4453125,
      "epoch": 0.2852233676975945,
      "grad_norm": 2.801651555129562,
      "kl": 0.02777099609375,
      "learning_rate": 8.573883161512027e-07,
      "loss": 0.0011,
      "reward": 1.8582192659378052,
      "reward_std": 0.10834803432226181,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8738442659378052,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.6640625,
      "epoch": 0.2860824742268041,
      "grad_norm": 11.067746215654495,
      "kl": 0.02642822265625,
      "learning_rate": 8.569587628865979e-07,
      "loss": 0.0011,
      "reward": 1.7104763984680176,
      "reward_std": 0.19199497997760773,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.74953892827034,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.6796875,
      "epoch": 0.2869415807560137,
      "grad_norm": 1.2489715002859365,
      "kl": 0.0211181640625,
      "learning_rate": 8.565292096219931e-07,
      "loss": 0.0008,
      "reward": 1.6395170092582703,
      "reward_std": 0.1772407442331314,
      "rewards/format_reward_gen": 0.9296875,
      "rewards/llm_reward": 0.7098294794559479,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.953125,
      "epoch": 0.28780068728522334,
      "grad_norm": 1.3905135992782467,
      "kl": 0.02386474609375,
      "learning_rate": 8.560996563573883e-07,
      "loss": 0.001,
      "reward": 1.6524567604064941,
      "reward_std": 0.22227772325277328,
      "rewards/format_reward_gen": 0.921875,
      "rewards/llm_reward": 0.7305817306041718,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.75,
      "epoch": 0.28865979381443296,
      "grad_norm": 6.054324722106658,
      "kl": 0.02081298828125,
      "learning_rate": 8.556701030927834e-07,
      "loss": 0.0008,
      "reward": 1.7611451148986816,
      "reward_std": 0.2188207283616066,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7923950850963593,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.796875,
      "epoch": 0.28951890034364264,
      "grad_norm": 3.7876750335169507,
      "kl": 0.021484375,
      "learning_rate": 8.552405498281786e-07,
      "loss": 0.0009,
      "reward": 1.681132197380066,
      "reward_std": 0.23194558173418045,
      "rewards/format_reward_gen": 0.9296875,
      "rewards/llm_reward": 0.7514447271823883,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.84375,
      "epoch": 0.29037800687285226,
      "grad_norm": 1.4224659056279383,
      "kl": 0.02587890625,
      "learning_rate": 8.548109965635738e-07,
      "loss": 0.001,
      "reward": 1.78468519449234,
      "reward_std": 0.2202548086643219,
      "rewards/format_reward_gen": 0.9453125,
      "rewards/llm_reward": 0.8393726646900177,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.328125,
      "epoch": 0.2912371134020619,
      "grad_norm": 1.811194222155832,
      "kl": 0.024658203125,
      "learning_rate": 8.54381443298969e-07,
      "loss": 0.001,
      "reward": 1.8332806825637817,
      "reward_std": 0.09518006816506386,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.8567183017730713,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.078125,
      "epoch": 0.2920962199312715,
      "grad_norm": 1.45115586999827,
      "kl": 0.02301025390625,
      "learning_rate": 8.539518900343642e-07,
      "loss": 0.0009,
      "reward": 1.7859101295471191,
      "reward_std": 0.14324617385864258,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.8249726593494415,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.6015625,
      "epoch": 0.2929553264604811,
      "grad_norm": 1.3891780059241028,
      "kl": 0.0213623046875,
      "learning_rate": 8.535223367697594e-07,
      "loss": 0.0009,
      "reward": 1.7494396567344666,
      "reward_std": 0.14654994010925293,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7572520971298218,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.5,
      "epoch": 0.29381443298969073,
      "grad_norm": 1.0515730604032878,
      "kl": 0.0264892578125,
      "learning_rate": 8.530927835051546e-07,
      "loss": 0.0011,
      "reward": 1.7149075269699097,
      "reward_std": 0.12536488845944405,
      "rewards/format_reward_gen": 0.9453125,
      "rewards/llm_reward": 0.7695950269699097,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 152.171875,
      "epoch": 0.29467353951890035,
      "grad_norm": 1.884237347147452,
      "kl": 0.0213623046875,
      "learning_rate": 8.526632302405498e-07,
      "loss": 0.0009,
      "reward": 1.7150232791900635,
      "reward_std": 0.17639710754156113,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7306482791900635,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.0390625,
      "epoch": 0.29553264604810997,
      "grad_norm": 1.0301696531429079,
      "kl": 0.02069091796875,
      "learning_rate": 8.52233676975945e-07,
      "loss": 0.0008,
      "reward": 1.8793914318084717,
      "reward_std": 0.04007810167968273,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8793914318084717,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0859375,
      "epoch": 0.2963917525773196,
      "grad_norm": 1.6887023989278636,
      "kl": 0.022216796875,
      "learning_rate": 8.518041237113401e-07,
      "loss": 0.0009,
      "reward": 1.7447214126586914,
      "reward_std": 0.16757074743509293,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7447213530540466,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.859375,
      "epoch": 0.2972508591065292,
      "grad_norm": 0.9804680400847956,
      "kl": 0.026123046875,
      "learning_rate": 8.513745704467353e-07,
      "loss": 0.001,
      "reward": 1.7684549689292908,
      "reward_std": 0.06802868098020554,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7762674689292908,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.203125,
      "epoch": 0.2981099656357388,
      "grad_norm": 1.4132838829734102,
      "kl": 0.02655029296875,
      "learning_rate": 8.509450171821305e-07,
      "loss": 0.0011,
      "reward": 1.7236329913139343,
      "reward_std": 0.15013974159955978,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7392580211162567,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.34375,
      "epoch": 0.29896907216494845,
      "grad_norm": 3.798539075577766,
      "kl": 0.03009033203125,
      "learning_rate": 8.505154639175257e-07,
      "loss": 0.0012,
      "reward": 1.7059985995292664,
      "reward_std": 0.13966944441199303,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7059986293315887,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.7578125,
      "epoch": 0.29982817869415807,
      "grad_norm": 1.4928606296691562,
      "kl": 0.05828857421875,
      "learning_rate": 8.500859106529209e-07,
      "loss": 0.0023,
      "reward": 1.7457636594772339,
      "reward_std": 0.16971006989479065,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7770136296749115,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.828125,
      "epoch": 0.3006872852233677,
      "grad_norm": 8.775228795942514,
      "kl": 0.02349853515625,
      "learning_rate": 8.496563573883161e-07,
      "loss": 0.0009,
      "reward": 1.75225168466568,
      "reward_std": 0.11466844379901886,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7522516846656799,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.2578125,
      "epoch": 0.3015463917525773,
      "grad_norm": 1.2582262027913427,
      "kl": 0.02496337890625,
      "learning_rate": 8.492268041237113e-07,
      "loss": 0.001,
      "reward": 1.8221701383590698,
      "reward_std": 0.09425491839647293,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8299825489521027,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 115.328125,
      "epoch": 0.3024054982817869,
      "grad_norm": 8.10337252047317,
      "kl": 0.029541015625,
      "learning_rate": 8.487972508591065e-07,
      "loss": 0.0012,
      "reward": 1.7997434735298157,
      "reward_std": 0.07890489138662815,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7997434437274933,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.4375,
      "epoch": 0.30326460481099654,
      "grad_norm": 0.8630573702061485,
      "kl": 0.025146484375,
      "learning_rate": 8.483676975945016e-07,
      "loss": 0.001,
      "reward": 1.735402524471283,
      "reward_std": 0.1313318181782961,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7510275840759277,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.671875,
      "epoch": 0.30412371134020616,
      "grad_norm": 2.294001840527455,
      "kl": 0.021240234375,
      "learning_rate": 8.479381443298968e-07,
      "loss": 0.0008,
      "reward": 1.8022547364234924,
      "reward_std": 0.14323440939188004,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.81006720662117,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.0625,
      "epoch": 0.30498281786941583,
      "grad_norm": 1.3365205015914472,
      "kl": 0.0291748046875,
      "learning_rate": 8.47508591065292e-07,
      "loss": 0.0012,
      "reward": 1.798896074295044,
      "reward_std": 0.14688794314861298,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.806708574295044,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 116.65625,
      "epoch": 0.30584192439862545,
      "grad_norm": 1.3467405037883613,
      "kl": 0.03033447265625,
      "learning_rate": 8.470790378006872e-07,
      "loss": 0.0012,
      "reward": 1.7370511293411255,
      "reward_std": 0.1430470496416092,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7448635101318359,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.3125,
      "epoch": 0.30670103092783507,
      "grad_norm": 1.4250852392657183,
      "kl": 0.02288818359375,
      "learning_rate": 8.466494845360824e-07,
      "loss": 0.0009,
      "reward": 1.7630817294120789,
      "reward_std": 0.099862240254879,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7630816698074341,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.484375,
      "epoch": 0.3075601374570447,
      "grad_norm": 1.001564962464479,
      "kl": 0.0289306640625,
      "learning_rate": 8.462199312714776e-07,
      "loss": 0.0012,
      "reward": 1.7671634554862976,
      "reward_std": 0.07000754773616791,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7671634256839752,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 110.234375,
      "epoch": 0.3084192439862543,
      "grad_norm": 1.7447567831684878,
      "kl": 0.02752685546875,
      "learning_rate": 8.457903780068728e-07,
      "loss": 0.0011,
      "reward": 1.6856686472892761,
      "reward_std": 0.12096000462770462,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6856685876846313,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.0078125,
      "epoch": 0.30927835051546393,
      "grad_norm": 1.2242293521408898,
      "kl": 0.02239990234375,
      "learning_rate": 8.45360824742268e-07,
      "loss": 0.0009,
      "reward": 1.8242759108543396,
      "reward_std": 0.08643890917301178,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8242759108543396,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.65625,
      "epoch": 0.31013745704467355,
      "grad_norm": 1.5243424904852039,
      "kl": 0.0262451171875,
      "learning_rate": 8.449312714776631e-07,
      "loss": 0.001,
      "reward": 1.7807704210281372,
      "reward_std": 0.0936017706990242,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7885829210281372,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.734375,
      "epoch": 0.31099656357388317,
      "grad_norm": 1.4884814724042141,
      "kl": 0.022216796875,
      "learning_rate": 8.445017182130583e-07,
      "loss": 0.0009,
      "reward": 1.8066073656082153,
      "reward_std": 0.09719736129045486,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8144198656082153,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.359375,
      "epoch": 0.3118556701030928,
      "grad_norm": 1.4940528435261182,
      "kl": 0.02490234375,
      "learning_rate": 8.440721649484535e-07,
      "loss": 0.001,
      "reward": 1.6919100880622864,
      "reward_std": 0.08997226506471634,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6919100880622864,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.2265625,
      "epoch": 0.3127147766323024,
      "grad_norm": 1.0097324218360122,
      "kl": 0.023681640625,
      "learning_rate": 8.436426116838487e-07,
      "loss": 0.0009,
      "reward": 1.7592207789421082,
      "reward_std": 0.12093230336904526,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7592207789421082,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.3984375,
      "epoch": 0.313573883161512,
      "grad_norm": 1.1761577149895857,
      "kl": 0.01922607421875,
      "learning_rate": 8.432130584192439e-07,
      "loss": 0.0008,
      "reward": 1.7229613065719604,
      "reward_std": 0.1399160847067833,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7229613661766052,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.0390625,
      "epoch": 0.31443298969072164,
      "grad_norm": 2.443207308672,
      "kl": 0.028076171875,
      "learning_rate": 8.427835051546391e-07,
      "loss": 0.0011,
      "reward": 1.78279447555542,
      "reward_std": 0.08378404378890991,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7827944755554199,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.15625,
      "epoch": 0.31529209621993126,
      "grad_norm": 1.3122123216803352,
      "kl": 0.0269775390625,
      "learning_rate": 8.423539518900344e-07,
      "loss": 0.0011,
      "reward": 1.7783340811729431,
      "reward_std": 0.13799217343330383,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7783340811729431,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.8125,
      "epoch": 0.3161512027491409,
      "grad_norm": 1.6680208275489512,
      "kl": 0.0198974609375,
      "learning_rate": 8.419243986254296e-07,
      "loss": 0.0008,
      "reward": 1.7162877321243286,
      "reward_std": 0.07326348312199116,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7162877917289734,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.3125,
      "epoch": 0.3170103092783505,
      "grad_norm": 1.382745000875992,
      "kl": 0.02130126953125,
      "learning_rate": 8.414948453608248e-07,
      "loss": 0.0009,
      "reward": 1.79585862159729,
      "reward_std": 0.13986750692129135,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.79585862159729,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.625,
      "epoch": 0.3178694158075601,
      "grad_norm": 1.1013774115401274,
      "kl": 0.02069091796875,
      "learning_rate": 8.410652920962199e-07,
      "loss": 0.0008,
      "reward": 1.753614366054535,
      "reward_std": 0.10165842995047569,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7536143660545349,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.4765625,
      "epoch": 0.31872852233676974,
      "grad_norm": 1.7113488988118148,
      "kl": 0.0211181640625,
      "learning_rate": 8.406357388316151e-07,
      "loss": 0.0008,
      "reward": 1.861440122127533,
      "reward_std": 0.06895541399717331,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8692527115345001,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.9765625,
      "epoch": 0.31958762886597936,
      "grad_norm": 1.103897326474956,
      "kl": 0.021484375,
      "learning_rate": 8.402061855670103e-07,
      "loss": 0.0009,
      "reward": 1.7671774625778198,
      "reward_std": 0.09223662130534649,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7671774327754974,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.59375,
      "epoch": 0.32044673539518903,
      "grad_norm": 1.1271147620986939,
      "kl": 0.018157958984375,
      "learning_rate": 8.397766323024055e-07,
      "loss": 0.0007,
      "reward": 1.7452142238616943,
      "reward_std": 0.12823403999209404,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7530267238616943,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.59375,
      "epoch": 0.32130584192439865,
      "grad_norm": 1.1312273949872451,
      "kl": 0.0181884765625,
      "learning_rate": 8.393470790378007e-07,
      "loss": 0.0007,
      "reward": 1.76444673538208,
      "reward_std": 0.08396412804722786,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7644466161727905,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.3359375,
      "epoch": 0.32216494845360827,
      "grad_norm": 1.1834607042303573,
      "kl": 0.02215576171875,
      "learning_rate": 8.389175257731959e-07,
      "loss": 0.0009,
      "reward": 1.7634702920913696,
      "reward_std": 0.07862638682126999,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7712827622890472,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.6796875,
      "epoch": 0.3230240549828179,
      "grad_norm": 1.218745212804485,
      "kl": 0.027587890625,
      "learning_rate": 8.384879725085911e-07,
      "loss": 0.0011,
      "reward": 1.773351788520813,
      "reward_std": 0.10650411248207092,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7889767587184906,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.5625,
      "epoch": 0.3238831615120275,
      "grad_norm": 2.9765457354255247,
      "kl": 0.0250244140625,
      "learning_rate": 8.380584192439863e-07,
      "loss": 0.001,
      "reward": 1.8011161088943481,
      "reward_std": 0.14274968206882477,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8089286386966705,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.703125,
      "epoch": 0.3247422680412371,
      "grad_norm": 4.935642518044648,
      "kl": 0.01898193359375,
      "learning_rate": 8.376288659793815e-07,
      "loss": 0.0008,
      "reward": 1.8446325659751892,
      "reward_std": 0.12704916298389435,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8524451553821564,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.2109375,
      "epoch": 0.32560137457044674,
      "grad_norm": 0.8982810314616194,
      "kl": 0.0203857421875,
      "learning_rate": 8.371993127147766e-07,
      "loss": 0.0008,
      "reward": 1.8438200950622559,
      "reward_std": 0.07897239178419113,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8438200354576111,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.6328125,
      "epoch": 0.32646048109965636,
      "grad_norm": 0.8997947654002385,
      "kl": 0.02166748046875,
      "learning_rate": 8.367697594501718e-07,
      "loss": 0.0009,
      "reward": 1.8457976579666138,
      "reward_std": 0.05379333579912782,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.845797598361969,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.171875,
      "epoch": 0.327319587628866,
      "grad_norm": 1.221630816341642,
      "kl": 0.0252685546875,
      "learning_rate": 8.36340206185567e-07,
      "loss": 0.001,
      "reward": 1.755448818206787,
      "reward_std": 0.12511924654245377,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7710737586021423,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.0390625,
      "epoch": 0.3281786941580756,
      "grad_norm": 1.7175446698553867,
      "kl": 0.0228271484375,
      "learning_rate": 8.359106529209622e-07,
      "loss": 0.0009,
      "reward": 1.8109247088432312,
      "reward_std": 0.10409371182322502,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8187372088432312,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.4296875,
      "epoch": 0.3290378006872852,
      "grad_norm": 1.4545443289160316,
      "kl": 0.0201416015625,
      "learning_rate": 8.354810996563574e-07,
      "loss": 0.0008,
      "reward": 1.8992087244987488,
      "reward_std": 0.07725157961249352,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8992086946964264,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.6015625,
      "epoch": 0.32989690721649484,
      "grad_norm": 2.1100358243584743,
      "kl": 0.0252685546875,
      "learning_rate": 8.350515463917526e-07,
      "loss": 0.001,
      "reward": 1.8205956816673279,
      "reward_std": 0.12146460637450218,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8284082114696503,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.9140625,
      "epoch": 0.33075601374570446,
      "grad_norm": 1.3804484761106177,
      "kl": 0.02508544921875,
      "learning_rate": 8.346219931271478e-07,
      "loss": 0.001,
      "reward": 1.7877458333969116,
      "reward_std": 0.13950634747743607,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7955583333969116,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.703125,
      "epoch": 0.3316151202749141,
      "grad_norm": 1.0084199177905135,
      "kl": 0.0240478515625,
      "learning_rate": 8.34192439862543e-07,
      "loss": 0.001,
      "reward": 1.7355512380599976,
      "reward_std": 0.11318296939134598,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7355512380599976,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.8125,
      "epoch": 0.3324742268041237,
      "grad_norm": 0.8344574864466167,
      "kl": 0.02178955078125,
      "learning_rate": 8.337628865979381e-07,
      "loss": 0.0009,
      "reward": 1.8301225900650024,
      "reward_std": 0.08988046087324619,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.83012256026268,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 121.3203125,
      "epoch": 0.3333333333333333,
      "grad_norm": 0.835913485605179,
      "kl": 0.0225830078125,
      "learning_rate": 8.333333333333333e-07,
      "loss": 0.0009,
      "reward": 1.7770044803619385,
      "reward_std": 0.11324162408709526,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7770044505596161,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 151.6640625,
      "epoch": 0.33419243986254293,
      "grad_norm": 1.0924953323879845,
      "kl": 0.02471923828125,
      "learning_rate": 8.329037800687285e-07,
      "loss": 0.001,
      "reward": 1.7987311482429504,
      "reward_std": 0.08528102748095989,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.806543618440628,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.8984375,
      "epoch": 0.33505154639175255,
      "grad_norm": 0.7425828858782676,
      "kl": 0.02239990234375,
      "learning_rate": 8.324742268041237e-07,
      "loss": 0.0009,
      "reward": 1.8864429593086243,
      "reward_std": 0.09466363862156868,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8942554891109467,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.359375,
      "epoch": 0.33591065292096217,
      "grad_norm": 1.4256854867993622,
      "kl": 0.020355224609375,
      "learning_rate": 8.320446735395189e-07,
      "loss": 0.0008,
      "reward": 1.805371105670929,
      "reward_std": 0.08615696430206299,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8053711354732513,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.6796875,
      "epoch": 0.33676975945017185,
      "grad_norm": 1.2154506440783912,
      "kl": 0.02349853515625,
      "learning_rate": 8.316151202749141e-07,
      "loss": 0.0009,
      "reward": 1.7635123133659363,
      "reward_std": 0.125723734498024,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7713248431682587,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.3203125,
      "epoch": 0.33762886597938147,
      "grad_norm": 1.063545112594486,
      "kl": 0.023193359375,
      "learning_rate": 8.311855670103093e-07,
      "loss": 0.0009,
      "reward": 1.7525449395179749,
      "reward_std": 0.09935981594026089,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7603574395179749,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.2734375,
      "epoch": 0.3384879725085911,
      "grad_norm": 1.1910010378396947,
      "kl": 0.0213623046875,
      "learning_rate": 8.307560137457045e-07,
      "loss": 0.0009,
      "reward": 1.8792944550514221,
      "reward_std": 0.07655412331223488,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8792945444583893,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.4453125,
      "epoch": 0.3393470790378007,
      "grad_norm": 1.0352744087470642,
      "kl": 0.0286865234375,
      "learning_rate": 8.303264604810996e-07,
      "loss": 0.0011,
      "reward": 1.817804753780365,
      "reward_std": 0.09332024306058884,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.825617253780365,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.984375,
      "epoch": 0.3402061855670103,
      "grad_norm": 3.4370614566686366,
      "kl": 0.02947998046875,
      "learning_rate": 8.298969072164948e-07,
      "loss": 0.0012,
      "reward": 1.7064816355705261,
      "reward_std": 0.11401160806417465,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7064816355705261,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.5546875,
      "epoch": 0.34106529209621994,
      "grad_norm": 1.8575760022730172,
      "kl": 0.02227783203125,
      "learning_rate": 8.2946735395189e-07,
      "loss": 0.0009,
      "reward": 1.7460897564888,
      "reward_std": 0.1139952689409256,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7539022862911224,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.09375,
      "epoch": 0.34192439862542956,
      "grad_norm": 1.2996032361936032,
      "kl": 0.02801513671875,
      "learning_rate": 8.290378006872852e-07,
      "loss": 0.0011,
      "reward": 1.8059646487236023,
      "reward_std": 0.1373230665922165,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8215896785259247,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.3984375,
      "epoch": 0.3427835051546392,
      "grad_norm": 1.6501841523869047,
      "kl": 0.0233154296875,
      "learning_rate": 8.286082474226804e-07,
      "loss": 0.0009,
      "reward": 1.809871256351471,
      "reward_std": 0.12983518466353416,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8098713159561157,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.34375,
      "epoch": 0.3436426116838488,
      "grad_norm": 1.111398957842247,
      "kl": 0.02349853515625,
      "learning_rate": 8.281786941580756e-07,
      "loss": 0.0009,
      "reward": 1.8126710057258606,
      "reward_std": 0.0854789987206459,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8204834461212158,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.875,
      "epoch": 0.3445017182130584,
      "grad_norm": 1.5863639489406471,
      "kl": 0.02459716796875,
      "learning_rate": 8.277491408934707e-07,
      "loss": 0.001,
      "reward": 1.7722662687301636,
      "reward_std": 0.0587652251124382,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.772266298532486,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.875,
      "epoch": 0.34536082474226804,
      "grad_norm": 1.1575507587682292,
      "kl": 0.04693603515625,
      "learning_rate": 8.273195876288659e-07,
      "loss": 0.0019,
      "reward": 1.8038502931594849,
      "reward_std": 0.11479467153549194,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8116627931594849,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.015625,
      "epoch": 0.34621993127147765,
      "grad_norm": 1.3175894104086836,
      "kl": 0.02630615234375,
      "learning_rate": 8.26890034364261e-07,
      "loss": 0.0011,
      "reward": 1.8177732825279236,
      "reward_std": 0.061465613543987274,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8177732825279236,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.4140625,
      "epoch": 0.3470790378006873,
      "grad_norm": 0.9613495919689865,
      "kl": 0.03057861328125,
      "learning_rate": 8.264604810996562e-07,
      "loss": 0.0012,
      "reward": 1.856289029121399,
      "reward_std": 0.08307300135493279,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8562889695167542,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 155.5546875,
      "epoch": 0.3479381443298969,
      "grad_norm": 1.2236294083563708,
      "kl": 0.02239990234375,
      "learning_rate": 8.260309278350514e-07,
      "loss": 0.0009,
      "reward": 1.86646169424057,
      "reward_std": 0.05629648268222809,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8664617240428925,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.40625,
      "epoch": 0.3487972508591065,
      "grad_norm": 4.196708925682456,
      "kl": 0.0263671875,
      "learning_rate": 8.256013745704466e-07,
      "loss": 0.0011,
      "reward": 1.6891456842422485,
      "reward_std": 0.13314834237098694,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6969581544399261,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.9765625,
      "epoch": 0.34965635738831613,
      "grad_norm": 3.3390151553242826,
      "kl": 0.02838134765625,
      "learning_rate": 8.251718213058418e-07,
      "loss": 0.0011,
      "reward": 1.7792019844055176,
      "reward_std": 0.08469441719353199,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7792020440101624,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.796875,
      "epoch": 0.35051546391752575,
      "grad_norm": 1.0417856723793362,
      "kl": 0.0313720703125,
      "learning_rate": 8.24742268041237e-07,
      "loss": 0.0013,
      "reward": 1.745001196861267,
      "reward_std": 0.08887993916869164,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7528136670589447,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.2421875,
      "epoch": 0.35137457044673537,
      "grad_norm": 1.5185692601329688,
      "kl": 0.02972412109375,
      "learning_rate": 8.243127147766322e-07,
      "loss": 0.0012,
      "reward": 1.743471622467041,
      "reward_std": 0.12599998712539673,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7434715926647186,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.3828125,
      "epoch": 0.35223367697594504,
      "grad_norm": 3.698150098713745,
      "kl": 0.037353515625,
      "learning_rate": 8.238831615120274e-07,
      "loss": 0.0015,
      "reward": 1.7748391032218933,
      "reward_std": 0.109963808208704,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7826516330242157,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.84375,
      "epoch": 0.35309278350515466,
      "grad_norm": 0.7340273563755917,
      "kl": 0.0301513671875,
      "learning_rate": 8.234536082474227e-07,
      "loss": 0.0012,
      "reward": 1.747934103012085,
      "reward_std": 0.029868584126234055,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7557465732097626,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.140625,
      "epoch": 0.3539518900343643,
      "grad_norm": 6.5869920419752885,
      "kl": 0.03033447265625,
      "learning_rate": 8.230240549828178e-07,
      "loss": 0.0012,
      "reward": 1.803978145122528,
      "reward_std": 0.11175262182950974,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8196031451225281,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.0625,
      "epoch": 0.3548109965635739,
      "grad_norm": 1.2110163382375696,
      "kl": 0.03582763671875,
      "learning_rate": 8.22594501718213e-07,
      "loss": 0.0014,
      "reward": 1.7075645923614502,
      "reward_std": 0.09112687408924103,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7075645625591278,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.2109375,
      "epoch": 0.3556701030927835,
      "grad_norm": 0.9440184144682974,
      "kl": 0.02886962890625,
      "learning_rate": 8.221649484536082e-07,
      "loss": 0.0012,
      "reward": 1.879349708557129,
      "reward_std": 0.077772606164217,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8871622085571289,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 158.203125,
      "epoch": 0.35652920962199314,
      "grad_norm": 1.1844513928260747,
      "kl": 0.024658203125,
      "learning_rate": 8.217353951890034e-07,
      "loss": 0.001,
      "reward": 1.8242889642715454,
      "reward_std": 0.11754781007766724,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8321015238761902,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.9453125,
      "epoch": 0.35738831615120276,
      "grad_norm": 1.1572419610665239,
      "kl": 0.02496337890625,
      "learning_rate": 8.213058419243986e-07,
      "loss": 0.001,
      "reward": 1.7361584305763245,
      "reward_std": 0.0901523232460022,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7439709305763245,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.4140625,
      "epoch": 0.3582474226804124,
      "grad_norm": 1.1187566144341665,
      "kl": 0.02642822265625,
      "learning_rate": 8.208762886597938e-07,
      "loss": 0.0011,
      "reward": 1.787147879600525,
      "reward_std": 0.08959689736366272,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7871478796005249,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.8515625,
      "epoch": 0.359106529209622,
      "grad_norm": 1.469623021469994,
      "kl": 0.0328369140625,
      "learning_rate": 8.20446735395189e-07,
      "loss": 0.0013,
      "reward": 1.7043437957763672,
      "reward_std": 0.13283608853816986,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7043437361717224,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.7890625,
      "epoch": 0.3599656357388316,
      "grad_norm": 1.7756842310117262,
      "kl": 0.03424072265625,
      "learning_rate": 8.200171821305842e-07,
      "loss": 0.0014,
      "reward": 1.8168611526489258,
      "reward_std": 0.09070071205496788,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8168611526489258,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.0078125,
      "epoch": 0.36082474226804123,
      "grad_norm": 1.9241728030153786,
      "kl": 0.03515625,
      "learning_rate": 8.195876288659793e-07,
      "loss": 0.0014,
      "reward": 1.7493263483047485,
      "reward_std": 0.12105973809957504,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7493264377117157,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.203125,
      "epoch": 0.36168384879725085,
      "grad_norm": 2.3633953737617848,
      "kl": 0.0306396484375,
      "learning_rate": 8.191580756013745e-07,
      "loss": 0.0012,
      "reward": 1.7810669541358948,
      "reward_std": 0.09037407860159874,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7810669541358948,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.0078125,
      "epoch": 0.36254295532646047,
      "grad_norm": 1.1965175464927247,
      "kl": 0.0252685546875,
      "learning_rate": 8.187285223367697e-07,
      "loss": 0.001,
      "reward": 1.82912939786911,
      "reward_std": 0.13231520354747772,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8369418382644653,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.6484375,
      "epoch": 0.3634020618556701,
      "grad_norm": 0.9630157435896298,
      "kl": 0.025634765625,
      "learning_rate": 8.182989690721649e-07,
      "loss": 0.001,
      "reward": 1.8102641105651855,
      "reward_std": 0.10265975818037987,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8102641105651855,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.0234375,
      "epoch": 0.3642611683848797,
      "grad_norm": 1.4555619232603647,
      "kl": 0.0316162109375,
      "learning_rate": 8.178694158075601e-07,
      "loss": 0.0013,
      "reward": 1.763112723827362,
      "reward_std": 0.15452294051647186,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7709251940250397,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.6953125,
      "epoch": 0.3651202749140893,
      "grad_norm": 1.6695763498386311,
      "kl": 0.02679443359375,
      "learning_rate": 8.174398625429553e-07,
      "loss": 0.0011,
      "reward": 1.8538659811019897,
      "reward_std": 0.06878830399364233,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8538659512996674,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.7734375,
      "epoch": 0.36597938144329895,
      "grad_norm": 1.714285388160138,
      "kl": 0.02813720703125,
      "learning_rate": 8.170103092783505e-07,
      "loss": 0.0011,
      "reward": 1.7433908581733704,
      "reward_std": 0.11577881872653961,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7512032985687256,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.375,
      "epoch": 0.36683848797250856,
      "grad_norm": 2.0652235283446543,
      "kl": 0.03082275390625,
      "learning_rate": 8.165807560137457e-07,
      "loss": 0.0012,
      "reward": 1.7070344686508179,
      "reward_std": 0.18474984169006348,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7226594686508179,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.4609375,
      "epoch": 0.36769759450171824,
      "grad_norm": 0.952329796803498,
      "kl": 0.03497314453125,
      "learning_rate": 8.161512027491409e-07,
      "loss": 0.0014,
      "reward": 1.8179508447647095,
      "reward_std": 0.06837813928723335,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8179508149623871,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.7578125,
      "epoch": 0.36855670103092786,
      "grad_norm": 1.6252786551878542,
      "kl": 0.0391845703125,
      "learning_rate": 8.15721649484536e-07,
      "loss": 0.0016,
      "reward": 1.7616538405418396,
      "reward_std": 0.0948004387319088,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7694664299488068,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1328125,
      "epoch": 0.3694158075601375,
      "grad_norm": 4.575833798673977,
      "kl": 0.02618408203125,
      "learning_rate": 8.152920962199312e-07,
      "loss": 0.001,
      "reward": 1.7446335554122925,
      "reward_std": 0.14209628105163574,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7446335852146149,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.5078125,
      "epoch": 0.3702749140893471,
      "grad_norm": 1.5029038984828853,
      "kl": 0.027099609375,
      "learning_rate": 8.148625429553264e-07,
      "loss": 0.0011,
      "reward": 1.7977319955825806,
      "reward_std": 0.12650492042303085,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7977319657802582,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.9609375,
      "epoch": 0.3711340206185567,
      "grad_norm": 3.124108570449623,
      "kl": 0.030517578125,
      "learning_rate": 8.144329896907216e-07,
      "loss": 0.0012,
      "reward": 1.763170838356018,
      "reward_std": 0.13480468839406967,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7631707787513733,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.1328125,
      "epoch": 0.37199312714776633,
      "grad_norm": 0.9956952014358291,
      "kl": 0.0367431640625,
      "learning_rate": 8.140034364261168e-07,
      "loss": 0.0015,
      "reward": 1.8633191585540771,
      "reward_std": 0.09405502304434776,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8633190989494324,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.8203125,
      "epoch": 0.37285223367697595,
      "grad_norm": 1.1621034752647021,
      "kl": 0.02874755859375,
      "learning_rate": 8.13573883161512e-07,
      "loss": 0.0012,
      "reward": 1.7654351592063904,
      "reward_std": 0.10451111197471619,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7654351592063904,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.84375,
      "epoch": 0.37371134020618557,
      "grad_norm": 1.0937552701332574,
      "kl": 0.02410888671875,
      "learning_rate": 8.131443298969072e-07,
      "loss": 0.001,
      "reward": 1.7504616975784302,
      "reward_std": 0.13275029510259628,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7582742869853973,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.40625,
      "epoch": 0.3745704467353952,
      "grad_norm": 1.173542025190629,
      "kl": 0.0269775390625,
      "learning_rate": 8.127147766323024e-07,
      "loss": 0.0011,
      "reward": 1.7958321571350098,
      "reward_std": 0.13957487791776657,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7958321869373322,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 155.5859375,
      "epoch": 0.3754295532646048,
      "grad_norm": 1.0809358432591143,
      "kl": 0.0223388671875,
      "learning_rate": 8.122852233676975e-07,
      "loss": 0.0009,
      "reward": 1.8025323748588562,
      "reward_std": 0.10576809197664261,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8103449046611786,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.296875,
      "epoch": 0.37628865979381443,
      "grad_norm": 0.9307591226779286,
      "kl": 0.0291748046875,
      "learning_rate": 8.118556701030927e-07,
      "loss": 0.0012,
      "reward": 1.745145320892334,
      "reward_std": 0.09199497848749161,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7607703506946564,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.3046875,
      "epoch": 0.37714776632302405,
      "grad_norm": 1.599730206934407,
      "kl": 0.025634765625,
      "learning_rate": 8.114261168384879e-07,
      "loss": 0.001,
      "reward": 1.7341384291648865,
      "reward_std": 0.08559693396091461,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7341383695602417,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.046875,
      "epoch": 0.37800687285223367,
      "grad_norm": 1.1302132744799622,
      "kl": 0.0223388671875,
      "learning_rate": 8.109965635738831e-07,
      "loss": 0.0009,
      "reward": 1.7777530550956726,
      "reward_std": 0.11446893960237503,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7855655252933502,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 158.390625,
      "epoch": 0.3788659793814433,
      "grad_norm": 0.8081005861278157,
      "kl": 0.03192138671875,
      "learning_rate": 8.105670103092783e-07,
      "loss": 0.0013,
      "reward": 1.7475718259811401,
      "reward_std": 0.12176471576094627,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7475718557834625,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.1640625,
      "epoch": 0.3797250859106529,
      "grad_norm": 1.4982803588386655,
      "kl": 0.03009033203125,
      "learning_rate": 8.101374570446735e-07,
      "loss": 0.0012,
      "reward": 1.7991742491722107,
      "reward_std": 0.09572646208107471,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8069867491722107,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.40625,
      "epoch": 0.3805841924398625,
      "grad_norm": 4.788054491524054,
      "kl": 0.02618408203125,
      "learning_rate": 8.097079037800687e-07,
      "loss": 0.001,
      "reward": 1.7099648118019104,
      "reward_std": 0.13577891886234283,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.725589781999588,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.9140625,
      "epoch": 0.38144329896907214,
      "grad_norm": 1.0187295128992484,
      "kl": 0.028564453125,
      "learning_rate": 8.092783505154639e-07,
      "loss": 0.0011,
      "reward": 1.7704007029533386,
      "reward_std": 0.07720200531184673,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.770400732755661,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.875,
      "epoch": 0.38230240549828176,
      "grad_norm": 1.617440205079106,
      "kl": 0.0537109375,
      "learning_rate": 8.08848797250859e-07,
      "loss": 0.0022,
      "reward": 1.745563805103302,
      "reward_std": 0.12642088159918785,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7533764243125916,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.46875,
      "epoch": 0.38316151202749144,
      "grad_norm": 1.047930515930632,
      "kl": 0.0303955078125,
      "learning_rate": 8.084192439862542e-07,
      "loss": 0.0012,
      "reward": 1.8345497846603394,
      "reward_std": 0.09045657515525818,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8423622250556946,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.21875,
      "epoch": 0.38402061855670105,
      "grad_norm": 1.4220569982461002,
      "kl": 0.0233154296875,
      "learning_rate": 8.079896907216494e-07,
      "loss": 0.0009,
      "reward": 1.7613440155982971,
      "reward_std": 0.11786005645990372,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7613440155982971,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.1875,
      "epoch": 0.3848797250859107,
      "grad_norm": 1.1920009174782409,
      "kl": 0.02923583984375,
      "learning_rate": 8.075601374570446e-07,
      "loss": 0.0012,
      "reward": 1.7767443656921387,
      "reward_std": 0.1390495002269745,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7845568954944611,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.0703125,
      "epoch": 0.3857388316151203,
      "grad_norm": 0.8285644733256662,
      "kl": 0.0247802734375,
      "learning_rate": 8.071305841924398e-07,
      "loss": 0.001,
      "reward": 1.8258004784584045,
      "reward_std": 0.06260296143591404,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8258004486560822,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.75,
      "epoch": 0.3865979381443299,
      "grad_norm": 1.7038040697442591,
      "kl": 0.026123046875,
      "learning_rate": 8.06701030927835e-07,
      "loss": 0.001,
      "reward": 1.8847057819366455,
      "reward_std": 0.1375606805086136,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.9081432819366455,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 158.8359375,
      "epoch": 0.38745704467353953,
      "grad_norm": 1.4889637606593127,
      "kl": 0.0345458984375,
      "learning_rate": 8.062714776632302e-07,
      "loss": 0.0014,
      "reward": 1.7658740282058716,
      "reward_std": 0.18101423978805542,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7893114984035492,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.59375,
      "epoch": 0.38831615120274915,
      "grad_norm": 1.046812904427794,
      "kl": 0.0262451171875,
      "learning_rate": 8.058419243986254e-07,
      "loss": 0.001,
      "reward": 1.7848870158195496,
      "reward_std": 0.2099006325006485,
      "rewards/format_reward_gen": 0.9375,
      "rewards/llm_reward": 0.8473870158195496,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 156.5625,
      "epoch": 0.38917525773195877,
      "grad_norm": 1.4677940684663608,
      "kl": 0.023193359375,
      "learning_rate": 8.054123711340207e-07,
      "loss": 0.0009,
      "reward": 1.597209393978119,
      "reward_std": 0.26560014486312866,
      "rewards/format_reward_gen": 0.921875,
      "rewards/llm_reward": 0.6753344535827637,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.2265625,
      "epoch": 0.3900343642611684,
      "grad_norm": 2.5890905532205957,
      "kl": 0.02532958984375,
      "learning_rate": 8.049828178694158e-07,
      "loss": 0.001,
      "reward": 1.741255521774292,
      "reward_std": 0.1423398032784462,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.772505521774292,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.7578125,
      "epoch": 0.390893470790378,
      "grad_norm": 1.877091810608803,
      "kl": 0.021728515625,
      "learning_rate": 8.04553264604811e-07,
      "loss": 0.0009,
      "reward": 1.746235966682434,
      "reward_std": 0.1634746640920639,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.7852984368801117,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.0625,
      "epoch": 0.3917525773195876,
      "grad_norm": 1.9322462103510145,
      "kl": 0.030029296875,
      "learning_rate": 8.041237113402062e-07,
      "loss": 0.0012,
      "reward": 1.7314363718032837,
      "reward_std": 0.19341952353715897,
      "rewards/format_reward_gen": 0.9453125,
      "rewards/llm_reward": 0.7861238718032837,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.34375,
      "epoch": 0.39261168384879724,
      "grad_norm": 1.761668192474341,
      "kl": 0.02813720703125,
      "learning_rate": 8.036941580756014e-07,
      "loss": 0.0011,
      "reward": 1.6722000241279602,
      "reward_std": 0.22043447196483612,
      "rewards/format_reward_gen": 0.96875,
      "rewards/llm_reward": 0.7034500241279602,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 177.1484375,
      "epoch": 0.39347079037800686,
      "grad_norm": 1.103623434836374,
      "kl": 0.021484375,
      "learning_rate": 8.032646048109966e-07,
      "loss": 0.0009,
      "reward": 1.7857837677001953,
      "reward_std": 0.16772552579641342,
      "rewards/format_reward_gen": 0.953125,
      "rewards/llm_reward": 0.8326588273048401,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.78125,
      "epoch": 0.3943298969072165,
      "grad_norm": 2.9558490613021475,
      "kl": 0.02484130859375,
      "learning_rate": 8.028350515463918e-07,
      "loss": 0.001,
      "reward": 1.8172647356987,
      "reward_std": 0.1397656500339508,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8172646760940552,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.109375,
      "epoch": 0.3951890034364261,
      "grad_norm": 1.0942493174264873,
      "kl": 0.02008056640625,
      "learning_rate": 8.02405498281787e-07,
      "loss": 0.0008,
      "reward": 1.7567135095596313,
      "reward_std": 0.08268255367875099,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.764525979757309,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.4140625,
      "epoch": 0.3960481099656357,
      "grad_norm": 1.724231638784223,
      "kl": 0.02337646484375,
      "learning_rate": 8.019759450171822e-07,
      "loss": 0.0009,
      "reward": 1.812518298625946,
      "reward_std": 0.1107301339507103,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8281432688236237,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.875,
      "epoch": 0.39690721649484534,
      "grad_norm": 0.8421480615888612,
      "kl": 0.02001953125,
      "learning_rate": 8.015463917525774e-07,
      "loss": 0.0008,
      "reward": 1.8146081566810608,
      "reward_std": 0.09295319765806198,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8224206566810608,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.2265625,
      "epoch": 0.39776632302405496,
      "grad_norm": 0.9758819512120601,
      "kl": 0.017822265625,
      "learning_rate": 8.011168384879725e-07,
      "loss": 0.0007,
      "reward": 1.7702763676643372,
      "reward_std": 0.10476777702569962,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7780888676643372,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.234375,
      "epoch": 0.39862542955326463,
      "grad_norm": 0.9223788317682841,
      "kl": 0.02032470703125,
      "learning_rate": 8.006872852233677e-07,
      "loss": 0.0008,
      "reward": 1.8337781429290771,
      "reward_std": 0.10091580078005791,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8337781429290771,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.0625,
      "epoch": 0.39948453608247425,
      "grad_norm": 1.1306004675623413,
      "kl": 0.02081298828125,
      "learning_rate": 8.002577319587629e-07,
      "loss": 0.0008,
      "reward": 1.6799623370170593,
      "reward_std": 0.1472223922610283,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6955873966217041,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.28125,
      "epoch": 0.40034364261168387,
      "grad_norm": 2.7421308952943497,
      "kl": 0.02642822265625,
      "learning_rate": 7.998281786941581e-07,
      "loss": 0.0011,
      "reward": 1.7180203199386597,
      "reward_std": 0.16847053170204163,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7336452901363373,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.8984375,
      "epoch": 0.4012027491408935,
      "grad_norm": 1.1905386765274455,
      "kl": 0.02166748046875,
      "learning_rate": 7.993986254295533e-07,
      "loss": 0.0009,
      "reward": 1.824146568775177,
      "reward_std": 0.09475822001695633,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8319591283798218,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.828125,
      "epoch": 0.4020618556701031,
      "grad_norm": 1.0030306949143268,
      "kl": 0.02276611328125,
      "learning_rate": 7.989690721649485e-07,
      "loss": 0.0009,
      "reward": 1.6890740394592285,
      "reward_std": 0.08577071130275726,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7046990394592285,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.390625,
      "epoch": 0.4029209621993127,
      "grad_norm": 0.6829732849279407,
      "kl": 0.020751953125,
      "learning_rate": 7.985395189003437e-07,
      "loss": 0.0008,
      "reward": 1.8722723126411438,
      "reward_std": 0.03336894512176514,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8722723126411438,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.703125,
      "epoch": 0.40378006872852235,
      "grad_norm": 3.6315379563427315,
      "kl": 0.08935546875,
      "learning_rate": 7.981099656357389e-07,
      "loss": 0.0036,
      "reward": 1.669398546218872,
      "reward_std": 0.16622114926576614,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6772109866142273,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.3828125,
      "epoch": 0.40463917525773196,
      "grad_norm": 2.201255320619489,
      "kl": 0.02001953125,
      "learning_rate": 7.97680412371134e-07,
      "loss": 0.0008,
      "reward": 1.689961850643158,
      "reward_std": 0.21605655923485756,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6977743208408356,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.15625,
      "epoch": 0.4054982817869416,
      "grad_norm": 7.197996250278638,
      "kl": 0.02166748046875,
      "learning_rate": 7.972508591065292e-07,
      "loss": 0.0009,
      "reward": 1.8866642117500305,
      "reward_std": 0.0431262981146574,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8866641819477081,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.5625,
      "epoch": 0.4063573883161512,
      "grad_norm": 0.9395439850082784,
      "kl": 0.02435302734375,
      "learning_rate": 7.968213058419243e-07,
      "loss": 0.001,
      "reward": 1.7944100499153137,
      "reward_std": 0.07158202305436134,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7944100797176361,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 127.46875,
      "epoch": 0.4072164948453608,
      "grad_norm": 10.764240409904978,
      "kl": 0.02557373046875,
      "learning_rate": 7.963917525773195e-07,
      "loss": 0.001,
      "reward": 1.7304428219795227,
      "reward_std": 0.19595059752464294,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7382553517818451,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 123.9140625,
      "epoch": 0.40807560137457044,
      "grad_norm": 1.4034901060673528,
      "kl": 0.02447509765625,
      "learning_rate": 7.959621993127147e-07,
      "loss": 0.001,
      "reward": 1.7872547507286072,
      "reward_std": 0.12548959627747536,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7950672507286072,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.5546875,
      "epoch": 0.40893470790378006,
      "grad_norm": 1.8853927915690178,
      "kl": 0.0291748046875,
      "learning_rate": 7.955326460481099e-07,
      "loss": 0.0012,
      "reward": 1.6977566480636597,
      "reward_std": 0.14235452935099602,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7055690884590149,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.859375,
      "epoch": 0.4097938144329897,
      "grad_norm": 2.0359538756787243,
      "kl": 0.02093505859375,
      "learning_rate": 7.951030927835051e-07,
      "loss": 0.0008,
      "reward": 1.8194365501403809,
      "reward_std": 0.14543933421373367,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8194365203380585,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.0625,
      "epoch": 0.4106529209621993,
      "grad_norm": 2.3120838651943827,
      "kl": 0.02349853515625,
      "learning_rate": 7.946735395189003e-07,
      "loss": 0.0009,
      "reward": 1.7549501657485962,
      "reward_std": 0.17499929666519165,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7627626061439514,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.671875,
      "epoch": 0.4115120274914089,
      "grad_norm": 0.846611479751614,
      "kl": 0.01953125,
      "learning_rate": 7.942439862542954e-07,
      "loss": 0.0008,
      "reward": 1.761724591255188,
      "reward_std": 0.1066037192940712,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7695370316505432,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.7421875,
      "epoch": 0.41237113402061853,
      "grad_norm": 1.0153354601072486,
      "kl": 0.021728515625,
      "learning_rate": 7.938144329896906e-07,
      "loss": 0.0009,
      "reward": 1.8040355443954468,
      "reward_std": 0.09103575721383095,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8118480145931244,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.34375,
      "epoch": 0.41323024054982815,
      "grad_norm": 1.3316805075661329,
      "kl": 0.03472900390625,
      "learning_rate": 7.933848797250858e-07,
      "loss": 0.0014,
      "reward": 1.8269373178482056,
      "reward_std": 0.0393183808773756,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8269373178482056,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.4453125,
      "epoch": 0.41408934707903783,
      "grad_norm": 1.2169011867171224,
      "kl": 0.0244140625,
      "learning_rate": 7.92955326460481e-07,
      "loss": 0.001,
      "reward": 1.808565378189087,
      "reward_std": 0.10053881630301476,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8163778781890869,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.3671875,
      "epoch": 0.41494845360824745,
      "grad_norm": 1.9556380092566035,
      "kl": 0.02239990234375,
      "learning_rate": 7.925257731958762e-07,
      "loss": 0.0009,
      "reward": 1.7945858240127563,
      "reward_std": 0.1085839495062828,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7945858240127563,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.2890625,
      "epoch": 0.41580756013745707,
      "grad_norm": 1.4377206374386469,
      "kl": 0.0299072265625,
      "learning_rate": 7.920962199312714e-07,
      "loss": 0.0012,
      "reward": 1.7909988164901733,
      "reward_std": 0.10544274002313614,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8066238462924957,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.484375,
      "epoch": 0.4166666666666667,
      "grad_norm": 1.1637028605621145,
      "kl": 0.0244140625,
      "learning_rate": 7.916666666666666e-07,
      "loss": 0.001,
      "reward": 1.8535531759262085,
      "reward_std": 0.09841496497392654,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8535531163215637,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.296875,
      "epoch": 0.4175257731958763,
      "grad_norm": 0.9089860152674513,
      "kl": 0.0196533203125,
      "learning_rate": 7.912371134020618e-07,
      "loss": 0.0008,
      "reward": 1.8009296655654907,
      "reward_std": 0.08241390809416771,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.800929605960846,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.875,
      "epoch": 0.4183848797250859,
      "grad_norm": 1.1175991581337885,
      "kl": 0.02105712890625,
      "learning_rate": 7.908075601374569e-07,
      "loss": 0.0008,
      "reward": 1.7471742033958435,
      "reward_std": 0.09226182475686073,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7471742033958435,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 138.65625,
      "epoch": 0.41924398625429554,
      "grad_norm": 1.5782041969321439,
      "kl": 0.0245361328125,
      "learning_rate": 7.903780068728521e-07,
      "loss": 0.001,
      "reward": 1.7752417922019958,
      "reward_std": 0.07173944637179375,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7752417623996735,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 126.328125,
      "epoch": 0.42010309278350516,
      "grad_norm": 1.712440072733616,
      "kl": 0.02490234375,
      "learning_rate": 7.899484536082473e-07,
      "loss": 0.001,
      "reward": 1.7801220417022705,
      "reward_std": 0.1850130558013916,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7957470417022705,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.96875,
      "epoch": 0.4209621993127148,
      "grad_norm": 1.1903287634500803,
      "kl": 0.021484375,
      "learning_rate": 7.895189003436425e-07,
      "loss": 0.0009,
      "reward": 1.797283411026001,
      "reward_std": 0.15069201588630676,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.797283411026001,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.5390625,
      "epoch": 0.4218213058419244,
      "grad_norm": 1.3823311832547454,
      "kl": 0.02471923828125,
      "learning_rate": 7.890893470790377e-07,
      "loss": 0.001,
      "reward": 1.6768372654914856,
      "reward_std": 0.14700639992952347,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6846497654914856,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.8828125,
      "epoch": 0.422680412371134,
      "grad_norm": 1.052318496217373,
      "kl": 0.02825927734375,
      "learning_rate": 7.886597938144329e-07,
      "loss": 0.0011,
      "reward": 1.7353659868240356,
      "reward_std": 0.10273649916052818,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.735366016626358,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.234375,
      "epoch": 0.42353951890034364,
      "grad_norm": 0.9659208234974426,
      "kl": 0.0198974609375,
      "learning_rate": 7.882302405498281e-07,
      "loss": 0.0008,
      "reward": 1.6551663875579834,
      "reward_std": 0.10088075697422028,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.670791357755661,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.6796875,
      "epoch": 0.42439862542955326,
      "grad_norm": 3.2851303393945175,
      "kl": 0.0211181640625,
      "learning_rate": 7.878006872852233e-07,
      "loss": 0.0008,
      "reward": 1.7861362099647522,
      "reward_std": 0.1402701437473297,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.786136269569397,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.0390625,
      "epoch": 0.4252577319587629,
      "grad_norm": 1.3709194983778912,
      "kl": 0.02056884765625,
      "learning_rate": 7.873711340206184e-07,
      "loss": 0.0008,
      "reward": 1.7891325950622559,
      "reward_std": 0.15913131088018417,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7891325950622559,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.5078125,
      "epoch": 0.4261168384879725,
      "grad_norm": 1.6084405058813327,
      "kl": 0.02093505859375,
      "learning_rate": 7.869415807560136e-07,
      "loss": 0.0008,
      "reward": 1.843230962753296,
      "reward_std": 0.11450556665658951,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8432309925556183,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.6015625,
      "epoch": 0.4269759450171821,
      "grad_norm": 0.9098919677048347,
      "kl": 0.0244140625,
      "learning_rate": 7.865120274914089e-07,
      "loss": 0.001,
      "reward": 1.8330826163291931,
      "reward_std": 0.13003884255886078,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8330826163291931,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.03125,
      "epoch": 0.42783505154639173,
      "grad_norm": 1.1077984476897802,
      "kl": 0.0185546875,
      "learning_rate": 7.860824742268041e-07,
      "loss": 0.0007,
      "reward": 1.7646691799163818,
      "reward_std": 0.09061707556247711,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7646692395210266,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.5,
      "epoch": 0.42869415807560135,
      "grad_norm": 1.5301258115042293,
      "kl": 0.021728515625,
      "learning_rate": 7.856529209621993e-07,
      "loss": 0.0009,
      "reward": 1.8485521078109741,
      "reward_std": 0.14281748235225677,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8485520780086517,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.15625,
      "epoch": 0.42955326460481097,
      "grad_norm": 0.6854626662078912,
      "kl": 0.0198974609375,
      "learning_rate": 7.852233676975945e-07,
      "loss": 0.0008,
      "reward": 1.8705613613128662,
      "reward_std": 0.057761300355196,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8705613017082214,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 156.5,
      "epoch": 0.43041237113402064,
      "grad_norm": 4.684754333390039,
      "kl": 0.02490234375,
      "learning_rate": 7.847938144329897e-07,
      "loss": 0.001,
      "reward": 1.7904819250106812,
      "reward_std": 0.11770961433649063,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.790481835603714,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.1015625,
      "epoch": 0.43127147766323026,
      "grad_norm": 1.3603984953615473,
      "kl": 0.0272216796875,
      "learning_rate": 7.843642611683849e-07,
      "loss": 0.0011,
      "reward": 1.8305404782295227,
      "reward_std": 0.08736425265669823,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8305404186248779,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.171875,
      "epoch": 0.4321305841924399,
      "grad_norm": 1.1418674217309672,
      "kl": 0.026123046875,
      "learning_rate": 7.839347079037801e-07,
      "loss": 0.001,
      "reward": 1.7259817123413086,
      "reward_std": 0.16148880869150162,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7416067719459534,
      "step": 503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 154.3359375,
      "epoch": 0.4329896907216495,
      "grad_norm": 1.1135192490403159,
      "kl": 0.0301513671875,
      "learning_rate": 7.835051546391752e-07,
      "loss": 0.0012,
      "reward": 1.675801932811737,
      "reward_std": 0.10446542128920555,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.6758019924163818,
      "step": 504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 158.8984375,
      "epoch": 0.4338487972508591,
      "grad_norm": 1.0026442135683145,
      "kl": 0.02264404296875,
      "learning_rate": 7.830756013745704e-07,
      "loss": 0.0009,
      "reward": 1.753090739250183,
      "reward_std": 0.11342515423893929,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7687157094478607,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 156.4609375,
      "epoch": 0.43470790378006874,
      "grad_norm": 0.63327553743734,
      "kl": 0.0238037109375,
      "learning_rate": 7.826460481099656e-07,
      "loss": 0.001,
      "reward": 1.7929630875587463,
      "reward_std": 0.06263483129441738,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8007755875587463,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.015625,
      "epoch": 0.43556701030927836,
      "grad_norm": 1.1405571467224787,
      "kl": 0.02972412109375,
      "learning_rate": 7.822164948453608e-07,
      "loss": 0.0012,
      "reward": 1.8316816091537476,
      "reward_std": 0.13786949962377548,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8394941091537476,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.7109375,
      "epoch": 0.436426116838488,
      "grad_norm": 2.491831127237855,
      "kl": 0.0272216796875,
      "learning_rate": 7.81786941580756e-07,
      "loss": 0.0011,
      "reward": 1.8621906638145447,
      "reward_std": 0.08054150827229023,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8621906936168671,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 145.6640625,
      "epoch": 0.4372852233676976,
      "grad_norm": 5.202435035199377,
      "kl": 0.02520751953125,
      "learning_rate": 7.813573883161512e-07,
      "loss": 0.001,
      "reward": 1.6291198134422302,
      "reward_std": 0.2496250867843628,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6447448432445526,
      "step": 509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 168.59375,
      "epoch": 0.4381443298969072,
      "grad_norm": 1.6848389247112312,
      "kl": 0.02227783203125,
      "learning_rate": 7.809278350515464e-07,
      "loss": 0.0009,
      "reward": 1.8260024189949036,
      "reward_std": 0.0773128978908062,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8260024785995483,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.8671875,
      "epoch": 0.43900343642611683,
      "grad_norm": 1.2536511830670496,
      "kl": 0.04034423828125,
      "learning_rate": 7.804982817869416e-07,
      "loss": 0.0016,
      "reward": 1.8234487175941467,
      "reward_std": 0.12906832993030548,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8390736877918243,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 162.9609375,
      "epoch": 0.43986254295532645,
      "grad_norm": 0.9056562066002685,
      "kl": 0.0228271484375,
      "learning_rate": 7.800687285223368e-07,
      "loss": 0.0009,
      "reward": 1.8035598993301392,
      "reward_std": 0.10758433863520622,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.819184809923172,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 160.625,
      "epoch": 0.44072164948453607,
      "grad_norm": 1.9749807757861202,
      "kl": 0.0284423828125,
      "learning_rate": 7.796391752577319e-07,
      "loss": 0.0011,
      "reward": 1.7375337481498718,
      "reward_std": 0.12667454779148102,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7375337779521942,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 177.203125,
      "epoch": 0.4415807560137457,
      "grad_norm": 2.7049577625952863,
      "kl": 0.025634765625,
      "learning_rate": 7.792096219931271e-07,
      "loss": 0.001,
      "reward": 1.665591835975647,
      "reward_std": 0.1844305470585823,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.6890293061733246,
      "step": 514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 176.421875,
      "epoch": 0.4424398625429553,
      "grad_norm": 1.1503646554833182,
      "kl": 0.020751953125,
      "learning_rate": 7.787800687285223e-07,
      "loss": 0.0008,
      "reward": 1.7526438236236572,
      "reward_std": 0.15297304093837738,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7526438236236572,
      "step": 515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 174.8203125,
      "epoch": 0.44329896907216493,
      "grad_norm": 2.528762349405692,
      "kl": 0.0223388671875,
      "learning_rate": 7.783505154639175e-07,
      "loss": 0.0009,
      "reward": 1.7877247333526611,
      "reward_std": 0.1644352711737156,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7877247333526611,
      "step": 516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 161.5859375,
      "epoch": 0.44415807560137455,
      "grad_norm": 1.0883366488533088,
      "kl": 0.02557373046875,
      "learning_rate": 7.779209621993127e-07,
      "loss": 0.001,
      "reward": 1.9020284414291382,
      "reward_std": 0.0875653550028801,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.9098409414291382,
      "step": 517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 168.234375,
      "epoch": 0.44501718213058417,
      "grad_norm": 1.238117408635782,
      "kl": 0.02325439453125,
      "learning_rate": 7.774914089347079e-07,
      "loss": 0.0009,
      "reward": 1.7550345659255981,
      "reward_std": 0.11340263113379478,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7706595659255981,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 161.953125,
      "epoch": 0.44587628865979384,
      "grad_norm": 3.2324811204812818,
      "kl": 0.02435302734375,
      "learning_rate": 7.770618556701031e-07,
      "loss": 0.001,
      "reward": 1.7546863555908203,
      "reward_std": 0.15567786246538162,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7624987959861755,
      "step": 519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 159.5390625,
      "epoch": 0.44673539518900346,
      "grad_norm": 1.1906445700635893,
      "kl": 0.0238037109375,
      "learning_rate": 7.766323024054983e-07,
      "loss": 0.001,
      "reward": 1.770099401473999,
      "reward_std": 0.15499432384967804,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7700993120670319,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 155.3359375,
      "epoch": 0.4475945017182131,
      "grad_norm": 1.4289037451333348,
      "kl": 0.02899169921875,
      "learning_rate": 7.762027491408934e-07,
      "loss": 0.0012,
      "reward": 1.7591895461082458,
      "reward_std": 0.20220742374658585,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7826270163059235,
      "step": 521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.859375,
      "epoch": 0.4484536082474227,
      "grad_norm": 0.9798567354541009,
      "kl": 0.0318603515625,
      "learning_rate": 7.757731958762886e-07,
      "loss": 0.0013,
      "reward": 1.8011258244514465,
      "reward_std": 0.1115933284163475,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8089383244514465,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 162.453125,
      "epoch": 0.4493127147766323,
      "grad_norm": 0.9870529145672341,
      "kl": 0.0245361328125,
      "learning_rate": 7.753436426116838e-07,
      "loss": 0.001,
      "reward": 1.7835025191307068,
      "reward_std": 0.10335457697510719,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7913150489330292,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.1796875,
      "epoch": 0.45017182130584193,
      "grad_norm": 1.590053867640443,
      "kl": 0.02679443359375,
      "learning_rate": 7.74914089347079e-07,
      "loss": 0.0011,
      "reward": 1.6806529760360718,
      "reward_std": 0.12486770376563072,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.6962779760360718,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.203125,
      "epoch": 0.45103092783505155,
      "grad_norm": 1.288976196272062,
      "kl": 0.0467529296875,
      "learning_rate": 7.744845360824742e-07,
      "loss": 0.0019,
      "reward": 1.7338435053825378,
      "reward_std": 0.21027009189128876,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7494684457778931,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.875,
      "epoch": 0.4518900343642612,
      "grad_norm": 1.3796315870979017,
      "kl": 0.03314208984375,
      "learning_rate": 7.740549828178694e-07,
      "loss": 0.0013,
      "reward": 1.813571572303772,
      "reward_std": 0.11932007595896721,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8135715425014496,
      "step": 526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 155.8359375,
      "epoch": 0.4527491408934708,
      "grad_norm": 1.0856638820035733,
      "kl": 0.02392578125,
      "learning_rate": 7.736254295532646e-07,
      "loss": 0.001,
      "reward": 1.7808946371078491,
      "reward_std": 0.15631069988012314,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7887071967124939,
      "step": 527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.53125,
      "epoch": 0.4536082474226804,
      "grad_norm": 1.2741002088470732,
      "kl": 0.0306396484375,
      "learning_rate": 7.731958762886598e-07,
      "loss": 0.0012,
      "reward": 1.7552229762077332,
      "reward_std": 0.16508981585502625,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7708480358123779,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 163.84375,
      "epoch": 0.45446735395189003,
      "grad_norm": 0.8166293208289542,
      "kl": 0.0262451171875,
      "learning_rate": 7.72766323024055e-07,
      "loss": 0.0011,
      "reward": 1.763866662979126,
      "reward_std": 0.11001107096672058,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.771679162979126,
      "step": 529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.6171875,
      "epoch": 0.45532646048109965,
      "grad_norm": 1.0906821881418176,
      "kl": 0.0263671875,
      "learning_rate": 7.723367697594501e-07,
      "loss": 0.0011,
      "reward": 1.7570544481277466,
      "reward_std": 0.09293561428785324,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7570544183254242,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 152.1328125,
      "epoch": 0.45618556701030927,
      "grad_norm": 0.6788651581622658,
      "kl": 0.0257568359375,
      "learning_rate": 7.719072164948453e-07,
      "loss": 0.001,
      "reward": 1.8436721563339233,
      "reward_std": 0.05900268629193306,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8514846563339233,
      "step": 531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.203125,
      "epoch": 0.4570446735395189,
      "grad_norm": 1.3523482508260602,
      "kl": 0.02862548828125,
      "learning_rate": 7.714776632302405e-07,
      "loss": 0.0011,
      "reward": 1.7728696465492249,
      "reward_std": 0.0960187017917633,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7728695869445801,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 157.2578125,
      "epoch": 0.4579037800687285,
      "grad_norm": 2.080449621498438,
      "kl": 0.02349853515625,
      "learning_rate": 7.710481099656357e-07,
      "loss": 0.0009,
      "reward": 1.7926985025405884,
      "reward_std": 0.10765637829899788,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7926985621452332,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 157.546875,
      "epoch": 0.4587628865979381,
      "grad_norm": 0.9990312365514723,
      "kl": 0.030029296875,
      "learning_rate": 7.706185567010309e-07,
      "loss": 0.0012,
      "reward": 1.6698935627937317,
      "reward_std": 0.1365356780588627,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6777060627937317,
      "step": 534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 147.921875,
      "epoch": 0.45962199312714774,
      "grad_norm": 1.423339665969349,
      "kl": 0.02587890625,
      "learning_rate": 7.701890034364261e-07,
      "loss": 0.001,
      "reward": 1.7801197171211243,
      "reward_std": 0.12286467850208282,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.780119776725769,
      "step": 535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 155.1328125,
      "epoch": 0.46048109965635736,
      "grad_norm": 2.0644152976523924,
      "kl": 0.0234375,
      "learning_rate": 7.697594501718213e-07,
      "loss": 0.0009,
      "reward": 1.8145374059677124,
      "reward_std": 0.15781568735837936,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8223499655723572,
      "step": 536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.828125,
      "epoch": 0.46134020618556704,
      "grad_norm": 2.419979165046296,
      "kl": 0.03106689453125,
      "learning_rate": 7.693298969072165e-07,
      "loss": 0.0012,
      "reward": 1.7789837718009949,
      "reward_std": 0.1027519591152668,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7789837718009949,
      "step": 537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.5859375,
      "epoch": 0.46219931271477666,
      "grad_norm": 0.9119452557469206,
      "kl": 0.02142333984375,
      "learning_rate": 7.689003436426116e-07,
      "loss": 0.0009,
      "reward": 1.8047268986701965,
      "reward_std": 0.0921391174197197,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8047268688678741,
      "step": 538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.75,
      "epoch": 0.4630584192439863,
      "grad_norm": 1.0752489048608882,
      "kl": 0.02978515625,
      "learning_rate": 7.684707903780069e-07,
      "loss": 0.0012,
      "reward": 1.7013397812843323,
      "reward_std": 0.10420958697795868,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7013397514820099,
      "step": 539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.21875,
      "epoch": 0.4639175257731959,
      "grad_norm": 1.879235291737055,
      "kl": 0.03424072265625,
      "learning_rate": 7.680412371134021e-07,
      "loss": 0.0014,
      "reward": 1.7587386965751648,
      "reward_std": 0.12074919883161783,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7665511965751648,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 144.1640625,
      "epoch": 0.4647766323024055,
      "grad_norm": 1.0694029027143328,
      "kl": 0.02728271484375,
      "learning_rate": 7.676116838487973e-07,
      "loss": 0.0011,
      "reward": 1.7316040396690369,
      "reward_std": 0.1069042906165123,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7316040396690369,
      "step": 541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.390625,
      "epoch": 0.46563573883161513,
      "grad_norm": 1.0560010325147446,
      "kl": 0.027099609375,
      "learning_rate": 7.671821305841925e-07,
      "loss": 0.0011,
      "reward": 1.7262669205665588,
      "reward_std": 0.1186705082654953,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7340793907642365,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.375,
      "epoch": 0.46649484536082475,
      "grad_norm": 0.8831242345243586,
      "kl": 0.02362060546875,
      "learning_rate": 7.667525773195877e-07,
      "loss": 0.0009,
      "reward": 1.8377864360809326,
      "reward_std": 0.07232390902936459,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8455990254878998,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.5625,
      "epoch": 0.46735395189003437,
      "grad_norm": 0.808526359455059,
      "kl": 0.025634765625,
      "learning_rate": 7.663230240549829e-07,
      "loss": 0.001,
      "reward": 1.8943662643432617,
      "reward_std": 0.08304097317159176,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8943662643432617,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.8359375,
      "epoch": 0.468213058419244,
      "grad_norm": 1.2364679060956676,
      "kl": 0.03729248046875,
      "learning_rate": 7.658934707903781e-07,
      "loss": 0.0015,
      "reward": 1.820011556148529,
      "reward_std": 0.10274770110845566,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8278240263462067,
      "step": 545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 165.7421875,
      "epoch": 0.4690721649484536,
      "grad_norm": 1.4203606446988248,
      "kl": 0.02764892578125,
      "learning_rate": 7.654639175257731e-07,
      "loss": 0.0011,
      "reward": 1.7455325722694397,
      "reward_std": 0.08128118142485619,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7455325424671173,
      "step": 546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.140625,
      "epoch": 0.4699312714776632,
      "grad_norm": 2.804119845115769,
      "kl": 0.03338623046875,
      "learning_rate": 7.650343642611683e-07,
      "loss": 0.0013,
      "reward": 1.6912947297096252,
      "reward_std": 0.1474270112812519,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.71473228931427,
      "step": 547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.09375,
      "epoch": 0.47079037800687284,
      "grad_norm": 1.9971642189793117,
      "kl": 0.07635498046875,
      "learning_rate": 7.646048109965635e-07,
      "loss": 0.0031,
      "reward": 1.7125971913337708,
      "reward_std": 0.17911146581172943,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.7360346615314484,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.2734375,
      "epoch": 0.47164948453608246,
      "grad_norm": 2.745551651788921,
      "kl": 0.02740478515625,
      "learning_rate": 7.641752577319587e-07,
      "loss": 0.0011,
      "reward": 1.8493961691856384,
      "reward_std": 0.10139979794621468,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8493961095809937,
      "step": 549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.359375,
      "epoch": 0.4725085910652921,
      "grad_norm": 3.8143164658012823,
      "kl": 0.02508544921875,
      "learning_rate": 7.637457044673539e-07,
      "loss": 0.001,
      "reward": 1.717983603477478,
      "reward_std": 0.11418138444423676,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7336086332798004,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 142.8671875,
      "epoch": 0.4733676975945017,
      "grad_norm": 1.0145675064339164,
      "kl": 0.02392578125,
      "learning_rate": 7.633161512027491e-07,
      "loss": 0.001,
      "reward": 1.7650344371795654,
      "reward_std": 0.07981048338115215,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7728469967842102,
      "step": 551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.8515625,
      "epoch": 0.4742268041237113,
      "grad_norm": 18.881776716124634,
      "kl": 0.0201416015625,
      "learning_rate": 7.628865979381443e-07,
      "loss": 0.0008,
      "reward": 1.7701881527900696,
      "reward_std": 0.17208566516637802,
      "rewards/format_reward_gen": 0.953125,
      "rewards/llm_reward": 0.817063182592392,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 148.9765625,
      "epoch": 0.47508591065292094,
      "grad_norm": 2.102100470034523,
      "kl": 0.02630615234375,
      "learning_rate": 7.624570446735395e-07,
      "loss": 0.0011,
      "reward": 1.8304003477096558,
      "reward_std": 0.1711219921708107,
      "rewards/format_reward_gen": 0.921875,
      "rewards/llm_reward": 0.908525288105011,
      "step": 553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 152.96875,
      "epoch": 0.47594501718213056,
      "grad_norm": 1.444717715857585,
      "kl": 0.021728515625,
      "learning_rate": 7.620274914089346e-07,
      "loss": 0.0009,
      "reward": 1.806252360343933,
      "reward_std": 0.1447359174489975,
      "rewards/format_reward_gen": 0.953125,
      "rewards/llm_reward": 0.8531273901462555,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.8828125,
      "epoch": 0.47680412371134023,
      "grad_norm": 2.156085264005566,
      "kl": 0.02764892578125,
      "learning_rate": 7.615979381443298e-07,
      "loss": 0.0011,
      "reward": 1.6459792852401733,
      "reward_std": 0.2492372989654541,
      "rewards/format_reward_gen": 0.921875,
      "rewards/llm_reward": 0.7241042256355286,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.4453125,
      "epoch": 0.47766323024054985,
      "grad_norm": 2.1944379120092123,
      "kl": 0.0213623046875,
      "learning_rate": 7.61168384879725e-07,
      "loss": 0.0009,
      "reward": 1.7091248035430908,
      "reward_std": 0.16284260153770447,
      "rewards/format_reward_gen": 0.953125,
      "rewards/llm_reward": 0.7559998035430908,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.359375,
      "epoch": 0.47852233676975947,
      "grad_norm": 1.607470492713128,
      "kl": 0.0281982421875,
      "learning_rate": 7.607388316151202e-07,
      "loss": 0.0011,
      "reward": 1.687252402305603,
      "reward_std": 0.16054657474160194,
      "rewards/format_reward_gen": 0.9765625,
      "rewards/llm_reward": 0.710689902305603,
      "step": 557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 125.53125,
      "epoch": 0.4793814432989691,
      "grad_norm": 0.9757519931508962,
      "kl": 0.0283203125,
      "learning_rate": 7.603092783505154e-07,
      "loss": 0.0011,
      "reward": 1.8052751421928406,
      "reward_std": 0.05359087511897087,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8209001123905182,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.125,
      "epoch": 0.4802405498281787,
      "grad_norm": 1.657372250071319,
      "kl": 0.03369140625,
      "learning_rate": 7.598797250859106e-07,
      "loss": 0.0013,
      "reward": 1.6079121828079224,
      "reward_std": 0.22965000569820404,
      "rewards/format_reward_gen": 0.9140625,
      "rewards/llm_reward": 0.6938497424125671,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.34375,
      "epoch": 0.48109965635738833,
      "grad_norm": 1.3334146432657494,
      "kl": 0.0301513671875,
      "learning_rate": 7.594501718213058e-07,
      "loss": 0.0012,
      "reward": 1.724092185497284,
      "reward_std": 0.12364434078335762,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.7631546556949615,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 107.3515625,
      "epoch": 0.48195876288659795,
      "grad_norm": 1.3103330552745027,
      "kl": 0.03033447265625,
      "learning_rate": 7.59020618556701e-07,
      "loss": 0.0012,
      "reward": 1.7695825695991516,
      "reward_std": 0.16746646538376808,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.8086450695991516,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.21875,
      "epoch": 0.48281786941580757,
      "grad_norm": 3.4007488158636634,
      "kl": 0.0413818359375,
      "learning_rate": 7.585910652920962e-07,
      "loss": 0.0017,
      "reward": 1.8109328150749207,
      "reward_std": 0.17322580516338348,
      "rewards/format_reward_gen": 0.9609375,
      "rewards/llm_reward": 0.8499953150749207,
      "step": 562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.859375,
      "epoch": 0.4836769759450172,
      "grad_norm": 4.0862178779047085,
      "kl": 0.02703857421875,
      "learning_rate": 7.581615120274913e-07,
      "loss": 0.0011,
      "reward": 1.7667149305343628,
      "reward_std": 0.08758777007460594,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7667149305343628,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.5859375,
      "epoch": 0.4845360824742268,
      "grad_norm": 1.073396750076447,
      "kl": 0.0284423828125,
      "learning_rate": 7.577319587628865e-07,
      "loss": 0.0011,
      "reward": 1.7671363949775696,
      "reward_std": 0.10686640068888664,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.774948924779892,
      "step": 564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.7109375,
      "epoch": 0.4853951890034364,
      "grad_norm": 1.6082673896426019,
      "kl": 0.02496337890625,
      "learning_rate": 7.573024054982817e-07,
      "loss": 0.001,
      "reward": 1.7323017120361328,
      "reward_std": 0.11195755004882812,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7479267120361328,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 122.03125,
      "epoch": 0.48625429553264604,
      "grad_norm": 1.3078742427842147,
      "kl": 0.02801513671875,
      "learning_rate": 7.568728522336769e-07,
      "loss": 0.0011,
      "reward": 1.7409408688545227,
      "reward_std": 0.11844996362924576,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7487533390522003,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 118.53125,
      "epoch": 0.48711340206185566,
      "grad_norm": 1.0128900337930118,
      "kl": 0.0277099609375,
      "learning_rate": 7.564432989690721e-07,
      "loss": 0.0011,
      "reward": 1.7667198181152344,
      "reward_std": 0.09644139185547829,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7745323181152344,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 101.28125,
      "epoch": 0.4879725085910653,
      "grad_norm": 1.2158578490452436,
      "kl": 0.0233154296875,
      "learning_rate": 7.560137457044673e-07,
      "loss": 0.0009,
      "reward": 1.8185061812400818,
      "reward_std": 0.07172945514321327,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8185062408447266,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.78125,
      "epoch": 0.4888316151202749,
      "grad_norm": 1.1584871999787036,
      "kl": 0.0294189453125,
      "learning_rate": 7.555841924398625e-07,
      "loss": 0.0012,
      "reward": 1.6857864260673523,
      "reward_std": 0.0634692870080471,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.6935989856719971,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.8984375,
      "epoch": 0.4896907216494845,
      "grad_norm": 1.5831208673085706,
      "kl": 0.02850341796875,
      "learning_rate": 7.551546391752577e-07,
      "loss": 0.0011,
      "reward": 1.7977246046066284,
      "reward_std": 0.17054561525583267,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.805537074804306,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.5078125,
      "epoch": 0.49054982817869414,
      "grad_norm": 0.7928407917060947,
      "kl": 0.0238037109375,
      "learning_rate": 7.547250859106528e-07,
      "loss": 0.001,
      "reward": 1.7591076493263245,
      "reward_std": 0.0366393206641078,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7669200897216797,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 134.4609375,
      "epoch": 0.49140893470790376,
      "grad_norm": 0.8346365718982691,
      "kl": 0.02227783203125,
      "learning_rate": 7.54295532646048e-07,
      "loss": 0.0009,
      "reward": 1.826495349407196,
      "reward_std": 0.05793471448123455,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8264953792095184,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 114.6484375,
      "epoch": 0.49226804123711343,
      "grad_norm": 1.852024426080103,
      "kl": 0.021484375,
      "learning_rate": 7.538659793814432e-07,
      "loss": 0.0009,
      "reward": 1.8043755292892456,
      "reward_std": 0.18020495027303696,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.804375559091568,
      "step": 573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.8203125,
      "epoch": 0.49312714776632305,
      "grad_norm": 26.47121113659565,
      "kl": 0.02374267578125,
      "learning_rate": 7.534364261168384e-07,
      "loss": 0.0009,
      "reward": 1.827117145061493,
      "reward_std": 0.10837876796722412,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8271171748638153,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 124.234375,
      "epoch": 0.49398625429553267,
      "grad_norm": 1.1310521518717915,
      "kl": 0.02288818359375,
      "learning_rate": 7.530068728522336e-07,
      "loss": 0.0009,
      "reward": 1.797199308872223,
      "reward_std": 0.10720982775092125,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8128242492675781,
      "step": 575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 120.9453125,
      "epoch": 0.4948453608247423,
      "grad_norm": 1.8553971162459708,
      "kl": 0.02252197265625,
      "learning_rate": 7.525773195876288e-07,
      "loss": 0.0009,
      "reward": 1.79475736618042,
      "reward_std": 0.13686570525169373,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7947573065757751,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.5703125,
      "epoch": 0.4957044673539519,
      "grad_norm": 1.1152662401016404,
      "kl": 0.026611328125,
      "learning_rate": 7.52147766323024e-07,
      "loss": 0.0011,
      "reward": 1.858827292919159,
      "reward_std": 0.15026213228702545,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8666398227214813,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.15625,
      "epoch": 0.4965635738831615,
      "grad_norm": 0.7945279843022707,
      "kl": 0.0247802734375,
      "learning_rate": 7.517182130584192e-07,
      "loss": 0.001,
      "reward": 1.8583158254623413,
      "reward_std": 0.08704648166894913,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8739407956600189,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.015625,
      "epoch": 0.49742268041237114,
      "grad_norm": 0.8545181390908309,
      "kl": 0.02587890625,
      "learning_rate": 7.512886597938143e-07,
      "loss": 0.001,
      "reward": 1.794776201248169,
      "reward_std": 0.07284862734377384,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8025886416435242,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.828125,
      "epoch": 0.49828178694158076,
      "grad_norm": 5.702979853199694,
      "kl": 0.02679443359375,
      "learning_rate": 7.508591065292095e-07,
      "loss": 0.0011,
      "reward": 1.7567557096481323,
      "reward_std": 0.10015341639518738,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7567557394504547,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 143.1796875,
      "epoch": 0.4991408934707904,
      "grad_norm": 3.2034971522784796,
      "kl": 0.0230712890625,
      "learning_rate": 7.504295532646047e-07,
      "loss": 0.0009,
      "reward": 1.8063377737998962,
      "reward_std": 0.11063193529844284,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8063377737998962,
      "step": 581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 129.734375,
      "epoch": 0.5,
      "grad_norm": 0.7129779180772821,
      "kl": 0.0211181640625,
      "learning_rate": 7.5e-07,
      "loss": 0.0008,
      "reward": 1.8224888443946838,
      "reward_std": 0.056552507914602757,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8303013443946838,
      "step": 582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.0625,
      "epoch": 0.5008591065292096,
      "grad_norm": 2.78986309606567,
      "kl": 0.0185546875,
      "learning_rate": 7.495704467353952e-07,
      "loss": 0.0007,
      "reward": 1.854720413684845,
      "reward_std": 0.10461413115262985,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8547204732894897,
      "step": 583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.1484375,
      "epoch": 0.5017182130584192,
      "grad_norm": 1.0824827966215953,
      "kl": 0.02435302734375,
      "learning_rate": 7.491408934707904e-07,
      "loss": 0.001,
      "reward": 1.7764517068862915,
      "reward_std": 0.10467999801039696,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.7920766472816467,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 128.765625,
      "epoch": 0.5025773195876289,
      "grad_norm": 1.252463130344048,
      "kl": 0.02105712890625,
      "learning_rate": 7.487113402061856e-07,
      "loss": 0.0008,
      "reward": 1.8154524564743042,
      "reward_std": 0.1660311445593834,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.8310775756835938,
      "step": 585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 119.734375,
      "epoch": 0.5034364261168385,
      "grad_norm": 0.9304730717779914,
      "kl": 0.023681640625,
      "learning_rate": 7.482817869415808e-07,
      "loss": 0.0009,
      "reward": 1.8320194482803345,
      "reward_std": 0.07787203788757324,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8398319780826569,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 151.5703125,
      "epoch": 0.5042955326460481,
      "grad_norm": 5.260833868032736,
      "kl": 0.0369873046875,
      "learning_rate": 7.47852233676976e-07,
      "loss": 0.0015,
      "reward": 1.7589982748031616,
      "reward_std": 0.14544661343097687,
      "rewards/format_reward_gen": 0.984375,
      "rewards/llm_reward": 0.774623304605484,
      "step": 587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 132.953125,
      "epoch": 0.5051546391752577,
      "grad_norm": 1.2660610390516958,
      "kl": 0.031005859375,
      "learning_rate": 7.474226804123711e-07,
      "loss": 0.0012,
      "reward": 1.822842538356781,
      "reward_std": 0.13162581250071526,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8306550085544586,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 158.7578125,
      "epoch": 0.5060137457044673,
      "grad_norm": 0.9382817085600501,
      "kl": 0.0223388671875,
      "learning_rate": 7.469931271477663e-07,
      "loss": 0.0009,
      "reward": 1.7842236161231995,
      "reward_std": 0.11218691617250443,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7842236161231995,
      "step": 589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 141.609375,
      "epoch": 0.506872852233677,
      "grad_norm": 1.5108030501592749,
      "kl": 0.0247802734375,
      "learning_rate": 7.465635738831615e-07,
      "loss": 0.001,
      "reward": 1.8379915952682495,
      "reward_std": 0.11418896913528442,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.8458041250705719,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 131.6640625,
      "epoch": 0.5077319587628866,
      "grad_norm": 0.9756789130203551,
      "kl": 0.02655029296875,
      "learning_rate": 7.461340206185567e-07,
      "loss": 0.0011,
      "reward": 1.8145712018013,
      "reward_std": 0.09588093496859074,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8145712018013,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 136.1953125,
      "epoch": 0.5085910652920962,
      "grad_norm": 1.78737635682799,
      "kl": 0.02001953125,
      "learning_rate": 7.457044673539519e-07,
      "loss": 0.0008,
      "reward": 1.7849004864692688,
      "reward_std": 0.09064675122499466,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7927128970623016,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 130.0234375,
      "epoch": 0.5094501718213058,
      "grad_norm": 1.5323474545376934,
      "kl": 0.0244140625,
      "learning_rate": 7.452749140893471e-07,
      "loss": 0.001,
      "reward": 1.8163361549377441,
      "reward_std": 0.06137897726148367,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.8163362145423889,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 146.65625,
      "epoch": 0.5103092783505154,
      "grad_norm": 1.167714380864763,
      "kl": 0.02984619140625,
      "learning_rate": 7.448453608247423e-07,
      "loss": 0.0012,
      "reward": 1.7876654863357544,
      "reward_std": 0.1253710240125656,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7954780161380768,
      "step": 594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.375,
      "epoch": 0.511168384879725,
      "grad_norm": 1.2346413570984562,
      "kl": 0.0233154296875,
      "learning_rate": 7.444158075601375e-07,
      "loss": 0.0009,
      "reward": 1.7417802214622498,
      "reward_std": 0.1481439284980297,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7417803108692169,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.828125,
      "epoch": 0.5120274914089347,
      "grad_norm": 1.4208055330522806,
      "kl": 0.0283203125,
      "learning_rate": 7.439862542955327e-07,
      "loss": 0.0011,
      "reward": 1.7719964981079102,
      "reward_std": 0.15724200010299683,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7798090577125549,
      "step": 596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 144.71875,
      "epoch": 0.5128865979381443,
      "grad_norm": 3.920082332866134,
      "kl": 0.02557373046875,
      "learning_rate": 7.435567010309278e-07,
      "loss": 0.001,
      "reward": 1.7663773894309998,
      "reward_std": 0.08243504166603088,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7663773894309998,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 137.40625,
      "epoch": 0.5137457044673539,
      "grad_norm": 1.2860662352628833,
      "kl": 0.02374267578125,
      "learning_rate": 7.43127147766323e-07,
      "loss": 0.0009,
      "reward": 1.7066620588302612,
      "reward_std": 0.06952378898859024,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7144745290279388,
      "step": 598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 140.2265625,
      "epoch": 0.5146048109965635,
      "grad_norm": 0.9580097497182469,
      "kl": 0.021728515625,
      "learning_rate": 7.426975945017182e-07,
      "loss": 0.0009,
      "reward": 1.7517217993736267,
      "reward_std": 0.09551467373967171,
      "rewards/format_reward_gen": 1.0,
      "rewards/llm_reward": 0.7517217695713043,
      "step": 599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 153.5625,
      "epoch": 0.5154639175257731,
      "grad_norm": 1.8465816930490027,
      "kl": 0.0545654296875,
      "learning_rate": 7.422680412371134e-07,
      "loss": 0.0022,
      "reward": 1.711679220199585,
      "reward_std": 0.10108717158436775,
      "rewards/format_reward_gen": 0.9921875,
      "rewards/llm_reward": 0.7194916903972626,
      "step": 600
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 2328,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}