{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 10000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 75.6,
      "epoch": 0.002,
      "grad_norm": 0.0,
      "kl": 0.0,
      "learning_rate": 5.0000000000000004e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 10
    },
    {
      "completion_length": 69.0,
      "epoch": 0.004,
      "grad_norm": 8.249282836914062e-05,
      "kl": 0.0007458075881004334,
      "learning_rate": 1.0000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 20
    },
    {
      "completion_length": 65.25,
      "epoch": 0.006,
      "grad_norm": 8.726119995117188e-05,
      "kl": 0.0008603519352618604,
      "learning_rate": 1.5000000000000002e-07,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 30
    },
    {
      "completion_length": 51.1,
      "epoch": 0.008,
      "grad_norm": 0.0001392364501953125,
      "kl": 0.0006604890164453536,
      "learning_rate": 2.0000000000000002e-07,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 40
    },
    {
      "completion_length": 81.475,
      "epoch": 0.01,
      "grad_norm": 10.3125,
      "kl": 0.0008120269441860728,
      "learning_rate": 2.5000000000000004e-07,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.3154700517654419,
      "rewards/reward_func": -0.2,
      "step": 50
    },
    {
      "completion_length": 82.3,
      "epoch": 0.012,
      "grad_norm": 0.000148773193359375,
      "kl": 0.0009021399382618256,
      "learning_rate": 3.0000000000000004e-07,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.2,
      "rewards/reward_func": -0.1,
      "step": 60
    },
    {
      "completion_length": 57.675,
      "epoch": 0.014,
      "grad_norm": 0.000453948974609375,
      "kl": 0.0008941800828324631,
      "learning_rate": 3.5000000000000004e-07,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 70
    },
    {
      "completion_length": 70.475,
      "epoch": 0.016,
      "grad_norm": 7.90625,
      "kl": 0.0010760451084934175,
      "learning_rate": 4.0000000000000003e-07,
      "loss": 0.0,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.2,
      "rewards/reward_func": -0.25,
      "step": 80
    },
    {
      "completion_length": 56.2,
      "epoch": 0.018,
      "grad_norm": 8.535385131835938e-05,
      "kl": 0.001845199626404792,
      "learning_rate": 4.5000000000000003e-07,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 90
    },
    {
      "completion_length": 63.5,
      "epoch": 0.02,
      "grad_norm": 0.00011777877807617188,
      "kl": 0.0007161700828874018,
      "learning_rate": 5.000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.725,
      "reward": -0.275,
      "reward_std": 0.20773502588272094,
      "rewards/reward_func": -0.275,
      "step": 100
    },
    {
      "completion_length": 88.175,
      "epoch": 0.022,
      "grad_norm": 0.000125885009765625,
      "kl": 0.001011227659182623,
      "learning_rate": 5.5e-07,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.1,
      "step": 110
    },
    {
      "completion_length": 69.725,
      "epoch": 0.024,
      "grad_norm": 0.00010633468627929688,
      "kl": 0.0008000041969353333,
      "learning_rate": 6.000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.25,
      "rewards/reward_func": -0.125,
      "step": 120
    },
    {
      "completion_length": 72.35,
      "epoch": 0.026,
      "grad_norm": 0.00012159347534179688,
      "kl": 0.0010485154576599597,
      "learning_rate": 6.5e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 130
    },
    {
      "completion_length": 75.575,
      "epoch": 0.028,
      "grad_norm": 0.00017547607421875,
      "kl": 0.0007810671289917081,
      "learning_rate": 7.000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 140
    },
    {
      "completion_length": 79.65,
      "epoch": 0.03,
      "grad_norm": 8.0625,
      "kl": 0.0008353532728506252,
      "learning_rate": 7.5e-07,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 150
    },
    {
      "completion_length": 52.075,
      "epoch": 0.032,
      "grad_norm": 33.25,
      "kl": 0.0031860046496149153,
      "learning_rate": 8.000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.15,
      "rewards/reward_func": -0.125,
      "step": 160
    },
    {
      "completion_length": 58.825,
      "epoch": 0.034,
      "grad_norm": 15.6875,
      "kl": 0.001403974276036024,
      "learning_rate": 8.500000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 170
    },
    {
      "completion_length": 70.95,
      "epoch": 0.036,
      "grad_norm": 0.0002956390380859375,
      "kl": 0.0009778408275451511,
      "learning_rate": 9.000000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 180
    },
    {
      "completion_length": 62.25,
      "epoch": 0.038,
      "grad_norm": 0.00011587142944335938,
      "kl": 0.001159800120512955,
      "learning_rate": 9.500000000000001e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 190
    },
    {
      "completion_length": 68.7,
      "epoch": 0.04,
      "grad_norm": 0.0001430511474609375,
      "kl": 0.0036639797501266,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 200
    },
    {
      "completion_length": 50.675,
      "epoch": 0.042,
      "grad_norm": 7.534027099609375e-05,
      "kl": 0.002666096478151303,
      "learning_rate": 1.0500000000000001e-06,
      "loss": 0.0,
      "match_ratio": 0.775,
      "reward": -0.225,
      "reward_std": 0.05,
      "rewards/reward_func": -0.225,
      "step": 210
    },
    {
      "completion_length": 36.8,
      "epoch": 0.044,
      "grad_norm": 0.00018215179443359375,
      "kl": 0.014809455376234838,
      "learning_rate": 1.1e-06,
      "loss": 0.0,
      "match_ratio": 0.725,
      "reward": -0.275,
      "reward_std": 0.15,
      "rewards/reward_func": -0.275,
      "step": 220
    },
    {
      "completion_length": 66.75,
      "epoch": 0.046,
      "grad_norm": 0.0004520416259765625,
      "kl": 0.007854271659743972,
      "learning_rate": 1.1500000000000002e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.20773502588272094,
      "rewards/reward_func": -0.125,
      "step": 230
    },
    {
      "completion_length": 76.35,
      "epoch": 0.048,
      "grad_norm": 17.875,
      "kl": 1.018031721841544,
      "learning_rate": 1.2000000000000002e-06,
      "loss": 0.0001,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.2,
      "rewards/reward_func": -0.2,
      "step": 240
    },
    {
      "completion_length": 59.075,
      "epoch": 0.05,
      "grad_norm": 8.869171142578125e-05,
      "kl": 0.13859437993960455,
      "learning_rate": 1.25e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 250
    },
    {
      "completion_length": 55.95,
      "epoch": 0.052,
      "grad_norm": 0.458984375,
      "kl": 0.44334062208363323,
      "learning_rate": 1.3e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 260
    },
    {
      "completion_length": 62.5,
      "epoch": 0.054,
      "grad_norm": 0.000209808349609375,
      "kl": 1.4304919777001488,
      "learning_rate": 1.3500000000000002e-06,
      "loss": 0.0001,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.15,
      "rewards/reward_func": -0.125,
      "step": 270
    },
    {
      "completion_length": 59.125,
      "epoch": 0.056,
      "grad_norm": 7.963180541992188e-05,
      "kl": 0.3974201448727399,
      "learning_rate": 1.4000000000000001e-06,
      "loss": 0.0,
      "match_ratio": 0.575,
      "reward": -0.425,
      "reward_std": 0.15,
      "rewards/reward_func": -0.425,
      "step": 280
    },
    {
      "completion_length": 61.75,
      "epoch": 0.058,
      "grad_norm": 0.00015926361083984375,
      "kl": 0.6299153287603986,
      "learning_rate": 1.45e-06,
      "loss": 0.0001,
      "match_ratio": 0.775,
      "reward": -0.225,
      "reward_std": 0.05,
      "rewards/reward_func": -0.225,
      "step": 290
    },
    {
      "completion_length": 94.275,
      "epoch": 0.06,
      "grad_norm": 0.00010347366333007812,
      "kl": 0.007541297184070572,
      "learning_rate": 1.5e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.15,
      "rewards/reward_func": -0.075,
      "step": 300
    },
    {
      "completion_length": 55.45,
      "epoch": 0.062,
      "grad_norm": 0.000507354736328125,
      "kl": 0.8210757704044227,
      "learning_rate": 1.5500000000000002e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 310
    },
    {
      "completion_length": 63.475,
      "epoch": 0.064,
      "grad_norm": 0.000396728515625,
      "kl": 0.10055320091196336,
      "learning_rate": 1.6000000000000001e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 320
    },
    {
      "completion_length": 82.15,
      "epoch": 0.066,
      "grad_norm": 0.000514984130859375,
      "kl": 0.37705419784761035,
      "learning_rate": 1.6500000000000003e-06,
      "loss": 0.0,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.175,
      "step": 330
    },
    {
      "completion_length": 51.475,
      "epoch": 0.068,
      "grad_norm": 0.00014400482177734375,
      "kl": 41.57116786188563,
      "learning_rate": 1.7000000000000002e-06,
      "loss": 0.0042,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 340
    },
    {
      "completion_length": 55.525,
      "epoch": 0.07,
      "grad_norm": 0.0002689361572265625,
      "kl": 1.14909179067472,
      "learning_rate": 1.75e-06,
      "loss": 0.0001,
      "match_ratio": 0.725,
      "reward": -0.275,
      "reward_std": 0.05,
      "rewards/reward_func": -0.275,
      "step": 350
    },
    {
      "completion_length": 47.725,
      "epoch": 0.072,
      "grad_norm": 4.28125,
      "kl": 1.577250469638966,
      "learning_rate": 1.8000000000000001e-06,
      "loss": 0.0002,
      "match_ratio": 0.675,
      "reward": -0.325,
      "reward_std": 0.05,
      "rewards/reward_func": -0.325,
      "step": 360
    },
    {
      "completion_length": 61.825,
      "epoch": 0.074,
      "grad_norm": 0.00017452239990234375,
      "kl": 0.34925971169723197,
      "learning_rate": 1.85e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 370
    },
    {
      "completion_length": 55.825,
      "epoch": 0.076,
      "grad_norm": 0.00066375732421875,
      "kl": 0.1755049143510405,
      "learning_rate": 1.9000000000000002e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.15,
      "rewards/reward_func": -0.075,
      "step": 380
    },
    {
      "completion_length": 49.9,
      "epoch": 0.078,
      "grad_norm": 0.0164794921875,
      "kl": 0.019992552557960154,
      "learning_rate": 1.9500000000000004e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 390
    },
    {
      "completion_length": 63.85,
      "epoch": 0.08,
      "grad_norm": 0.0002002716064453125,
      "kl": 4.795912343251985,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 0.0005,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.2,
      "rewards/reward_func": -0.1,
      "step": 400
    },
    {
      "completion_length": 56.6,
      "epoch": 0.082,
      "grad_norm": 0.0004520416259765625,
      "kl": 1.1443448643549345,
      "learning_rate": 2.05e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 410
    },
    {
      "completion_length": 58.725,
      "epoch": 0.084,
      "grad_norm": 0.0002765655517578125,
      "kl": 0.0028327183797955515,
      "learning_rate": 2.1000000000000002e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 420
    },
    {
      "completion_length": 47.1,
      "epoch": 0.086,
      "grad_norm": 0.0010986328125,
      "kl": 0.008324940234888344,
      "learning_rate": 2.15e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 430
    },
    {
      "completion_length": 65.75,
      "epoch": 0.088,
      "grad_norm": 11.9375,
      "kl": 0.006597463192883879,
      "learning_rate": 2.2e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 440
    },
    {
      "completion_length": 68.975,
      "epoch": 0.09,
      "grad_norm": 2064.0,
      "kl": 405.5571417377796,
      "learning_rate": 2.25e-06,
      "loss": 0.0406,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 450
    },
    {
      "completion_length": 56.975,
      "epoch": 0.092,
      "grad_norm": 0.00038909912109375,
      "kl": 0.05630149020580575,
      "learning_rate": 2.3000000000000004e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 460
    },
    {
      "completion_length": 61.075,
      "epoch": 0.094,
      "grad_norm": 0.000446319580078125,
      "kl": 0.005178993823938072,
      "learning_rate": 2.35e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 470
    },
    {
      "completion_length": 73.325,
      "epoch": 0.096,
      "grad_norm": 0.0002613067626953125,
      "kl": 4.652890888956608,
      "learning_rate": 2.4000000000000003e-06,
      "loss": 0.0005,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 480
    },
    {
      "completion_length": 68.625,
      "epoch": 0.098,
      "grad_norm": 0.000164031982421875,
      "kl": 15.9451186191116,
      "learning_rate": 2.4500000000000003e-06,
      "loss": 0.0016,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 490
    },
    {
      "completion_length": 82.025,
      "epoch": 0.1,
      "grad_norm": 0.000453948974609375,
      "kl": 0.006236090854508802,
      "learning_rate": 2.5e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 500
    },
    {
      "completion_length": 73.05,
      "epoch": 0.102,
      "grad_norm": 0.00064849853515625,
      "kl": 0.0709857388283126,
      "learning_rate": 2.55e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.15,
      "rewards/reward_func": -0.125,
      "step": 510
    },
    {
      "completion_length": 69.325,
      "epoch": 0.104,
      "grad_norm": 0.00048828125,
      "kl": 517.9831573915319,
      "learning_rate": 2.6e-06,
      "loss": 0.0518,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 520
    },
    {
      "completion_length": 57.95,
      "epoch": 0.106,
      "grad_norm": 0.00028228759765625,
      "kl": 0.006235601624939591,
      "learning_rate": 2.6500000000000005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 530
    },
    {
      "completion_length": 39.375,
      "epoch": 0.108,
      "grad_norm": 0.0023345947265625,
      "kl": 592.0278737243498,
      "learning_rate": 2.7000000000000004e-06,
      "loss": 0.0592,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 540
    },
    {
      "completion_length": 62.875,
      "epoch": 0.11,
      "grad_norm": 0.0003910064697265625,
      "kl": 0.004806909896433354,
      "learning_rate": 2.7500000000000004e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 550
    },
    {
      "completion_length": 56.45,
      "epoch": 0.112,
      "grad_norm": 0.0002117156982421875,
      "kl": 0.004208524071145803,
      "learning_rate": 2.8000000000000003e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 560
    },
    {
      "completion_length": 67.7,
      "epoch": 0.114,
      "grad_norm": 0.00016689300537109375,
      "kl": 3.4651028811815197,
      "learning_rate": 2.85e-06,
      "loss": 0.0003,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 570
    },
    {
      "completion_length": 51.525,
      "epoch": 0.116,
      "grad_norm": 0.040283203125,
      "kl": 0.03394674692535773,
      "learning_rate": 2.9e-06,
      "loss": 0.0,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 580
    },
    {
      "completion_length": 69.275,
      "epoch": 0.118,
      "grad_norm": 0.000823974609375,
      "kl": 0.0029081626795232295,
      "learning_rate": 2.95e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 590
    },
    {
      "completion_length": 71.975,
      "epoch": 0.12,
      "grad_norm": 0.0004711151123046875,
      "kl": 0.2187046888633631,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.20773502588272094,
      "rewards/reward_func": -0.125,
      "step": 600
    },
    {
      "completion_length": 61.85,
      "epoch": 0.122,
      "grad_norm": 24.25,
      "kl": 13.38376448857598,
      "learning_rate": 3.05e-06,
      "loss": 0.0013,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 610
    },
    {
      "completion_length": 53.5,
      "epoch": 0.124,
      "grad_norm": 0.0002498626708984375,
      "kl": 0.006577977701090277,
      "learning_rate": 3.1000000000000004e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 620
    },
    {
      "completion_length": 59.925,
      "epoch": 0.126,
      "grad_norm": 0.000690460205078125,
      "kl": 0.03976157886208966,
      "learning_rate": 3.1500000000000003e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 630
    },
    {
      "completion_length": 51.275,
      "epoch": 0.128,
      "grad_norm": 284.0,
      "kl": 128.89907464290735,
      "learning_rate": 3.2000000000000003e-06,
      "loss": 0.0129,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 640
    },
    {
      "completion_length": 57.525,
      "epoch": 0.13,
      "grad_norm": 0.000217437744140625,
      "kl": 0.15632477974286302,
      "learning_rate": 3.2500000000000002e-06,
      "loss": 0.0,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.1,
      "rewards/reward_func": -0.25,
      "step": 650
    },
    {
      "completion_length": 48.75,
      "epoch": 0.132,
      "grad_norm": 0.00035858154296875,
      "kl": 0.012460133875720203,
      "learning_rate": 3.3000000000000006e-06,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 660
    },
    {
      "completion_length": 46.375,
      "epoch": 0.134,
      "grad_norm": 0.000560760498046875,
      "kl": 0.005184722866397351,
      "learning_rate": 3.3500000000000005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 670
    },
    {
      "completion_length": 65.7,
      "epoch": 0.136,
      "grad_norm": 21.625,
      "kl": 0.03178581706015393,
      "learning_rate": 3.4000000000000005e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 680
    },
    {
      "completion_length": 70.9,
      "epoch": 0.138,
      "grad_norm": 0.00025177001953125,
      "kl": 0.0047074495116248725,
      "learning_rate": 3.45e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 690
    },
    {
      "completion_length": 50.8,
      "epoch": 0.14,
      "grad_norm": 0.0001735687255859375,
      "kl": 2.7392968325410036,
      "learning_rate": 3.5e-06,
      "loss": 0.0003,
      "match_ratio": 0.725,
      "reward": -0.275,
      "reward_std": 0.05,
      "rewards/reward_func": -0.275,
      "step": 700
    },
    {
      "completion_length": 57.025,
      "epoch": 0.142,
      "grad_norm": 0.039794921875,
      "kl": 179.23230375794228,
      "learning_rate": 3.5500000000000003e-06,
      "loss": 0.0179,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 710
    },
    {
      "completion_length": 67.825,
      "epoch": 0.144,
      "grad_norm": 0.00042724609375,
      "kl": 0.07836510783527047,
      "learning_rate": 3.6000000000000003e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 720
    },
    {
      "completion_length": 60.7,
      "epoch": 0.146,
      "grad_norm": 936.0,
      "kl": 213.31693772624712,
      "learning_rate": 3.65e-06,
      "loss": 0.0213,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.15,
      "rewards/reward_func": -0.125,
      "step": 730
    },
    {
      "completion_length": 56.275,
      "epoch": 0.148,
      "grad_norm": 0.00104522705078125,
      "kl": 0.2562748788390309,
      "learning_rate": 3.7e-06,
      "loss": 0.0,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.25,
      "step": 740
    },
    {
      "completion_length": 63.85,
      "epoch": 0.15,
      "grad_norm": 0.000640869140625,
      "kl": 0.869432492996566,
      "learning_rate": 3.7500000000000005e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 750
    },
    {
      "completion_length": 68.85,
      "epoch": 0.152,
      "grad_norm": 0.0002613067626953125,
      "kl": 0.003684952133335173,
      "learning_rate": 3.8000000000000005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 760
    },
    {
      "completion_length": 59.675,
      "epoch": 0.154,
      "grad_norm": 0.000286102294921875,
      "kl": 0.011132878507487476,
      "learning_rate": 3.85e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 770
    },
    {
      "completion_length": 56.925,
      "epoch": 0.156,
      "grad_norm": 0.000244140625,
      "kl": 431.3150466301013,
      "learning_rate": 3.900000000000001e-06,
      "loss": 0.0431,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 780
    },
    {
      "completion_length": 59.075,
      "epoch": 0.158,
      "grad_norm": 0.00026702880859375,
      "kl": 0.004525382234714925,
      "learning_rate": 3.95e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 790
    },
    {
      "completion_length": 41.475,
      "epoch": 0.16,
      "grad_norm": 0.000392913818359375,
      "kl": 0.8990198554703965,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0001,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.25,
      "step": 800
    },
    {
      "completion_length": 49.35,
      "epoch": 0.162,
      "grad_norm": 0.0006103515625,
      "kl": 10258.552996213792,
      "learning_rate": 4.05e-06,
      "loss": 1.0259,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 810
    },
    {
      "completion_length": 47.675,
      "epoch": 0.164,
      "grad_norm": 0.00102996826171875,
      "kl": 3237.7607058377935,
      "learning_rate": 4.1e-06,
      "loss": 0.3238,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 820
    },
    {
      "completion_length": 63.375,
      "epoch": 0.166,
      "grad_norm": 0.0003414154052734375,
      "kl": 4.472107960679568,
      "learning_rate": 4.15e-06,
      "loss": 0.0004,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 830
    },
    {
      "completion_length": 70.375,
      "epoch": 0.168,
      "grad_norm": 0.00023174285888671875,
      "kl": 0.005188186629675328,
      "learning_rate": 4.2000000000000004e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 840
    },
    {
      "completion_length": 64.975,
      "epoch": 0.17,
      "grad_norm": 0.00022029876708984375,
      "kl": 0.019074565428309143,
      "learning_rate": 4.25e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 850
    },
    {
      "completion_length": 52.925,
      "epoch": 0.172,
      "grad_norm": 0.00029754638671875,
      "kl": 4.278230914589949,
      "learning_rate": 4.3e-06,
      "loss": 0.0004,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.1,
      "step": 860
    },
    {
      "completion_length": 66.075,
      "epoch": 0.174,
      "grad_norm": 0.00021457672119140625,
      "kl": 0.006756197474896908,
      "learning_rate": 4.350000000000001e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 870
    },
    {
      "completion_length": 62.075,
      "epoch": 0.176,
      "grad_norm": 7.9375,
      "kl": 1679.0881600409746,
      "learning_rate": 4.4e-06,
      "loss": 0.1679,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 880
    },
    {
      "completion_length": 57.475,
      "epoch": 0.178,
      "grad_norm": 0.000576019287109375,
      "kl": 0.06974834711290896,
      "learning_rate": 4.450000000000001e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 890
    },
    {
      "completion_length": 59.25,
      "epoch": 0.18,
      "grad_norm": 0.000835418701171875,
      "kl": 0.0047427960205823185,
      "learning_rate": 4.5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 900
    },
    {
      "completion_length": 49.925,
      "epoch": 0.182,
      "grad_norm": 0.000652313232421875,
      "kl": 0.0047499775420874355,
      "learning_rate": 4.5500000000000005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 910
    },
    {
      "completion_length": 63.825,
      "epoch": 0.184,
      "grad_norm": 72192.0,
      "kl": 24955.292986106546,
      "learning_rate": 4.600000000000001e-06,
      "loss": 2.4955,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 920
    },
    {
      "completion_length": 52.475,
      "epoch": 0.186,
      "grad_norm": 0.000362396240234375,
      "kl": 0.6700008324347436,
      "learning_rate": 4.65e-06,
      "loss": 0.0001,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 930
    },
    {
      "completion_length": 58.925,
      "epoch": 0.188,
      "grad_norm": 0.0002956390380859375,
      "kl": 0.019399669324047863,
      "learning_rate": 4.7e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 940
    },
    {
      "completion_length": 62.1,
      "epoch": 0.19,
      "grad_norm": 0.0019378662109375,
      "kl": 0.008174928580410778,
      "learning_rate": 4.75e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 950
    },
    {
      "completion_length": 57.75,
      "epoch": 0.192,
      "grad_norm": 0.0004138946533203125,
      "kl": 1.113649177318439,
      "learning_rate": 4.800000000000001e-06,
      "loss": 0.0001,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 960
    },
    {
      "completion_length": 56.225,
      "epoch": 0.194,
      "grad_norm": 0.00069427490234375,
      "kl": 0.5510010560508818,
      "learning_rate": 4.85e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 970
    },
    {
      "completion_length": 47.175,
      "epoch": 0.196,
      "grad_norm": 0.0002155303955078125,
      "kl": 1441.3666083157761,
      "learning_rate": 4.9000000000000005e-06,
      "loss": 0.1441,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 980
    },
    {
      "completion_length": 46.85,
      "epoch": 0.198,
      "grad_norm": 0.000720977783203125,
      "kl": 1.4619831766700373,
      "learning_rate": 4.95e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 990
    },
    {
      "completion_length": 65.55,
      "epoch": 0.2,
      "grad_norm": 0.0003871917724609375,
      "kl": 7.501803689775988,
      "learning_rate": 5e-06,
      "loss": 0.0008,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 1000
    },
    {
      "completion_length": 62.975,
      "epoch": 0.202,
      "grad_norm": 15.5625,
      "kl": 0.10273585927207023,
      "learning_rate": 4.999984769144476e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1010
    },
    {
      "completion_length": 57.35,
      "epoch": 0.204,
      "grad_norm": 0.0004520416259765625,
      "kl": 3.634061038820073,
      "learning_rate": 4.999939076763487e-06,
      "loss": 0.0004,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 1020
    },
    {
      "completion_length": 59.675,
      "epoch": 0.206,
      "grad_norm": 0.000507354736328125,
      "kl": 0.45058006714098153,
      "learning_rate": 4.999862923413781e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 1030
    },
    {
      "completion_length": 44.05,
      "epoch": 0.208,
      "grad_norm": 0.0005645751953125,
      "kl": 0.08316081156954169,
      "learning_rate": 4.999756310023261e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1040
    },
    {
      "completion_length": 79.225,
      "epoch": 0.21,
      "grad_norm": 0.00121307373046875,
      "kl": 0.018303806148469447,
      "learning_rate": 4.9996192378909785e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 1050
    },
    {
      "completion_length": 57.4,
      "epoch": 0.212,
      "grad_norm": 0.00104522705078125,
      "kl": 17796.077124893247,
      "learning_rate": 4.999451708687114e-06,
      "loss": 1.7796,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1060
    },
    {
      "completion_length": 52.8,
      "epoch": 0.214,
      "grad_norm": 8.821487426757812e-05,
      "kl": 6.741311503923498,
      "learning_rate": 4.9992537244529585e-06,
      "loss": 0.0007,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 1070
    },
    {
      "completion_length": 56.675,
      "epoch": 0.216,
      "grad_norm": 0.00084686279296875,
      "kl": 0.024167609214782716,
      "learning_rate": 4.999025287600886e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1080
    },
    {
      "completion_length": 61.325,
      "epoch": 0.218,
      "grad_norm": 0.0004634857177734375,
      "kl": 0.03314157268032432,
      "learning_rate": 4.998766400914329e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1090
    },
    {
      "completion_length": 78.3,
      "epoch": 0.22,
      "grad_norm": 0.0003376007080078125,
      "kl": 0.01341487793251872,
      "learning_rate": 4.99847706754774e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1100
    },
    {
      "completion_length": 54.725,
      "epoch": 0.222,
      "grad_norm": 0.000560760498046875,
      "kl": 674.1007295364049,
      "learning_rate": 4.998157291026553e-06,
      "loss": 0.0674,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1110
    },
    {
      "completion_length": 54.8,
      "epoch": 0.224,
      "grad_norm": 0.00119781494140625,
      "kl": 10.044112924486399,
      "learning_rate": 4.997807075247147e-06,
      "loss": 0.001,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 1120
    },
    {
      "completion_length": 49.275,
      "epoch": 0.226,
      "grad_norm": 0.0010833740234375,
      "kl": 0.017626433167606592,
      "learning_rate": 4.997426424476787e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1130
    },
    {
      "completion_length": 57.625,
      "epoch": 0.228,
      "grad_norm": 0.0020599365234375,
      "kl": 0.44624101794324816,
      "learning_rate": 4.9970153433535855e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1140
    },
    {
      "completion_length": 61.475,
      "epoch": 0.23,
      "grad_norm": 0.00159454345703125,
      "kl": 0.052927281521260736,
      "learning_rate": 4.9965738368864345e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1150
    },
    {
      "completion_length": 71.8,
      "epoch": 0.232,
      "grad_norm": 0.00103759765625,
      "kl": 0.011898941779509186,
      "learning_rate": 4.996101910454953e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1160
    },
    {
      "completion_length": 56.775,
      "epoch": 0.234,
      "grad_norm": 0.006256103515625,
      "kl": 0.440633371565491,
      "learning_rate": 4.995599569809414e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 1170
    },
    {
      "completion_length": 61.0,
      "epoch": 0.236,
      "grad_norm": 0.00174713134765625,
      "kl": 0.04742448972538114,
      "learning_rate": 4.9950668210706795e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1180
    },
    {
      "completion_length": 61.45,
      "epoch": 0.238,
      "grad_norm": 16.625,
      "kl": 0.04350157366134226,
      "learning_rate": 4.994503670730126e-06,
      "loss": 0.0,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.1,
      "rewards/reward_func": -0.25,
      "step": 1190
    },
    {
      "completion_length": 66.725,
      "epoch": 0.24,
      "grad_norm": 0.00063323974609375,
      "kl": 0.013561246497556568,
      "learning_rate": 4.993910125649561e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1200
    },
    {
      "completion_length": 73.775,
      "epoch": 0.242,
      "grad_norm": 0.0018310546875,
      "kl": 0.008337400993332267,
      "learning_rate": 4.993286193061145e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1210
    },
    {
      "completion_length": 46.075,
      "epoch": 0.244,
      "grad_norm": 38.75,
      "kl": 0.10523022091947495,
      "learning_rate": 4.992631880567301e-06,
      "loss": 0.0,
      "match_ratio": 0.775,
      "reward": -0.225,
      "reward_std": 0.05,
      "rewards/reward_func": -0.225,
      "step": 1220
    },
    {
      "completion_length": 44.275,
      "epoch": 0.246,
      "grad_norm": 0.0004405975341796875,
      "kl": 0.289978933124803,
      "learning_rate": 4.991947196140619e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 1230
    },
    {
      "completion_length": 57.0,
      "epoch": 0.248,
      "grad_norm": 326.0,
      "kl": 595.9660109838471,
      "learning_rate": 4.9912321481237616e-06,
      "loss": 0.0596,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.15,
      "rewards/reward_func": -0.175,
      "step": 1240
    },
    {
      "completion_length": 82.45,
      "epoch": 0.25,
      "grad_norm": 0.0002574920654296875,
      "kl": 0.023628250462934375,
      "learning_rate": 4.990486745229364e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1250
    },
    {
      "completion_length": 78.125,
      "epoch": 0.252,
      "grad_norm": 0.0023651123046875,
      "kl": 0.013969364436343312,
      "learning_rate": 4.989710996539926e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1260
    },
    {
      "completion_length": 55.5,
      "epoch": 0.254,
      "grad_norm": 0.0003910064697265625,
      "kl": 2.1650808176025746,
      "learning_rate": 4.9889049115077e-06,
      "loss": 0.0002,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 1270
    },
    {
      "completion_length": 71.45,
      "epoch": 0.256,
      "grad_norm": 0.00060272216796875,
      "kl": 0.02477358910255134,
      "learning_rate": 4.988068499954578e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1280
    },
    {
      "completion_length": 57.5,
      "epoch": 0.258,
      "grad_norm": 0.0024261474609375,
      "kl": 0.028649515146389602,
      "learning_rate": 4.987201772071971e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1290
    },
    {
      "completion_length": 59.425,
      "epoch": 0.26,
      "grad_norm": 0.0003948211669921875,
      "kl": 0.02502227737568319,
      "learning_rate": 4.986304738420684e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1300
    },
    {
      "completion_length": 53.9,
      "epoch": 0.262,
      "grad_norm": 0.000690460205078125,
      "kl": 0.028165359469130635,
      "learning_rate": 4.985377409930789e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1310
    },
    {
      "completion_length": 53.075,
      "epoch": 0.264,
      "grad_norm": 0.00091552734375,
      "kl": 30.839417777769267,
      "learning_rate": 4.984419797901491e-06,
      "loss": 0.0031,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1320
    },
    {
      "completion_length": 71.55,
      "epoch": 0.266,
      "grad_norm": 0.00034332275390625,
      "kl": 0.011334103159606456,
      "learning_rate": 4.983431914000991e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1330
    },
    {
      "completion_length": 47.675,
      "epoch": 0.268,
      "grad_norm": 0.00049591064453125,
      "kl": 0.21036937911994755,
      "learning_rate": 4.9824137702663424e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1340
    },
    {
      "completion_length": 59.8,
      "epoch": 0.27,
      "grad_norm": 0.000598907470703125,
      "kl": 0.0196828240994364,
      "learning_rate": 4.981365379103306e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1350
    },
    {
      "completion_length": 57.1,
      "epoch": 0.272,
      "grad_norm": 0.000858306884765625,
      "kl": 0.010027467273175716,
      "learning_rate": 4.980286753286196e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1360
    },
    {
      "completion_length": 64.575,
      "epoch": 0.274,
      "grad_norm": 0.00262451171875,
      "kl": 0.03173879962414503,
      "learning_rate": 4.979177905957726e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1370
    },
    {
      "completion_length": 38.225,
      "epoch": 0.276,
      "grad_norm": 0.001953125,
      "kl": 0.10523775820620358,
      "learning_rate": 4.978038850628855e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1380
    },
    {
      "completion_length": 64.175,
      "epoch": 0.278,
      "grad_norm": 0.00165557861328125,
      "kl": 0.022827543993480505,
      "learning_rate": 4.9768696011786095e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1390
    },
    {
      "completion_length": 51.525,
      "epoch": 0.28,
      "grad_norm": 0.00408935546875,
      "kl": 0.040360532607883214,
      "learning_rate": 4.975670171853926e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1400
    },
    {
      "completion_length": 65.125,
      "epoch": 0.282,
      "grad_norm": 0.001129150390625,
      "kl": 0.010821055877022446,
      "learning_rate": 4.974440577269473e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1410
    },
    {
      "completion_length": 57.675,
      "epoch": 0.284,
      "grad_norm": 0.00049591064453125,
      "kl": 0.028958286670967937,
      "learning_rate": 4.973180832407471e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 1420
    },
    {
      "completion_length": 60.3,
      "epoch": 0.286,
      "grad_norm": 0.00051116943359375,
      "kl": 0.015607311762869358,
      "learning_rate": 4.971890952617515e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1430
    },
    {
      "completion_length": 90.0,
      "epoch": 0.288,
      "grad_norm": 0.000392913818359375,
      "kl": 0.007699519535526634,
      "learning_rate": 4.970570953616383e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1440
    },
    {
      "completion_length": 51.625,
      "epoch": 0.29,
      "grad_norm": 0.00067138671875,
      "kl": 0.05114179509691894,
      "learning_rate": 4.9692208514878445e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 1450
    },
    {
      "completion_length": 53.2,
      "epoch": 0.292,
      "grad_norm": 0.0004673004150390625,
      "kl": 0.015444098180159927,
      "learning_rate": 4.96784066268247e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1460
    },
    {
      "completion_length": 51.15,
      "epoch": 0.294,
      "grad_norm": 0.0004558563232421875,
      "kl": 0.028699404350481926,
      "learning_rate": 4.966430404017424e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1470
    },
    {
      "completion_length": 65.125,
      "epoch": 0.296,
      "grad_norm": 0.00144195556640625,
      "kl": 481.04375956221486,
      "learning_rate": 4.964990092676263e-06,
      "loss": 0.0481,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1480
    },
    {
      "completion_length": 54.275,
      "epoch": 0.298,
      "grad_norm": 0.000766754150390625,
      "kl": 0.056382374046370386,
      "learning_rate": 4.963519746208726e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 1490
    },
    {
      "completion_length": 64.05,
      "epoch": 0.3,
      "grad_norm": 0.000438690185546875,
      "kl": 0.3169886400224641,
      "learning_rate": 4.962019382530521e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1500
    },
    {
      "completion_length": 62.525,
      "epoch": 0.302,
      "grad_norm": 0.00079345703125,
      "kl": 0.022879413142800332,
      "learning_rate": 4.960489019923105e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1510
    },
    {
      "completion_length": 46.15,
      "epoch": 0.304,
      "grad_norm": 29.25,
      "kl": 0.05621479714754969,
      "learning_rate": 4.958928677033465e-06,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 1520
    },
    {
      "completion_length": 55.775,
      "epoch": 0.306,
      "grad_norm": 0.0005035400390625,
      "kl": 0.23808469655923545,
      "learning_rate": 4.957338372873886e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 1530
    },
    {
      "completion_length": 54.0,
      "epoch": 0.308,
      "grad_norm": 0.000518798828125,
      "kl": 0.052838593162596224,
      "learning_rate": 4.9557181268217225e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1540
    },
    {
      "completion_length": 59.875,
      "epoch": 0.31,
      "grad_norm": 7.0,
      "kl": 0.020038261311128736,
      "learning_rate": 4.9540679586191605e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 1550
    },
    {
      "completion_length": 48.025,
      "epoch": 0.312,
      "grad_norm": 0.0013885498046875,
      "kl": 0.037048061547102405,
      "learning_rate": 4.9523878883729794e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1560
    },
    {
      "completion_length": 71.175,
      "epoch": 0.314,
      "grad_norm": 0.000621795654296875,
      "kl": 0.011885163560509681,
      "learning_rate": 4.9506779365543054e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1570
    },
    {
      "completion_length": 55.575,
      "epoch": 0.316,
      "grad_norm": 0.000339508056640625,
      "kl": 0.022388620488345622,
      "learning_rate": 4.94893812399836e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 1580
    },
    {
      "completion_length": 59.975,
      "epoch": 0.318,
      "grad_norm": 0.0004520416259765625,
      "kl": 0.009310156595893205,
      "learning_rate": 4.947168471904213e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1590
    },
    {
      "completion_length": 61.125,
      "epoch": 0.32,
      "grad_norm": 0.000823974609375,
      "kl": 0.016036251094192266,
      "learning_rate": 4.9453690018345144e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1600
    },
    {
      "completion_length": 66.625,
      "epoch": 0.322,
      "grad_norm": 0.034912109375,
      "kl": 0.03369634412229061,
      "learning_rate": 4.9435397357152406e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1610
    },
    {
      "completion_length": 58.8,
      "epoch": 0.324,
      "grad_norm": 0.0047607421875,
      "kl": 0.0254040343221277,
      "learning_rate": 4.9416806958354206e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1620
    },
    {
      "completion_length": 46.0,
      "epoch": 0.326,
      "grad_norm": 0.0011138916015625,
      "kl": 0.07715323262382298,
      "learning_rate": 4.939791904846869e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1630
    },
    {
      "completion_length": 46.125,
      "epoch": 0.328,
      "grad_norm": 0.00060272216796875,
      "kl": 0.01129569010809064,
      "learning_rate": 4.937873385763909e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1640
    },
    {
      "completion_length": 66.1,
      "epoch": 0.33,
      "grad_norm": 0.0003204345703125,
      "kl": 0.008372989785857498,
      "learning_rate": 4.935925161963089e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1650
    },
    {
      "completion_length": 56.675,
      "epoch": 0.332,
      "grad_norm": 0.0007171630859375,
      "kl": 0.01157067040912807,
      "learning_rate": 4.933947257182901e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1660
    },
    {
      "completion_length": 44.15,
      "epoch": 0.334,
      "grad_norm": 0.00087738037109375,
      "kl": 0.016125927586108445,
      "learning_rate": 4.9319396955234925e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1670
    },
    {
      "completion_length": 64.3,
      "epoch": 0.336,
      "grad_norm": 0.0004673004150390625,
      "kl": 0.008596798940561711,
      "learning_rate": 4.9299025014463665e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1680
    },
    {
      "completion_length": 57.15,
      "epoch": 0.338,
      "grad_norm": 0.000705718994140625,
      "kl": 0.08002093653194606,
      "learning_rate": 4.92783569977409e-06,
      "loss": 0.0,
      "match_ratio": 0.65,
      "reward": -0.35,
      "reward_std": 0.1,
      "rewards/reward_func": -0.35,
      "step": 1690
    },
    {
      "completion_length": 52.65,
      "epoch": 0.34,
      "grad_norm": 0.0004558563232421875,
      "kl": 0.020256173936650156,
      "learning_rate": 4.925739315689991e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1700
    },
    {
      "completion_length": 58.75,
      "epoch": 0.342,
      "grad_norm": 0.00150299072265625,
      "kl": 0.02677068072371185,
      "learning_rate": 4.923613374737848e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1710
    },
    {
      "completion_length": 55.0,
      "epoch": 0.344,
      "grad_norm": 0.002227783203125,
      "kl": 0.03761114357039332,
      "learning_rate": 4.921457902821578e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 1720
    },
    {
      "completion_length": 54.575,
      "epoch": 0.346,
      "grad_norm": 23.125,
      "kl": 0.01751216114498675,
      "learning_rate": 4.9192729262049285e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1730
    },
    {
      "completion_length": 60.45,
      "epoch": 0.348,
      "grad_norm": 0.00075531005859375,
      "kl": 0.011810581240570172,
      "learning_rate": 4.917058471511149e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1740
    },
    {
      "completion_length": 45.65,
      "epoch": 0.35,
      "grad_norm": 0.004730224609375,
      "kl": 0.040744514157995584,
      "learning_rate": 4.914814565722671e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1750
    },
    {
      "completion_length": 59.5,
      "epoch": 0.352,
      "grad_norm": 0.000579833984375,
      "kl": 0.01578701629769057,
      "learning_rate": 4.912541236180779e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1760
    },
    {
      "completion_length": 50.7,
      "epoch": 0.354,
      "grad_norm": 0.000629425048828125,
      "kl": 0.11596511220559477,
      "learning_rate": 4.910238510585275e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1770
    },
    {
      "completion_length": 48.8,
      "epoch": 0.356,
      "grad_norm": 46.75,
      "kl": 0.2281810746062547,
      "learning_rate": 4.907906416994146e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1780
    },
    {
      "completion_length": 57.4,
      "epoch": 0.358,
      "grad_norm": 0.00173187255859375,
      "kl": 0.02588364710099995,
      "learning_rate": 4.905544983823214e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1790
    },
    {
      "completion_length": 39.475,
      "epoch": 0.36,
      "grad_norm": 0.00136566162109375,
      "kl": 0.12392290020361543,
      "learning_rate": 4.903154239845798e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1800
    },
    {
      "completion_length": 57.725,
      "epoch": 0.362,
      "grad_norm": 0.0037689208984375,
      "kl": 0.7656503105536103,
      "learning_rate": 4.900734214192358e-06,
      "loss": 0.0001,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.1,
      "rewards/reward_func": -0.25,
      "step": 1810
    },
    {
      "completion_length": 51.025,
      "epoch": 0.364,
      "grad_norm": 0.00032806396484375,
      "kl": 1.4439625646919012,
      "learning_rate": 4.898284936350144e-06,
      "loss": 0.0001,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1820
    },
    {
      "completion_length": 83.85,
      "epoch": 0.366,
      "grad_norm": 0.00061798095703125,
      "kl": 0.013502365676686168,
      "learning_rate": 4.8958064361628334e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1830
    },
    {
      "completion_length": 57.3,
      "epoch": 0.368,
      "grad_norm": 0.00040435791015625,
      "kl": 0.02529239854775369,
      "learning_rate": 4.893298743830168e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1840
    },
    {
      "completion_length": 57.875,
      "epoch": 0.37,
      "grad_norm": 0.00067901611328125,
      "kl": 0.013202862720936537,
      "learning_rate": 4.890761889907589e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1850
    },
    {
      "completion_length": 61.725,
      "epoch": 0.372,
      "grad_norm": 0.00250244140625,
      "kl": 2.53927280055359,
      "learning_rate": 4.888195905305859e-06,
      "loss": 0.0003,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 1860
    },
    {
      "completion_length": 57.6,
      "epoch": 0.374,
      "grad_norm": 0.0003833770751953125,
      "kl": 0.02102891537360847,
      "learning_rate": 4.885600821290692e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1870
    },
    {
      "completion_length": 53.675,
      "epoch": 0.376,
      "grad_norm": 0.000583648681640625,
      "kl": 0.0170896818395704,
      "learning_rate": 4.882976669482368e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1880
    },
    {
      "completion_length": 47.525,
      "epoch": 0.378,
      "grad_norm": 0.0037384033203125,
      "kl": 0.01905378680676222,
      "learning_rate": 4.880323481855347e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1890
    },
    {
      "completion_length": 48.275,
      "epoch": 0.38,
      "grad_norm": 0.08203125,
      "kl": 0.1403908584266901,
      "learning_rate": 4.8776412907378845e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1900
    },
    {
      "completion_length": 73.425,
      "epoch": 0.382,
      "grad_norm": 0.000560760498046875,
      "kl": 0.007492217188701034,
      "learning_rate": 4.874930128811631e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1910
    },
    {
      "completion_length": 67.0,
      "epoch": 0.384,
      "grad_norm": 0.00122833251953125,
      "kl": 0.014595681196078658,
      "learning_rate": 4.8721900291112415e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1920
    },
    {
      "completion_length": 65.125,
      "epoch": 0.386,
      "grad_norm": 0.000484466552734375,
      "kl": 0.01310229734517634,
      "learning_rate": 4.869421025023965e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 1930
    },
    {
      "completion_length": 67.325,
      "epoch": 0.388,
      "grad_norm": 0.00048065185546875,
      "kl": 0.012927077431231736,
      "learning_rate": 4.866623150289241e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 1940
    },
    {
      "completion_length": 53.425,
      "epoch": 0.39,
      "grad_norm": 0.001068115234375,
      "kl": 0.009116059914231301,
      "learning_rate": 4.863796438998293e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1950
    },
    {
      "completion_length": 58.7,
      "epoch": 0.392,
      "grad_norm": 0.00176239013671875,
      "kl": 0.22789982098620384,
      "learning_rate": 4.860940925593703e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 1960
    },
    {
      "completion_length": 76.2,
      "epoch": 0.394,
      "grad_norm": 0.00179290771484375,
      "kl": 0.010708777070976793,
      "learning_rate": 4.858056644869002e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 1970
    },
    {
      "completion_length": 53.675,
      "epoch": 0.396,
      "grad_norm": 0.0010528564453125,
      "kl": 0.033514925348572436,
      "learning_rate": 4.855143631968242e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 1980
    },
    {
      "completion_length": 48.75,
      "epoch": 0.398,
      "grad_norm": 0.0004596710205078125,
      "kl": 0.02880375348031521,
      "learning_rate": 4.852201922385564e-06,
      "loss": 0.0,
      "match_ratio": 0.775,
      "reward": -0.225,
      "reward_std": 0.05,
      "rewards/reward_func": -0.225,
      "step": 1990
    },
    {
      "completion_length": 56.05,
      "epoch": 0.4,
      "grad_norm": 7.82012939453125e-05,
      "kl": 0.020885943528264762,
      "learning_rate": 4.849231551964771e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2000
    },
    {
      "completion_length": 57.825,
      "epoch": 0.402,
      "grad_norm": 0.0007781982421875,
      "kl": 0.013335178885608912,
      "learning_rate": 4.84623255689889e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2010
    },
    {
      "completion_length": 76.475,
      "epoch": 0.404,
      "grad_norm": 0.000461578369140625,
      "kl": 0.013225622242316604,
      "learning_rate": 4.84320497372973e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2020
    },
    {
      "completion_length": 65.45,
      "epoch": 0.406,
      "grad_norm": 0.0016632080078125,
      "kl": 0.020334804011508823,
      "learning_rate": 4.840148839347434e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 2030
    },
    {
      "completion_length": 87.85,
      "epoch": 0.408,
      "grad_norm": 0.000423431396484375,
      "kl": 0.020761342905461787,
      "learning_rate": 4.837064190990036e-06,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 2040
    },
    {
      "completion_length": 60.15,
      "epoch": 0.41,
      "grad_norm": 0.000713348388671875,
      "kl": 0.01624767268076539,
      "learning_rate": 4.833951066243004e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2050
    },
    {
      "completion_length": 52.075,
      "epoch": 0.412,
      "grad_norm": 0.007171630859375,
      "kl": 0.039952522004023196,
      "learning_rate": 4.830809503038781e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2060
    },
    {
      "completion_length": 64.925,
      "epoch": 0.414,
      "grad_norm": 0.00063323974609375,
      "kl": 0.4387159863486886,
      "learning_rate": 4.8276395396563215e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2070
    },
    {
      "completion_length": 52.35,
      "epoch": 0.416,
      "grad_norm": 0.000301361083984375,
      "kl": 0.018893744330853224,
      "learning_rate": 4.824441214720629e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2080
    },
    {
      "completion_length": 72.125,
      "epoch": 0.418,
      "grad_norm": 0.000850677490234375,
      "kl": 0.014612970128655433,
      "learning_rate": 4.821214567202284e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2090
    },
    {
      "completion_length": 62.525,
      "epoch": 0.42,
      "grad_norm": 0.003662109375,
      "kl": 0.11677258219569922,
      "learning_rate": 4.817959636416969e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2100
    },
    {
      "completion_length": 41.975,
      "epoch": 0.422,
      "grad_norm": 0.0006256103515625,
      "kl": 0.014599576778709888,
      "learning_rate": 4.814676462024988e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2110
    },
    {
      "completion_length": 61.65,
      "epoch": 0.424,
      "grad_norm": 0.00052642822265625,
      "kl": 0.010790122766047716,
      "learning_rate": 4.811365084030784e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2120
    },
    {
      "completion_length": 45.65,
      "epoch": 0.426,
      "grad_norm": 0.00095367431640625,
      "kl": 0.011601420305669307,
      "learning_rate": 4.808025542782453e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2130
    },
    {
      "completion_length": 64.25,
      "epoch": 0.428,
      "grad_norm": 0.0003986358642578125,
      "kl": 0.5641481504775584,
      "learning_rate": 4.804657878971252e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2140
    },
    {
      "completion_length": 56.675,
      "epoch": 0.43,
      "grad_norm": 0.000957489013671875,
      "kl": 0.013445794116705656,
      "learning_rate": 4.801262133631101e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2150
    },
    {
      "completion_length": 75.425,
      "epoch": 0.432,
      "grad_norm": 0.00055694580078125,
      "kl": 0.012692990363575518,
      "learning_rate": 4.7978383481380865e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2160
    },
    {
      "completion_length": 62.375,
      "epoch": 0.434,
      "grad_norm": 6.96875,
      "kl": 0.023554344521835448,
      "learning_rate": 4.794386564209953e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2170
    },
    {
      "completion_length": 65.475,
      "epoch": 0.436,
      "grad_norm": 0.0003337860107421875,
      "kl": 0.06298564318567515,
      "learning_rate": 4.790906823905599e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 2180
    },
    {
      "completion_length": 59.8,
      "epoch": 0.438,
      "grad_norm": 0.000537872314453125,
      "kl": 0.013637619884684682,
      "learning_rate": 4.787399169624562e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2190
    },
    {
      "completion_length": 54.875,
      "epoch": 0.44,
      "grad_norm": 0.000759124755859375,
      "kl": 0.01425664583221078,
      "learning_rate": 4.783863644106502e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2200
    },
    {
      "completion_length": 50.1,
      "epoch": 0.442,
      "grad_norm": 0.001800537109375,
      "kl": 0.0958622452802956,
      "learning_rate": 4.780300290430683e-06,
      "loss": 0.0,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 2210
    },
    {
      "completion_length": 66.8,
      "epoch": 0.444,
      "grad_norm": 0.00043487548828125,
      "kl": 0.00994320074096322,
      "learning_rate": 4.776709152015443e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2220
    },
    {
      "completion_length": 73.9,
      "epoch": 0.446,
      "grad_norm": 0.000705718994140625,
      "kl": 0.016998659167438746,
      "learning_rate": 4.773090272617672e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2230
    },
    {
      "completion_length": 65.45,
      "epoch": 0.448,
      "grad_norm": 0.00054931640625,
      "kl": 0.015969987539574505,
      "learning_rate": 4.769443696332272e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2240
    },
    {
      "completion_length": 58.0,
      "epoch": 0.45,
      "grad_norm": 0.000514984130859375,
      "kl": 0.05210723381023854,
      "learning_rate": 4.765769467591626e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 2250
    },
    {
      "completion_length": 55.35,
      "epoch": 0.452,
      "grad_norm": 0.0002841949462890625,
      "kl": 0.2783783482853323,
      "learning_rate": 4.762067631165049e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 2260
    },
    {
      "completion_length": 60.1,
      "epoch": 0.454,
      "grad_norm": 0.00119781494140625,
      "kl": 0.05332662384025753,
      "learning_rate": 4.7583382321582525e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2270
    },
    {
      "completion_length": 59.625,
      "epoch": 0.456,
      "grad_norm": 0.0014495849609375,
      "kl": 0.015380131197161973,
      "learning_rate": 4.754581316012785e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2280
    },
    {
      "completion_length": 56.1,
      "epoch": 0.458,
      "grad_norm": 0.000885009765625,
      "kl": 0.04605462467297912,
      "learning_rate": 4.750796928505484e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2290
    },
    {
      "completion_length": 76.35,
      "epoch": 0.46,
      "grad_norm": 0.000579833984375,
      "kl": 0.053115089796483515,
      "learning_rate": 4.746985115747918e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2300
    },
    {
      "completion_length": 50.8,
      "epoch": 0.462,
      "grad_norm": 0.000637054443359375,
      "kl": 0.01561843790113926,
      "learning_rate": 4.743145924185821e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2310
    },
    {
      "completion_length": 65.575,
      "epoch": 0.464,
      "grad_norm": 0.00061798095703125,
      "kl": 0.01594538043718785,
      "learning_rate": 4.7392794005985324e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2320
    },
    {
      "completion_length": 44.15,
      "epoch": 0.466,
      "grad_norm": 0.0005340576171875,
      "kl": 12.877768159005791,
      "learning_rate": 4.735385592098421e-06,
      "loss": 0.0013,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 2330
    },
    {
      "completion_length": 51.45,
      "epoch": 0.468,
      "grad_norm": 0.00055694580078125,
      "kl": 0.019627093384042383,
      "learning_rate": 4.731464546130315e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2340
    },
    {
      "completion_length": 67.975,
      "epoch": 0.47,
      "grad_norm": 0.0024261474609375,
      "kl": 0.018453579442575575,
      "learning_rate": 4.72751631047092e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 2350
    },
    {
      "completion_length": 73.7,
      "epoch": 0.472,
      "grad_norm": 0.00049591064453125,
      "kl": 0.011441022157669067,
      "learning_rate": 4.723540933228245e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2360
    },
    {
      "completion_length": 68.525,
      "epoch": 0.474,
      "grad_norm": 0.000537872314453125,
      "kl": 0.010118643706664442,
      "learning_rate": 4.719538462841003e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2370
    },
    {
      "completion_length": 42.325,
      "epoch": 0.476,
      "grad_norm": 0.00110626220703125,
      "kl": 23.094405939802527,
      "learning_rate": 4.715508948078037e-06,
      "loss": 0.0023,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 2380
    },
    {
      "completion_length": 51.85,
      "epoch": 0.478,
      "grad_norm": 0.0002574920654296875,
      "kl": 0.01785165797919035,
      "learning_rate": 4.71145243803771e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2390
    },
    {
      "completion_length": 76.575,
      "epoch": 0.48,
      "grad_norm": 0.00049591064453125,
      "kl": 0.02041715644299984,
      "learning_rate": 4.707368982147318e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2400
    },
    {
      "completion_length": 53.65,
      "epoch": 0.482,
      "grad_norm": 0.0007171630859375,
      "kl": 0.011072598048485816,
      "learning_rate": 4.703258630162481e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2410
    },
    {
      "completion_length": 45.65,
      "epoch": 0.484,
      "grad_norm": 0.0010986328125,
      "kl": 1306590.1205121286,
      "learning_rate": 4.699121432166542e-06,
      "loss": 130.659,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2420
    },
    {
      "completion_length": 66.625,
      "epoch": 0.486,
      "grad_norm": 0.0004425048828125,
      "kl": 0.018537986697629093,
      "learning_rate": 4.6949574385699514e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2430
    },
    {
      "completion_length": 50.825,
      "epoch": 0.488,
      "grad_norm": 0.00121307373046875,
      "kl": 0.027628638222813605,
      "learning_rate": 4.690766700109659e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2440
    },
    {
      "completion_length": 72.275,
      "epoch": 0.49,
      "grad_norm": 0.0006103515625,
      "kl": 0.012821279000490904,
      "learning_rate": 4.68654926784849e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2450
    },
    {
      "completion_length": 59.025,
      "epoch": 0.492,
      "grad_norm": 0.0006256103515625,
      "kl": 1.4881786234676837,
      "learning_rate": 4.682305193174524e-06,
      "loss": 0.0001,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 2460
    },
    {
      "completion_length": 70.65,
      "epoch": 0.494,
      "grad_norm": 32.25,
      "kl": 0.13413287354633213,
      "learning_rate": 4.6780345278004744e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2470
    },
    {
      "completion_length": 60.875,
      "epoch": 0.496,
      "grad_norm": 0.00067138671875,
      "kl": 0.022409677878022193,
      "learning_rate": 4.673737323763048e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2480
    },
    {
      "completion_length": 57.875,
      "epoch": 0.498,
      "grad_norm": 18.75,
      "kl": 0.49180023511871696,
      "learning_rate": 4.669413633422322e-06,
      "loss": 0.0,
      "match_ratio": 0.775,
      "reward": -0.225,
      "reward_std": 0.05,
      "rewards/reward_func": -0.225,
      "step": 2490
    },
    {
      "completion_length": 63.95,
      "epoch": 0.5,
      "grad_norm": 0.000720977783203125,
      "kl": 0.0207068151794374,
      "learning_rate": 4.665063509461098e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2500
    },
    {
      "completion_length": 70.75,
      "epoch": 0.502,
      "grad_norm": 0.0004634857177734375,
      "kl": 0.01583680328913033,
      "learning_rate": 4.6606870048842626e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2510
    },
    {
      "completion_length": 41.6,
      "epoch": 0.504,
      "grad_norm": 0.001251220703125,
      "kl": 0.024196008208673448,
      "learning_rate": 4.656284173018144e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2520
    },
    {
      "completion_length": 60.475,
      "epoch": 0.506,
      "grad_norm": 0.0008697509765625,
      "kl": 0.02790404809638858,
      "learning_rate": 4.65185506750986e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2530
    },
    {
      "completion_length": 63.1,
      "epoch": 0.508,
      "grad_norm": 0.0005950927734375,
      "kl": 0.037049750238656996,
      "learning_rate": 4.6473997423266615e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2540
    },
    {
      "completion_length": 64.025,
      "epoch": 0.51,
      "grad_norm": 0.00121307373046875,
      "kl": 0.05311856884509325,
      "learning_rate": 4.642918251755281e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2550
    },
    {
      "completion_length": 48.925,
      "epoch": 0.512,
      "grad_norm": 0.00482177734375,
      "kl": 0.07079303860664368,
      "learning_rate": 4.638410650401267e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2560
    },
    {
      "completion_length": 61.8,
      "epoch": 0.514,
      "grad_norm": 0.0010986328125,
      "kl": 0.024926586542278528,
      "learning_rate": 4.633876993188319e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2570
    },
    {
      "completion_length": 49.6,
      "epoch": 0.516,
      "grad_norm": 0.000766754150390625,
      "kl": 0.019125528051517904,
      "learning_rate": 4.62931733535762e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2580
    },
    {
      "completion_length": 66.35,
      "epoch": 0.518,
      "grad_norm": 0.00159454345703125,
      "kl": 0.020169223845005035,
      "learning_rate": 4.62473173246716e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2590
    },
    {
      "completion_length": 55.7,
      "epoch": 0.52,
      "grad_norm": 0.00067138671875,
      "kl": 0.24018120649270713,
      "learning_rate": 4.620120240391065e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2600
    },
    {
      "completion_length": 57.725,
      "epoch": 0.522,
      "grad_norm": 0.00087738037109375,
      "kl": 2.7745140019804237,
      "learning_rate": 4.6154829153189105e-06,
      "loss": 0.0003,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 2610
    },
    {
      "completion_length": 48.7,
      "epoch": 0.524,
      "grad_norm": 0.000659942626953125,
      "kl": 0.023752238228917123,
      "learning_rate": 4.610819813755038e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2620
    },
    {
      "completion_length": 57.225,
      "epoch": 0.526,
      "grad_norm": 0.00150299072265625,
      "kl": 0.02745365663431585,
      "learning_rate": 4.60613099251787e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2630
    },
    {
      "completion_length": 48.5,
      "epoch": 0.528,
      "grad_norm": 0.00133514404296875,
      "kl": 0.021794071048498155,
      "learning_rate": 4.601416508739211e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2640
    },
    {
      "completion_length": 61.7,
      "epoch": 0.53,
      "grad_norm": 0.000823974609375,
      "kl": 0.01592640457674861,
      "learning_rate": 4.596676419863561e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2650
    },
    {
      "completion_length": 55.825,
      "epoch": 0.532,
      "grad_norm": 0.000431060791015625,
      "kl": 4393.635703391675,
      "learning_rate": 4.591910783647405e-06,
      "loss": 0.4394,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2660
    },
    {
      "completion_length": 42.725,
      "epoch": 0.534,
      "grad_norm": 0.0005340576171875,
      "kl": 3.308469070494175,
      "learning_rate": 4.587119658158517e-06,
      "loss": 0.0003,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2670
    },
    {
      "completion_length": 53.675,
      "epoch": 0.536,
      "grad_norm": 0.000362396240234375,
      "kl": 0.017304986575618388,
      "learning_rate": 4.582303101775249e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2680
    },
    {
      "completion_length": 46.825,
      "epoch": 0.538,
      "grad_norm": 0.000438690185546875,
      "kl": 1.4977983684279024,
      "learning_rate": 4.577461173185821e-06,
      "loss": 0.0001,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2690
    },
    {
      "completion_length": 49.425,
      "epoch": 0.54,
      "grad_norm": 0.000637054443359375,
      "kl": 0.04018927337601781,
      "learning_rate": 4.572593931387604e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2700
    },
    {
      "completion_length": 61.475,
      "epoch": 0.542,
      "grad_norm": 0.00025177001953125,
      "kl": 0.029250907758250833,
      "learning_rate": 4.567701435686405e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2710
    },
    {
      "completion_length": 61.3,
      "epoch": 0.544,
      "grad_norm": 0.00107574462890625,
      "kl": 0.041718969354406,
      "learning_rate": 4.562783745695738e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 2720
    },
    {
      "completion_length": 71.225,
      "epoch": 0.546,
      "grad_norm": 0.00101470947265625,
      "kl": 0.017608029022812843,
      "learning_rate": 4.5578409213361055e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2730
    },
    {
      "completion_length": 68.825,
      "epoch": 0.548,
      "grad_norm": 0.00084686279296875,
      "kl": 0.01787120271474123,
      "learning_rate": 4.55287302283426e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2740
    },
    {
      "completion_length": 40.975,
      "epoch": 0.55,
      "grad_norm": 0.0004405975341796875,
      "kl": 0.08943550041876733,
      "learning_rate": 4.54788011072248e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2750
    },
    {
      "completion_length": 60.275,
      "epoch": 0.552,
      "grad_norm": 0.000888824462890625,
      "kl": 0.012318810448050499,
      "learning_rate": 4.542862245837821e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2760
    },
    {
      "completion_length": 50.225,
      "epoch": 0.554,
      "grad_norm": 0.000396728515625,
      "kl": 0.1667893348261714,
      "learning_rate": 4.537819489321385e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2770
    },
    {
      "completion_length": 71.5,
      "epoch": 0.556,
      "grad_norm": 0.000873565673828125,
      "kl": 0.009898501250427216,
      "learning_rate": 4.5327519026175694e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2780
    },
    {
      "completion_length": 66.525,
      "epoch": 0.558,
      "grad_norm": 0.0013275146484375,
      "kl": 0.016875687218271197,
      "learning_rate": 4.527659547473317e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2790
    },
    {
      "completion_length": 54.8,
      "epoch": 0.56,
      "grad_norm": 0.000308990478515625,
      "kl": 0.40478452597744763,
      "learning_rate": 4.522542485937369e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 2800
    },
    {
      "completion_length": 63.175,
      "epoch": 0.562,
      "grad_norm": 0.002197265625,
      "kl": 0.01588670499622822,
      "learning_rate": 4.517400780359505e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2810
    },
    {
      "completion_length": 66.025,
      "epoch": 0.564,
      "grad_norm": 0.0015411376953125,
      "kl": 0.9499945601448416,
      "learning_rate": 4.512234493389785e-06,
      "loss": 0.0001,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 2820
    },
    {
      "completion_length": 52.925,
      "epoch": 0.566,
      "grad_norm": 0.000408172607421875,
      "kl": 0.7477384469937534,
      "learning_rate": 4.507043687977787e-06,
      "loss": 0.0001,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 2830
    },
    {
      "completion_length": 54.05,
      "epoch": 0.568,
      "grad_norm": 0.00145721435546875,
      "kl": 0.02171561080031097,
      "learning_rate": 4.501828427371834e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2840
    },
    {
      "completion_length": 67.425,
      "epoch": 0.57,
      "grad_norm": 0.00038909912109375,
      "kl": 46.97684473299887,
      "learning_rate": 4.496588775118232e-06,
      "loss": 0.0047,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 2850
    },
    {
      "completion_length": 49.9,
      "epoch": 0.572,
      "grad_norm": 0.000713348388671875,
      "kl": 0.04041039999574423,
      "learning_rate": 4.491324795060491e-06,
      "loss": 0.0,
      "match_ratio": 0.7,
      "reward": -0.3,
      "reward_std": 0.0,
      "rewards/reward_func": -0.3,
      "step": 2860
    },
    {
      "completion_length": 46.8,
      "epoch": 0.574,
      "grad_norm": 0.0003948211669921875,
      "kl": 0.013312188815325499,
      "learning_rate": 4.4860365513385456e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2870
    },
    {
      "completion_length": 46.825,
      "epoch": 0.576,
      "grad_norm": 0.0003490447998046875,
      "kl": 0.4708886262029409,
      "learning_rate": 4.4807241083879774e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 2880
    },
    {
      "completion_length": 65.625,
      "epoch": 0.578,
      "grad_norm": 0.00121307373046875,
      "kl": 0.0194290304556489,
      "learning_rate": 4.475387530939226e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2890
    },
    {
      "completion_length": 56.075,
      "epoch": 0.58,
      "grad_norm": 0.000579833984375,
      "kl": 0.04038618067279458,
      "learning_rate": 4.470026884016805e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 2900
    },
    {
      "completion_length": 47.725,
      "epoch": 0.582,
      "grad_norm": 0.00043487548828125,
      "kl": 16.41964945977088,
      "learning_rate": 4.464642232938505e-06,
      "loss": 0.0016,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 2910
    },
    {
      "completion_length": 63.9,
      "epoch": 0.584,
      "grad_norm": 0.0013885498046875,
      "kl": 0.01390684423968196,
      "learning_rate": 4.4592336433146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2920
    },
    {
      "completion_length": 44.3,
      "epoch": 0.586,
      "grad_norm": 0.00023651123046875,
      "kl": 0.010492815752513707,
      "learning_rate": 4.453801181047047e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2930
    },
    {
      "completion_length": 72.45,
      "epoch": 0.588,
      "grad_norm": 0.00032806396484375,
      "kl": 0.01113151153549552,
      "learning_rate": 4.448344912328686e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2940
    },
    {
      "completion_length": 61.05,
      "epoch": 0.59,
      "grad_norm": 0.0004787445068359375,
      "kl": 0.008429582207463681,
      "learning_rate": 4.442864903642428e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2950
    },
    {
      "completion_length": 51.9,
      "epoch": 0.592,
      "grad_norm": 0.00080108642578125,
      "kl": 0.0231597448233515,
      "learning_rate": 4.437361221760449e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2960
    },
    {
      "completion_length": 47.825,
      "epoch": 0.594,
      "grad_norm": 0.0002899169921875,
      "kl": 0.03416364281438291,
      "learning_rate": 4.431833933743378e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 2970
    },
    {
      "completion_length": 54.475,
      "epoch": 0.596,
      "grad_norm": 29.75,
      "kl": 25.77551784273237,
      "learning_rate": 4.426283106939474e-06,
      "loss": 0.0026,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 2980
    },
    {
      "completion_length": 61.175,
      "epoch": 0.598,
      "grad_norm": 0.00060272216796875,
      "kl": 0.010949767334386707,
      "learning_rate": 4.420708808983809e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 2990
    },
    {
      "completion_length": 58.05,
      "epoch": 0.6,
      "grad_norm": 0.00049591064453125,
      "kl": 0.03518106024712324,
      "learning_rate": 4.415111107797445e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3000
    },
    {
      "completion_length": 48.7,
      "epoch": 0.602,
      "grad_norm": 0.0006103515625,
      "kl": 0.015348105784505605,
      "learning_rate": 4.409490071586606e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3010
    },
    {
      "completion_length": 59.9,
      "epoch": 0.604,
      "grad_norm": 0.0004119873046875,
      "kl": 0.011616118438541888,
      "learning_rate": 4.403845768841842e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3020
    },
    {
      "completion_length": 69.0,
      "epoch": 0.606,
      "grad_norm": 0.000640869140625,
      "kl": 0.011708037834614516,
      "learning_rate": 4.398178268337202e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3030
    },
    {
      "completion_length": 52.125,
      "epoch": 0.608,
      "grad_norm": 0.000896453857421875,
      "kl": 0.04663766893791035,
      "learning_rate": 4.3924876391293915e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3040
    },
    {
      "completion_length": 54.175,
      "epoch": 0.61,
      "grad_norm": 0.000701904296875,
      "kl": 9.442724062688649,
      "learning_rate": 4.386773950556931e-06,
      "loss": 0.0009,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 3050
    },
    {
      "completion_length": 51.075,
      "epoch": 0.612,
      "grad_norm": 25.75,
      "kl": 18.313499209098516,
      "learning_rate": 4.381037272239311e-06,
      "loss": 0.0018,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 3060
    },
    {
      "completion_length": 59.45,
      "epoch": 0.614,
      "grad_norm": 0.00145721435546875,
      "kl": 0.05719580026343465,
      "learning_rate": 4.3752776740761495e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 3070
    },
    {
      "completion_length": 59.75,
      "epoch": 0.616,
      "grad_norm": 0.000865936279296875,
      "kl": 955.0316817238461,
      "learning_rate": 4.36949522624633e-06,
      "loss": 0.0955,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.1,
      "rewards/reward_func": -0.15,
      "step": 3080
    },
    {
      "completion_length": 49.25,
      "epoch": 0.618,
      "grad_norm": 0.0947265625,
      "kl": 0.12424529809504747,
      "learning_rate": 4.3636899992071555e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 3090
    },
    {
      "completion_length": 63.45,
      "epoch": 0.62,
      "grad_norm": 0.000774383544921875,
      "kl": 0.021155705489218236,
      "learning_rate": 4.357862063693486e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3100
    },
    {
      "completion_length": 58.675,
      "epoch": 0.622,
      "grad_norm": 0.00130462646484375,
      "kl": 0.03269129507243633,
      "learning_rate": 4.352011490716875e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3110
    },
    {
      "completion_length": 51.25,
      "epoch": 0.624,
      "grad_norm": 0.000720977783203125,
      "kl": 0.01852965746074915,
      "learning_rate": 4.346138351564711e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3120
    },
    {
      "completion_length": 42.025,
      "epoch": 0.626,
      "grad_norm": 0.0014495849609375,
      "kl": 0.13662478388287128,
      "learning_rate": 4.340242717799337e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 3130
    },
    {
      "completion_length": 55.225,
      "epoch": 0.628,
      "grad_norm": 0.0002536773681640625,
      "kl": 0.011004617274738848,
      "learning_rate": 4.334324661257191e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3140
    },
    {
      "completion_length": 36.075,
      "epoch": 0.63,
      "grad_norm": 0.000701904296875,
      "kl": 0.03598860376514494,
      "learning_rate": 4.328384254047927e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3150
    },
    {
      "completion_length": 70.7,
      "epoch": 0.632,
      "grad_norm": 0.0004825592041015625,
      "kl": 0.014078293647617101,
      "learning_rate": 4.322421568553529e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3160
    },
    {
      "completion_length": 41.4,
      "epoch": 0.634,
      "grad_norm": 0.00093841552734375,
      "kl": 0.02731174589134753,
      "learning_rate": 4.316436677427441e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3170
    },
    {
      "completion_length": 62.625,
      "epoch": 0.636,
      "grad_norm": 0.00035858154296875,
      "kl": 0.015678783506155015,
      "learning_rate": 4.3104296535936695e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3180
    },
    {
      "completion_length": 61.0,
      "epoch": 0.638,
      "grad_norm": 0.00091552734375,
      "kl": 0.019854954723268748,
      "learning_rate": 4.3044005702459055e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3190
    },
    {
      "completion_length": 52.95,
      "epoch": 0.64,
      "grad_norm": 0.000514984130859375,
      "kl": 0.13356436253525317,
      "learning_rate": 4.2983495008466285e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 3200
    },
    {
      "completion_length": 66.475,
      "epoch": 0.642,
      "grad_norm": 0.000316619873046875,
      "kl": 0.013267815671861171,
      "learning_rate": 4.2922765191262075e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3210
    },
    {
      "completion_length": 66.475,
      "epoch": 0.644,
      "grad_norm": 0.000701904296875,
      "kl": 0.07887385552749038,
      "learning_rate": 4.286181699082008e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 3220
    },
    {
      "completion_length": 61.55,
      "epoch": 0.646,
      "grad_norm": 0.000118255615234375,
      "kl": 0.011934885568916798,
      "learning_rate": 4.280065114977492e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3230
    },
    {
      "completion_length": 61.625,
      "epoch": 0.648,
      "grad_norm": 0.0004558563232421875,
      "kl": 0.015098626213148236,
      "learning_rate": 4.273926841341303e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3240
    },
    {
      "completion_length": 47.7,
      "epoch": 0.65,
      "grad_norm": 0.00048065185546875,
      "kl": 0.017614057380706073,
      "learning_rate": 4.267766952966369e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3250
    },
    {
      "completion_length": 47.35,
      "epoch": 0.652,
      "grad_norm": 0.000911712646484375,
      "kl": 19.53749562408775,
      "learning_rate": 4.261585524908987e-06,
      "loss": 0.002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 3260
    },
    {
      "completion_length": 67.85,
      "epoch": 0.654,
      "grad_norm": 0.0004367828369140625,
      "kl": 0.011493841698393226,
      "learning_rate": 4.255382632487907e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3270
    },
    {
      "completion_length": 43.525,
      "epoch": 0.656,
      "grad_norm": 0.0004062652587890625,
      "kl": 0.06476088264025748,
      "learning_rate": 4.249158351283414e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 3280
    },
    {
      "completion_length": 50.575,
      "epoch": 0.658,
      "grad_norm": 0.000812530517578125,
      "kl": 0.044670914835296574,
      "learning_rate": 4.242912757136412e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3290
    },
    {
      "completion_length": 48.95,
      "epoch": 0.66,
      "grad_norm": 0.054931640625,
      "kl": 0.05593093540519476,
      "learning_rate": 4.236645926147493e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3300
    },
    {
      "completion_length": 55.425,
      "epoch": 0.662,
      "grad_norm": 0.0004024505615234375,
      "kl": 0.02145648035220802,
      "learning_rate": 4.230357934676017e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3310
    },
    {
      "completion_length": 62.2,
      "epoch": 0.664,
      "grad_norm": 0.000499725341796875,
      "kl": 0.012483126670122146,
      "learning_rate": 4.224048859339175e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3320
    },
    {
      "completion_length": 61.9,
      "epoch": 0.666,
      "grad_norm": 0.0002689361572265625,
      "kl": 0.03277284097857773,
      "learning_rate": 4.217718777011058e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3330
    },
    {
      "completion_length": 70.675,
      "epoch": 0.668,
      "grad_norm": 0.00103759765625,
      "kl": 0.32424843702465295,
      "learning_rate": 4.211367764821722e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 3340
    },
    {
      "completion_length": 41.375,
      "epoch": 0.67,
      "grad_norm": 0.0004367828369140625,
      "kl": 0.05056889692787081,
      "learning_rate": 4.204995900156247e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 3350
    },
    {
      "completion_length": 59.675,
      "epoch": 0.672,
      "grad_norm": 0.00106048583984375,
      "kl": 0.04967752741649747,
      "learning_rate": 4.198603260653792e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3360
    },
    {
      "completion_length": 53.425,
      "epoch": 0.674,
      "grad_norm": 0.0003681182861328125,
      "kl": 0.01580333085730672,
      "learning_rate": 4.192189924206652e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3370
    },
    {
      "completion_length": 50.75,
      "epoch": 0.676,
      "grad_norm": 0.002899169921875,
      "kl": 1.8218023491092026,
      "learning_rate": 4.185755968959308e-06,
      "loss": 0.0002,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 3380
    },
    {
      "completion_length": 62.375,
      "epoch": 0.678,
      "grad_norm": 0.000156402587890625,
      "kl": 0.047494524717330934,
      "learning_rate": 4.179301473307476e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3390
    },
    {
      "completion_length": 69.3,
      "epoch": 0.68,
      "grad_norm": 0.0002651214599609375,
      "kl": 0.014598681312054395,
      "learning_rate": 4.172826515897146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3400
    },
    {
      "completion_length": 63.575,
      "epoch": 0.682,
      "grad_norm": 0.000423431396484375,
      "kl": 0.04025569665245712,
      "learning_rate": 4.166331175623631e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3410
    },
    {
      "completion_length": 48.05,
      "epoch": 0.684,
      "grad_norm": 0.0006561279296875,
      "kl": 913.4162682918599,
      "learning_rate": 4.159815531630604e-06,
      "loss": 0.0913,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 3420
    },
    {
      "completion_length": 52.575,
      "epoch": 0.686,
      "grad_norm": 0.0006561279296875,
      "kl": 0.0610341252759099,
      "learning_rate": 4.15327966330913e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3430
    },
    {
      "completion_length": 57.6,
      "epoch": 0.688,
      "grad_norm": 0.0024871826171875,
      "kl": 0.9020142253488302,
      "learning_rate": 4.146723650296701e-06,
      "loss": 0.0001,
      "match_ratio": 0.85,
      "reward": -0.15,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.15,
      "step": 3440
    },
    {
      "completion_length": 56.25,
      "epoch": 0.69,
      "grad_norm": 0.0045166015625,
      "kl": 0.14730083039030434,
      "learning_rate": 4.140147572476269e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3450
    },
    {
      "completion_length": 72.55,
      "epoch": 0.692,
      "grad_norm": 0.0004024505615234375,
      "kl": 0.05645229946821928,
      "learning_rate": 4.133551509975264e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3460
    },
    {
      "completion_length": 62.0,
      "epoch": 0.694,
      "grad_norm": 0.00063323974609375,
      "kl": 0.018445250298827886,
      "learning_rate": 4.126935543164628e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3470
    },
    {
      "completion_length": 54.125,
      "epoch": 0.696,
      "grad_norm": 0.004608154296875,
      "kl": 0.03874910874292255,
      "learning_rate": 4.120299752657828e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 3480
    },
    {
      "completion_length": 35.4,
      "epoch": 0.698,
      "grad_norm": 0.020751953125,
      "kl": 0.07965331296436488,
      "learning_rate": 4.113644219309877e-06,
      "loss": 0.0,
      "match_ratio": 0.675,
      "reward": -0.325,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.325,
      "step": 3490
    },
    {
      "completion_length": 61.6,
      "epoch": 0.7,
      "grad_norm": 0.00079345703125,
      "kl": 0.02872077892534435,
      "learning_rate": 4.106969024216348e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3500
    },
    {
      "completion_length": 57.175,
      "epoch": 0.702,
      "grad_norm": 0.00157928466796875,
      "kl": 0.017641184292733668,
      "learning_rate": 4.1002742487123896e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3510
    },
    {
      "completion_length": 62.6,
      "epoch": 0.704,
      "grad_norm": 0.0002651214599609375,
      "kl": 0.020063164038583638,
      "learning_rate": 4.093559974371725e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3520
    },
    {
      "completion_length": 51.125,
      "epoch": 0.706,
      "grad_norm": 0.000576019287109375,
      "kl": 0.6725238669663668,
      "learning_rate": 4.086826283005669e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 3530
    },
    {
      "completion_length": 56.625,
      "epoch": 0.708,
      "grad_norm": 0.000579833984375,
      "kl": 0.01952581750229001,
      "learning_rate": 4.080073256662128e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3540
    },
    {
      "completion_length": 62.7,
      "epoch": 0.71,
      "grad_norm": 0.0035247802734375,
      "kl": 0.07669782191514969,
      "learning_rate": 4.073300977624594e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 3550
    },
    {
      "completion_length": 68.325,
      "epoch": 0.712,
      "grad_norm": 0.0014801025390625,
      "kl": 0.026417199242860078,
      "learning_rate": 4.066509528411151e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3560
    },
    {
      "completion_length": 68.925,
      "epoch": 0.714,
      "grad_norm": 0.0003643035888671875,
      "kl": 0.01686573908664286,
      "learning_rate": 4.059698991773466e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3570
    },
    {
      "completion_length": 65.325,
      "epoch": 0.716,
      "grad_norm": 0.0003261566162109375,
      "kl": 0.009878239961108193,
      "learning_rate": 4.052869450695776e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3580
    },
    {
      "completion_length": 54.125,
      "epoch": 0.718,
      "grad_norm": 0.00054168701171875,
      "kl": 0.017347801569849254,
      "learning_rate": 4.046020988393886e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3590
    },
    {
      "completion_length": 62.45,
      "epoch": 0.72,
      "grad_norm": 0.000362396240234375,
      "kl": 0.010593670699745417,
      "learning_rate": 4.039153688314146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3600
    },
    {
      "completion_length": 56.225,
      "epoch": 0.722,
      "grad_norm": 0.0007476806640625,
      "kl": 20.069431526213883,
      "learning_rate": 4.032267634132442e-06,
      "loss": 0.002,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 3610
    },
    {
      "completion_length": 61.925,
      "epoch": 0.724,
      "grad_norm": 0.0003643035888671875,
      "kl": 0.023581979051232337,
      "learning_rate": 4.02536290975317e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3620
    },
    {
      "completion_length": 71.05,
      "epoch": 0.726,
      "grad_norm": 0.0006561279296875,
      "kl": 0.019050255604088306,
      "learning_rate": 4.018439599308217e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3630
    },
    {
      "completion_length": 63.85,
      "epoch": 0.728,
      "grad_norm": 0.0002498626708984375,
      "kl": 0.023293742351233958,
      "learning_rate": 4.011497787155938e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3640
    },
    {
      "completion_length": 72.65,
      "epoch": 0.73,
      "grad_norm": 0.000347137451171875,
      "kl": 0.011120679695159197,
      "learning_rate": 4.0045375578801216e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3650
    },
    {
      "completion_length": 62.95,
      "epoch": 0.732,
      "grad_norm": 0.00042724609375,
      "kl": 0.017660227511078118,
      "learning_rate": 3.997558996288965e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3660
    },
    {
      "completion_length": 56.05,
      "epoch": 0.734,
      "grad_norm": 0.000881195068359375,
      "kl": 0.35850770082324745,
      "learning_rate": 3.9905621874140396e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3670
    },
    {
      "completion_length": 50.5,
      "epoch": 0.736,
      "grad_norm": 0.0004291534423828125,
      "kl": 0.022859503608196975,
      "learning_rate": 3.983547216509254e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3680
    },
    {
      "completion_length": 65.675,
      "epoch": 0.738,
      "grad_norm": 0.0003795623779296875,
      "kl": 0.01089323298074305,
      "learning_rate": 3.976514169049814e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3690
    },
    {
      "completion_length": 55.825,
      "epoch": 0.74,
      "grad_norm": 0.0002765655517578125,
      "kl": 0.026947349560214207,
      "learning_rate": 3.969463130731183e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3700
    },
    {
      "completion_length": 51.9,
      "epoch": 0.742,
      "grad_norm": 0.000720977783203125,
      "kl": 101.020502169244,
      "learning_rate": 3.96239418746804e-06,
      "loss": 0.0101,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3710
    },
    {
      "completion_length": 41.925,
      "epoch": 0.744,
      "grad_norm": 0.0004367828369140625,
      "kl": 1.0482193630887195,
      "learning_rate": 3.955307425393224e-06,
      "loss": 0.0001,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3720
    },
    {
      "completion_length": 43.075,
      "epoch": 0.746,
      "grad_norm": 0.0010986328125,
      "kl": 0.011403680918738246,
      "learning_rate": 3.948202930856697e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3730
    },
    {
      "completion_length": 64.75,
      "epoch": 0.748,
      "grad_norm": 0.0004062652587890625,
      "kl": 0.1115244179032743,
      "learning_rate": 3.941080790424483e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3740
    },
    {
      "completion_length": 59.65,
      "epoch": 0.75,
      "grad_norm": 0.000553131103515625,
      "kl": 0.011348503362387418,
      "learning_rate": 3.933941090877615e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3750
    },
    {
      "completion_length": 50.1,
      "epoch": 0.752,
      "grad_norm": 0.0004749298095703125,
      "kl": 0.01582015200983733,
      "learning_rate": 3.92678391921108e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3760
    },
    {
      "completion_length": 58.2,
      "epoch": 0.754,
      "grad_norm": 0.0004024505615234375,
      "kl": 0.02131882361136377,
      "learning_rate": 3.9196093626327535e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3770
    },
    {
      "completion_length": 56.45,
      "epoch": 0.756,
      "grad_norm": 0.000766754150390625,
      "kl": 0.024744509416632355,
      "learning_rate": 3.912417508562345e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3780
    },
    {
      "completion_length": 48.5,
      "epoch": 0.758,
      "grad_norm": 0.0005950927734375,
      "kl": 0.015975080896168947,
      "learning_rate": 3.905208444630326e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3790
    },
    {
      "completion_length": 69.525,
      "epoch": 0.76,
      "grad_norm": 0.000537872314453125,
      "kl": 0.009961457317695021,
      "learning_rate": 3.897982258676867e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 3800
    },
    {
      "completion_length": 60.575,
      "epoch": 0.762,
      "grad_norm": 0.000507354736328125,
      "kl": 0.011210405128076672,
      "learning_rate": 3.890739038750763e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3810
    },
    {
      "completion_length": 46.2,
      "epoch": 0.764,
      "grad_norm": 0.000255584716796875,
      "kl": 1.3439840027829633,
      "learning_rate": 3.88347887310836e-06,
      "loss": 0.0001,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 3820
    },
    {
      "completion_length": 54.325,
      "epoch": 0.766,
      "grad_norm": 0.000637054443359375,
      "kl": 0.09195185881108045,
      "learning_rate": 3.876201850212489e-06,
      "loss": 0.0,
      "match_ratio": 0.675,
      "reward": -0.325,
      "reward_std": 0.05,
      "rewards/reward_func": -0.325,
      "step": 3830
    },
    {
      "completion_length": 74.35,
      "epoch": 0.768,
      "grad_norm": 0.000225067138671875,
      "kl": 0.01327997730113566,
      "learning_rate": 3.868908058731376e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3840
    },
    {
      "completion_length": 50.125,
      "epoch": 0.77,
      "grad_norm": 0.000370025634765625,
      "kl": 0.0457455332390964,
      "learning_rate": 3.861597587537568e-06,
      "loss": 0.0,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.05,
      "rewards/reward_func": -0.125,
      "step": 3850
    },
    {
      "completion_length": 67.225,
      "epoch": 0.772,
      "grad_norm": 0.0009765625,
      "kl": 0.019194579031318427,
      "learning_rate": 3.85427052570685e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3860
    },
    {
      "completion_length": 67.425,
      "epoch": 0.774,
      "grad_norm": 0.000522613525390625,
      "kl": 0.16493179565295576,
      "learning_rate": 3.846926962517158e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 3870
    },
    {
      "completion_length": 58.725,
      "epoch": 0.776,
      "grad_norm": 0.0007171630859375,
      "kl": 0.0165805596858263,
      "learning_rate": 3.839566987447492e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3880
    },
    {
      "completion_length": 64.575,
      "epoch": 0.778,
      "grad_norm": 0.00061798095703125,
      "kl": 0.014522301172837615,
      "learning_rate": 3.832190690176825e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3890
    },
    {
      "completion_length": 49.6,
      "epoch": 0.78,
      "grad_norm": 5.435943603515625e-05,
      "kl": 0.014127893140539527,
      "learning_rate": 3.824798160583012e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3900
    },
    {
      "completion_length": 49.8,
      "epoch": 0.782,
      "grad_norm": 0.000278472900390625,
      "kl": 0.11558867986313999,
      "learning_rate": 3.817389488741694e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.0,
      "rewards/reward_func": -0.2,
      "step": 3910
    },
    {
      "completion_length": 65.35,
      "epoch": 0.784,
      "grad_norm": 0.000690460205078125,
      "kl": 0.013915874017402529,
      "learning_rate": 3.8099647649251984e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3920
    },
    {
      "completion_length": 63.1,
      "epoch": 0.786,
      "grad_norm": 0.000270843505859375,
      "kl": 0.26083877284545454,
      "learning_rate": 3.802524079601442e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3930
    },
    {
      "completion_length": 57.475,
      "epoch": 0.788,
      "grad_norm": 0.000476837158203125,
      "kl": 0.44110607262700796,
      "learning_rate": 3.795067523432826e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 3940
    },
    {
      "completion_length": 50.125,
      "epoch": 0.79,
      "grad_norm": 0.0004062652587890625,
      "kl": 0.03131135320290923,
      "learning_rate": 3.787595187275136e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3950
    },
    {
      "completion_length": 47.525,
      "epoch": 0.792,
      "grad_norm": 0.000949859619140625,
      "kl": 0.056715600471943614,
      "learning_rate": 3.780107162176429e-06,
      "loss": 0.0,
      "match_ratio": 0.8,
      "reward": -0.2,
      "reward_std": 0.1,
      "rewards/reward_func": -0.2,
      "step": 3960
    },
    {
      "completion_length": 47.1,
      "epoch": 0.794,
      "grad_norm": 52.75,
      "kl": 0.48424787069670855,
      "learning_rate": 3.772603539375929e-06,
      "loss": 0.0,
      "match_ratio": 0.75,
      "reward": -0.25,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.25,
      "step": 3970
    },
    {
      "completion_length": 64.875,
      "epoch": 0.796,
      "grad_norm": 0.000530242919921875,
      "kl": 0.08614660077728331,
      "learning_rate": 3.7650844103029093e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 3980
    },
    {
      "completion_length": 64.725,
      "epoch": 0.798,
      "grad_norm": 0.000347137451171875,
      "kl": 0.012265483383089304,
      "learning_rate": 3.7575498665755884e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 3990
    },
    {
      "completion_length": 61.2,
      "epoch": 0.8,
      "grad_norm": 0.00040435791015625,
      "kl": 0.032297836942598225,
      "learning_rate": 3.7500000000000005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4000
    },
    {
      "completion_length": 65.8,
      "epoch": 0.802,
      "grad_norm": 0.00167083740234375,
      "kl": 0.016016237577423452,
      "learning_rate": 3.742434902568889e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4010
    },
    {
      "completion_length": 56.125,
      "epoch": 0.804,
      "grad_norm": 0.0004100799560546875,
      "kl": 0.06678674127906561,
      "learning_rate": 3.7348546664605777e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4020
    },
    {
      "completion_length": 52.825,
      "epoch": 0.806,
      "grad_norm": 13.9375,
      "kl": 39.51057905447669,
      "learning_rate": 3.7272593840378526e-06,
      "loss": 0.004,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.175,
      "step": 4030
    },
    {
      "completion_length": 56.375,
      "epoch": 0.808,
      "grad_norm": 0.000736236572265625,
      "kl": 76.7652599786874,
      "learning_rate": 3.7196491478468322e-06,
      "loss": 0.0077,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 4040
    },
    {
      "completion_length": 59.775,
      "epoch": 0.81,
      "grad_norm": 23.625,
      "kl": 2.1401951428037136,
      "learning_rate": 3.7120240506158433e-06,
      "loss": 0.0002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4050
    },
    {
      "completion_length": 78.925,
      "epoch": 0.812,
      "grad_norm": 0.000362396240234375,
      "kl": 0.019226322788745163,
      "learning_rate": 3.7043841852542884e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4060
    },
    {
      "completion_length": 58.425,
      "epoch": 0.814,
      "grad_norm": 0.00078582763671875,
      "kl": 0.04994579209014773,
      "learning_rate": 3.6967296448515176e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4070
    },
    {
      "completion_length": 57.025,
      "epoch": 0.816,
      "grad_norm": 0.000553131103515625,
      "kl": 0.11473355963826179,
      "learning_rate": 3.689060522675689e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4080
    },
    {
      "completion_length": 59.875,
      "epoch": 0.818,
      "grad_norm": 0.0164794921875,
      "kl": 0.05073905866593122,
      "learning_rate": 3.6813769121726356e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4090
    },
    {
      "completion_length": 44.25,
      "epoch": 0.82,
      "grad_norm": 0.0002727508544921875,
      "kl": 0.7498203465249389,
      "learning_rate": 3.6736789069647273e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 4100
    },
    {
      "completion_length": 58.15,
      "epoch": 0.822,
      "grad_norm": 0.0007781982421875,
      "kl": 0.022883613361045718,
      "learning_rate": 3.6659666008497287e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4110
    },
    {
      "completion_length": 64.35,
      "epoch": 0.824,
      "grad_norm": 0.0005035400390625,
      "kl": 3.837466208729893,
      "learning_rate": 3.658240087799655e-06,
      "loss": 0.0004,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4120
    },
    {
      "completion_length": 63.35,
      "epoch": 0.826,
      "grad_norm": 0.0002651214599609375,
      "kl": 0.03254580916836858,
      "learning_rate": 3.6504994619596295e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4130
    },
    {
      "completion_length": 45.65,
      "epoch": 0.828,
      "grad_norm": 0.000396728515625,
      "kl": 17.52819751542993,
      "learning_rate": 3.642744817646736e-06,
      "loss": 0.0018,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 4140
    },
    {
      "completion_length": 62.8,
      "epoch": 0.83,
      "grad_norm": 0.05712890625,
      "kl": 0.056678724475204945,
      "learning_rate": 3.634976249348867e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4150
    },
    {
      "completion_length": 49.45,
      "epoch": 0.832,
      "grad_norm": 0.0004863739013671875,
      "kl": 0.02377572702243924,
      "learning_rate": 3.627193851723577e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4160
    },
    {
      "completion_length": 56.3,
      "epoch": 0.834,
      "grad_norm": 0.0010528564453125,
      "kl": 0.23082902017049492,
      "learning_rate": 3.6193977195969243e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4170
    },
    {
      "completion_length": 46.35,
      "epoch": 0.836,
      "grad_norm": 0.000518798828125,
      "kl": 0.037431746069341895,
      "learning_rate": 3.611587947962319e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4180
    },
    {
      "completion_length": 66.15,
      "epoch": 0.838,
      "grad_norm": 0.00054168701171875,
      "kl": 0.014803345128893853,
      "learning_rate": 3.6037646319793635e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4190
    },
    {
      "completion_length": 54.525,
      "epoch": 0.84,
      "grad_norm": 0.000743865966796875,
      "kl": 0.03277415055781603,
      "learning_rate": 3.595927866972694e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4200
    },
    {
      "completion_length": 74.5,
      "epoch": 0.842,
      "grad_norm": 0.0004863739013671875,
      "kl": 0.014684983342885972,
      "learning_rate": 3.5880777484308193e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4210
    },
    {
      "completion_length": 63.125,
      "epoch": 0.844,
      "grad_norm": 0.000640869140625,
      "kl": 0.02462619331199676,
      "learning_rate": 3.5802143720049565e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4220
    },
    {
      "completion_length": 56.275,
      "epoch": 0.846,
      "grad_norm": 0.000415802001953125,
      "kl": 4.124728001933545,
      "learning_rate": 3.5723378335078653e-06,
      "loss": 0.0004,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 4230
    },
    {
      "completion_length": 53.25,
      "epoch": 0.848,
      "grad_norm": 0.000759124755859375,
      "kl": 0.024984571058303116,
      "learning_rate": 3.564448228912682e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4240
    },
    {
      "completion_length": 56.15,
      "epoch": 0.85,
      "grad_norm": 0.00075531005859375,
      "kl": 8.048016933631152,
      "learning_rate": 3.556545654351749e-06,
      "loss": 0.0008,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4250
    },
    {
      "completion_length": 46.275,
      "epoch": 0.852,
      "grad_norm": 0.00115203857421875,
      "kl": 0.02151933144778013,
      "learning_rate": 3.5486302061154433e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4260
    },
    {
      "completion_length": 56.525,
      "epoch": 0.854,
      "grad_norm": 0.000476837158203125,
      "kl": 0.042470036540180445,
      "learning_rate": 3.5407019806510035e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4270
    },
    {
      "completion_length": 73.7,
      "epoch": 0.856,
      "grad_norm": 0.00035858154296875,
      "kl": 0.01676445291377604,
      "learning_rate": 3.532761074561355e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4280
    },
    {
      "completion_length": 58.625,
      "epoch": 0.858,
      "grad_norm": 0.00136566162109375,
      "kl": 13.48712082421407,
      "learning_rate": 3.524807584603932e-06,
      "loss": 0.0013,
      "match_ratio": 0.825,
      "reward": -0.175,
      "reward_std": 0.05,
      "rewards/reward_func": -0.175,
      "step": 4290
    },
    {
      "completion_length": 72.7,
      "epoch": 0.86,
      "grad_norm": 0.001922607421875,
      "kl": 0.01346550565212965,
      "learning_rate": 3.516841607689501e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4300
    },
    {
      "completion_length": 48.225,
      "epoch": 0.862,
      "grad_norm": 0.000934600830078125,
      "kl": 1.3110887278337031,
      "learning_rate": 3.5088632408809757e-06,
      "loss": 0.0001,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4310
    },
    {
      "completion_length": 42.375,
      "epoch": 0.864,
      "grad_norm": 0.000583648681640625,
      "kl": 32.864392778254114,
      "learning_rate": 3.5008725813922383e-06,
      "loss": 0.0033,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 4320
    },
    {
      "completion_length": 54.675,
      "epoch": 0.866,
      "grad_norm": 0.00145721435546875,
      "kl": 0.43685728376731275,
      "learning_rate": 3.4928697265869516e-06,
      "loss": 0.0,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 4330
    },
    {
      "completion_length": 61.15,
      "epoch": 0.868,
      "grad_norm": 0.0004787445068359375,
      "kl": 0.02581656016409397,
      "learning_rate": 3.4848547739773782e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4340
    },
    {
      "completion_length": 52.5,
      "epoch": 0.87,
      "grad_norm": 0.0003643035888671875,
      "kl": 0.013556264666840434,
      "learning_rate": 3.476827821223184e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4350
    },
    {
      "completion_length": 56.025,
      "epoch": 0.872,
      "grad_norm": 0.00144195556640625,
      "kl": 0.05843255072832108,
      "learning_rate": 3.4687889661302577e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4360
    },
    {
      "completion_length": 68.225,
      "epoch": 0.874,
      "grad_norm": 0.000438690185546875,
      "kl": 0.025629992503672837,
      "learning_rate": 3.460738306649509e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4370
    },
    {
      "completion_length": 40.225,
      "epoch": 0.876,
      "grad_norm": 0.000537872314453125,
      "kl": 20.029893927741796,
      "learning_rate": 3.452675940875686e-06,
      "loss": 0.002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4380
    },
    {
      "completion_length": 53.5,
      "epoch": 0.878,
      "grad_norm": 0.000759124755859375,
      "kl": 0.0226501208730042,
      "learning_rate": 3.4446019670461684e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4390
    },
    {
      "completion_length": 55.325,
      "epoch": 0.88,
      "grad_norm": 0.00035858154296875,
      "kl": 1.9279290955979378,
      "learning_rate": 3.436516483539781e-06,
      "loss": 0.0002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4400
    },
    {
      "completion_length": 70.05,
      "epoch": 0.882,
      "grad_norm": 10.0,
      "kl": 0.05578553443774581,
      "learning_rate": 3.4284195888755877e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4410
    },
    {
      "completion_length": 58.05,
      "epoch": 0.884,
      "grad_norm": 0.00086212158203125,
      "kl": 0.019159636087715627,
      "learning_rate": 3.4203113817116955e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4420
    },
    {
      "completion_length": 62.425,
      "epoch": 0.886,
      "grad_norm": 0.000614166259765625,
      "kl": 0.10707788309082389,
      "learning_rate": 3.412191960844049e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4430
    },
    {
      "completion_length": 56.225,
      "epoch": 0.888,
      "grad_norm": 0.00061798095703125,
      "kl": 3.764384925994091,
      "learning_rate": 3.4040614252052305e-06,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4440
    },
    {
      "completion_length": 57.075,
      "epoch": 0.89,
      "grad_norm": 0.002838134765625,
      "kl": 0.04788713352754712,
      "learning_rate": 3.39591987386325e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4450
    },
    {
      "completion_length": 62.575,
      "epoch": 0.892,
      "grad_norm": 0.000339508056640625,
      "kl": 0.024599794298410416,
      "learning_rate": 3.387767406020343e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4460
    },
    {
      "completion_length": 47.85,
      "epoch": 0.894,
      "grad_norm": 0.000522613525390625,
      "kl": 419.8108845547773,
      "learning_rate": 3.3796041210117545e-06,
      "loss": 0.042,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4470
    },
    {
      "completion_length": 56.55,
      "epoch": 0.896,
      "grad_norm": 0.0016326904296875,
      "kl": 4.869295587006491,
      "learning_rate": 3.3714301183045382e-06,
      "loss": 0.0005,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 4480
    },
    {
      "completion_length": 65.65,
      "epoch": 0.898,
      "grad_norm": 0.000926971435546875,
      "kl": 0.021241254778578876,
      "learning_rate": 3.3632454974963368e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4490
    },
    {
      "completion_length": 51.7,
      "epoch": 0.9,
      "grad_norm": 0.00225830078125,
      "kl": 0.06338205388747156,
      "learning_rate": 3.3550503583141726e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4500
    },
    {
      "completion_length": 52.65,
      "epoch": 0.902,
      "grad_norm": 0.0003948211669921875,
      "kl": 0.3547412235289812,
      "learning_rate": 3.346844800613229e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 4510
    },
    {
      "completion_length": 66.475,
      "epoch": 0.904,
      "grad_norm": 0.000637054443359375,
      "kl": 0.01707718223333359,
      "learning_rate": 3.338628924375638e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4520
    },
    {
      "completion_length": 69.775,
      "epoch": 0.906,
      "grad_norm": 0.00017642974853515625,
      "kl": 4.814126300462521,
      "learning_rate": 3.3304028297092583e-06,
      "loss": 0.0005,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1154700517654419,
      "rewards/reward_func": -0.1,
      "step": 4530
    },
    {
      "completion_length": 72.7,
      "epoch": 0.908,
      "grad_norm": 0.00022411346435546875,
      "kl": 0.0319039260270074,
      "learning_rate": 3.3221666168464584e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4540
    },
    {
      "completion_length": 63.75,
      "epoch": 0.91,
      "grad_norm": 0.002227783203125,
      "kl": 0.02470703413709998,
      "learning_rate": 3.313920386142892e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4550
    },
    {
      "completion_length": 49.6,
      "epoch": 0.912,
      "grad_norm": 0.0012969970703125,
      "kl": 2.1085672612302004,
      "learning_rate": 3.3056642380762783e-06,
      "loss": 0.0002,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4560
    },
    {
      "completion_length": 70.425,
      "epoch": 0.914,
      "grad_norm": 0.0013885498046875,
      "kl": 0.016354763973504306,
      "learning_rate": 3.2973982732451753e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4570
    },
    {
      "completion_length": 59.775,
      "epoch": 0.916,
      "grad_norm": 0.000934600830078125,
      "kl": 0.035655501671135424,
      "learning_rate": 3.2891225923677565e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4580
    },
    {
      "completion_length": 54.775,
      "epoch": 0.918,
      "grad_norm": 0.01336669921875,
      "kl": 4.602624000795186,
      "learning_rate": 3.280837296280582e-06,
      "loss": 0.0005,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4590
    },
    {
      "completion_length": 46.175,
      "epoch": 0.92,
      "grad_norm": 0.0003147125244140625,
      "kl": 0.02292898967862129,
      "learning_rate": 3.272542485937369e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4600
    },
    {
      "completion_length": 48.85,
      "epoch": 0.922,
      "grad_norm": 0.000362396240234375,
      "kl": 0.08314138883724809,
      "learning_rate": 3.2642382624077647e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4610
    },
    {
      "completion_length": 49.15,
      "epoch": 0.924,
      "grad_norm": 0.0006103515625,
      "kl": 0.01725058164447546,
      "learning_rate": 3.2559247268761117e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4620
    },
    {
      "completion_length": 56.0,
      "epoch": 0.926,
      "grad_norm": 0.000514984130859375,
      "kl": 0.02017789352685213,
      "learning_rate": 3.247601980640217e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4630
    },
    {
      "completion_length": 48.975,
      "epoch": 0.928,
      "grad_norm": 0.00102996826171875,
      "kl": 0.015598981559742242,
      "learning_rate": 3.2392701251101172e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4640
    },
    {
      "completion_length": 54.35,
      "epoch": 0.93,
      "grad_norm": 0.00016307830810546875,
      "kl": 0.017281436000484974,
      "learning_rate": 3.230929261806842e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4650
    },
    {
      "completion_length": 62.25,
      "epoch": 0.932,
      "grad_norm": 0.00055694580078125,
      "kl": 27.82315392717719,
      "learning_rate": 3.222579492361179e-06,
      "loss": 0.0028,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4660
    },
    {
      "completion_length": 51.0,
      "epoch": 0.934,
      "grad_norm": 0.00083160400390625,
      "kl": 0.18145442437380552,
      "learning_rate": 3.214220918512434e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4670
    },
    {
      "completion_length": 53.025,
      "epoch": 0.936,
      "grad_norm": 0.000469207763671875,
      "kl": 0.26395926494151356,
      "learning_rate": 3.205853642107192e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4680
    },
    {
      "completion_length": 46.575,
      "epoch": 0.938,
      "grad_norm": 32.75,
      "kl": 50.37309080436826,
      "learning_rate": 3.1974777650980737e-06,
      "loss": 0.005,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 4690
    },
    {
      "completion_length": 54.75,
      "epoch": 0.94,
      "grad_norm": 0.00131988525390625,
      "kl": 1.135747592896223,
      "learning_rate": 3.189093389542498e-06,
      "loss": 0.0001,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 4700
    },
    {
      "completion_length": 45.35,
      "epoch": 0.942,
      "grad_norm": 0.000698089599609375,
      "kl": 8.475377059169114,
      "learning_rate": 3.180700617601436e-06,
      "loss": 0.0008,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4710
    },
    {
      "completion_length": 62.7,
      "epoch": 0.944,
      "grad_norm": 0.000827789306640625,
      "kl": 0.10563798192888499,
      "learning_rate": 3.1722995515381644e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4720
    },
    {
      "completion_length": 51.2,
      "epoch": 0.946,
      "grad_norm": 0.000637054443359375,
      "kl": 0.041765560209751126,
      "learning_rate": 3.1638902937170224e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4730
    },
    {
      "completion_length": 53.425,
      "epoch": 0.948,
      "grad_norm": 0.037841796875,
      "kl": 90.9074339528568,
      "learning_rate": 3.155472946602162e-06,
      "loss": 0.0091,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 4740
    },
    {
      "completion_length": 56.025,
      "epoch": 0.95,
      "grad_norm": 0.000637054443359375,
      "kl": 0.14690550537779928,
      "learning_rate": 3.147047612756302e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4750
    },
    {
      "completion_length": 59.45,
      "epoch": 0.952,
      "grad_norm": 0.00110626220703125,
      "kl": 0.021767212729901075,
      "learning_rate": 3.1386143948394764e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4760
    },
    {
      "completion_length": 49.15,
      "epoch": 0.954,
      "grad_norm": 68.0,
      "kl": 50.01492289174348,
      "learning_rate": 3.130173395607785e-06,
      "loss": 0.005,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 4770
    },
    {
      "completion_length": 50.925,
      "epoch": 0.956,
      "grad_norm": 0.000579833984375,
      "kl": 3.767396915424615,
      "learning_rate": 3.121724717912138e-06,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4780
    },
    {
      "completion_length": 54.85,
      "epoch": 0.958,
      "grad_norm": 0.00109100341796875,
      "kl": 0.024809733917936682,
      "learning_rate": 3.1132684646970068e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4790
    },
    {
      "completion_length": 60.0,
      "epoch": 0.96,
      "grad_norm": 0.000492095947265625,
      "kl": 22.365475433226674,
      "learning_rate": 3.1048047389991693e-06,
      "loss": 0.0022,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.1,
      "step": 4800
    },
    {
      "completion_length": 48.5,
      "epoch": 0.962,
      "grad_norm": 0.00107574462890625,
      "kl": 0.01949691798072308,
      "learning_rate": 3.0963336439464527e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4810
    },
    {
      "completion_length": 73.975,
      "epoch": 0.964,
      "grad_norm": 0.0017242431640625,
      "kl": 0.0853988635353744,
      "learning_rate": 3.087855282756475e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4820
    },
    {
      "completion_length": 61.175,
      "epoch": 0.966,
      "grad_norm": 0.00046539306640625,
      "kl": 0.05258291512727738,
      "learning_rate": 3.079369758735393e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4830
    },
    {
      "completion_length": 69.55,
      "epoch": 0.968,
      "grad_norm": 0.08447265625,
      "kl": 0.19851951650343835,
      "learning_rate": 3.0708771752766397e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4840
    },
    {
      "completion_length": 52.35,
      "epoch": 0.97,
      "grad_norm": 0.00139617919921875,
      "kl": 0.016163587383925915,
      "learning_rate": 3.062377635859663e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4850
    },
    {
      "completion_length": 59.4,
      "epoch": 0.972,
      "grad_norm": 0.000415802001953125,
      "kl": 0.07268630117177963,
      "learning_rate": 3.053871244048669e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4860
    },
    {
      "completion_length": 52.725,
      "epoch": 0.974,
      "grad_norm": 0.000751495361328125,
      "kl": 0.08838214613497257,
      "learning_rate": 3.045358103491357e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4870
    },
    {
      "completion_length": 47.25,
      "epoch": 0.976,
      "grad_norm": 0.0015411376953125,
      "kl": 0.8490013023838401,
      "learning_rate": 3.0368383179176584e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4880
    },
    {
      "completion_length": 51.9,
      "epoch": 0.978,
      "grad_norm": 0.000576019287109375,
      "kl": 0.06923787947744131,
      "learning_rate": 3.0283119911384724e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4890
    },
    {
      "completion_length": 45.05,
      "epoch": 0.98,
      "grad_norm": 0.000823974609375,
      "kl": 0.06741849109530448,
      "learning_rate": 3.019779227044398e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4900
    },
    {
      "completion_length": 59.025,
      "epoch": 0.982,
      "grad_norm": 0.000885009765625,
      "kl": 0.20571241448633373,
      "learning_rate": 3.0112401296044756e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4910
    },
    {
      "completion_length": 58.85,
      "epoch": 0.984,
      "grad_norm": 0.0003299713134765625,
      "kl": 0.0659916253760457,
      "learning_rate": 3.002694802864912e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4920
    },
    {
      "completion_length": 57.3,
      "epoch": 0.986,
      "grad_norm": 0.0010223388671875,
      "kl": 0.9824612125754356,
      "learning_rate": 2.9941433509478157e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 4930
    },
    {
      "completion_length": 63.675,
      "epoch": 0.988,
      "grad_norm": 0.0008544921875,
      "kl": 0.08037902340292931,
      "learning_rate": 2.98558587804993e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4940
    },
    {
      "completion_length": 61.775,
      "epoch": 0.99,
      "grad_norm": 0.0003757476806640625,
      "kl": 0.05214073383249342,
      "learning_rate": 2.9770224884413625e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4950
    },
    {
      "completion_length": 58.125,
      "epoch": 0.992,
      "grad_norm": 0.003936767578125,
      "kl": 0.307067746296525,
      "learning_rate": 2.9684532864643123e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4960
    },
    {
      "completion_length": 47.575,
      "epoch": 0.994,
      "grad_norm": 0.00148773193359375,
      "kl": 0.4293937426991761,
      "learning_rate": 2.9598783765318005e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4970
    },
    {
      "completion_length": 61.5,
      "epoch": 0.996,
      "grad_norm": 0.000698089599609375,
      "kl": 0.055739361047744754,
      "learning_rate": 2.9512978631264006e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 4980
    },
    {
      "completion_length": 61.25,
      "epoch": 0.998,
      "grad_norm": 0.0030517578125,
      "kl": 0.31474687876179813,
      "learning_rate": 2.942711850798959e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 4990
    },
    {
      "completion_length": 55.975,
      "epoch": 1.0,
      "grad_norm": 0.000606536865234375,
      "kl": 10.358118780329823,
      "learning_rate": 2.9341204441673267e-06,
      "loss": 0.001,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 5000
    },
    {
      "completion_length": 67.125,
      "epoch": 1.002,
      "grad_norm": 0.0023193359375,
      "kl": 0.02981336957309395,
      "learning_rate": 2.9255237479150815e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5010
    },
    {
      "completion_length": 46.975,
      "epoch": 1.004,
      "grad_norm": 0.0004596710205078125,
      "kl": 0.18872954780235887,
      "learning_rate": 2.9169218667902562e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5020
    },
    {
      "completion_length": 81.2,
      "epoch": 1.006,
      "grad_norm": 0.0005950927734375,
      "kl": 0.019520534854382276,
      "learning_rate": 2.908314905604056e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5030
    },
    {
      "completion_length": 84.75,
      "epoch": 1.008,
      "grad_norm": 11.1875,
      "kl": 0.21086107967421414,
      "learning_rate": 2.8997029692295875e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5040
    },
    {
      "completion_length": 67.175,
      "epoch": 1.01,
      "grad_norm": 0.000514984130859375,
      "kl": 0.026521979738026856,
      "learning_rate": 2.8910861626005774e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5050
    },
    {
      "completion_length": 49.025,
      "epoch": 1.012,
      "grad_norm": 0.00106048583984375,
      "kl": 0.05570605006068945,
      "learning_rate": 2.8824645907100957e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5060
    },
    {
      "completion_length": 57.875,
      "epoch": 1.014,
      "grad_norm": 0.000522613525390625,
      "kl": 0.02027883781120181,
      "learning_rate": 2.8738383586092745e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5070
    },
    {
      "completion_length": 64.325,
      "epoch": 1.016,
      "grad_norm": 0.0004520416259765625,
      "kl": 0.35661591766402123,
      "learning_rate": 2.8652075714060296e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5080
    },
    {
      "completion_length": 47.125,
      "epoch": 1.018,
      "grad_norm": 0.0033111572265625,
      "kl": 0.16099169924855233,
      "learning_rate": 2.8565723342637797e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5090
    },
    {
      "completion_length": 58.325,
      "epoch": 1.02,
      "grad_norm": 0.00066375732421875,
      "kl": 36.65545420385897,
      "learning_rate": 2.847932752400164e-06,
      "loss": 0.0037,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5100
    },
    {
      "completion_length": 54.975,
      "epoch": 1.022,
      "grad_norm": 0.0054931640625,
      "kl": 0.14035283839330076,
      "learning_rate": 2.8392889310857615e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5110
    },
    {
      "completion_length": 44.15,
      "epoch": 1.024,
      "grad_norm": 0.0006103515625,
      "kl": 0.05474662664346397,
      "learning_rate": 2.8306409756428067e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5120
    },
    {
      "completion_length": 51.425,
      "epoch": 1.026,
      "grad_norm": 0.00077056884765625,
      "kl": 0.9017472909763455,
      "learning_rate": 2.8219889914439073e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5130
    },
    {
      "completion_length": 56.425,
      "epoch": 1.028,
      "grad_norm": 0.0030059814453125,
      "kl": 920.3957623304799,
      "learning_rate": 2.813333083910761e-06,
      "loss": 0.092,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 5140
    },
    {
      "completion_length": 61.8,
      "epoch": 1.03,
      "grad_norm": 0.002166748046875,
      "kl": 0.06634964090771973,
      "learning_rate": 2.804673358512869e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5150
    },
    {
      "completion_length": 60.6,
      "epoch": 1.032,
      "grad_norm": 0.0009613037109375,
      "kl": 0.10043707201257349,
      "learning_rate": 2.7960099207662535e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5160
    },
    {
      "completion_length": 60.625,
      "epoch": 1.034,
      "grad_norm": 0.00049591064453125,
      "kl": 2957.9408455969765,
      "learning_rate": 2.7873428762321667e-06,
      "loss": 0.2958,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5170
    },
    {
      "completion_length": 48.175,
      "epoch": 1.036,
      "grad_norm": 0.000431060791015625,
      "kl": 0.44579824099782855,
      "learning_rate": 2.778672330515814e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 5180
    },
    {
      "completion_length": 69.6,
      "epoch": 1.038,
      "grad_norm": 0.000530242919921875,
      "kl": 0.1467185489833355,
      "learning_rate": 2.769998389265057e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5190
    },
    {
      "completion_length": 46.4,
      "epoch": 1.04,
      "grad_norm": 0.0022735595703125,
      "kl": 0.1808565909974277,
      "learning_rate": 2.761321158169134e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5200
    },
    {
      "completion_length": 66.075,
      "epoch": 1.042,
      "grad_norm": 0.00151824951171875,
      "kl": 0.10877533163875341,
      "learning_rate": 2.752640742957366e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5210
    },
    {
      "completion_length": 64.8,
      "epoch": 1.044,
      "grad_norm": 0.000827789306640625,
      "kl": 36.803903768444435,
      "learning_rate": 2.743957249397874e-06,
      "loss": 0.0037,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5220
    },
    {
      "completion_length": 61.525,
      "epoch": 1.046,
      "grad_norm": 0.0040283203125,
      "kl": 0.021204144693911076,
      "learning_rate": 2.7352707832962865e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5230
    },
    {
      "completion_length": 45.65,
      "epoch": 1.048,
      "grad_norm": 0.003173828125,
      "kl": 2.3442719845101236,
      "learning_rate": 2.726581450494451e-06,
      "loss": 0.0002,
      "match_ratio": 0.875,
      "reward": -0.125,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.125,
      "step": 5240
    },
    {
      "completion_length": 54.725,
      "epoch": 1.05,
      "grad_norm": 0.0003566741943359375,
      "kl": 0.08242949154227971,
      "learning_rate": 2.717889356869146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5250
    },
    {
      "completion_length": 68.075,
      "epoch": 1.052,
      "grad_norm": 0.0003337860107421875,
      "kl": 0.0425911046564579,
      "learning_rate": 2.70919460833079e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5260
    },
    {
      "completion_length": 58.25,
      "epoch": 1.054,
      "grad_norm": 0.00067901611328125,
      "kl": 0.05532512974459678,
      "learning_rate": 2.700497310822147e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5270
    },
    {
      "completion_length": 67.3,
      "epoch": 1.056,
      "grad_norm": 1616.0,
      "kl": 654.5019911365816,
      "learning_rate": 2.6917975703170466e-06,
      "loss": 0.0655,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5280
    },
    {
      "completion_length": 50.725,
      "epoch": 1.058,
      "grad_norm": 0.0004863739013671875,
      "kl": 314.1969824824482,
      "learning_rate": 2.6830954928190795e-06,
      "loss": 0.0314,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 5290
    },
    {
      "completion_length": 52.65,
      "epoch": 1.06,
      "grad_norm": 0.000690460205078125,
      "kl": 3050.6620800592004,
      "learning_rate": 2.6743911843603134e-06,
      "loss": 0.3051,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1,
      "rewards/reward_func": -0.1,
      "step": 5300
    },
    {
      "completion_length": 45.2,
      "epoch": 1.062,
      "grad_norm": 0.000606536865234375,
      "kl": 0.6229335282929241,
      "learning_rate": 2.6656847510000013e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5310
    },
    {
      "completion_length": 59.825,
      "epoch": 1.064,
      "grad_norm": 0.00023555755615234375,
      "kl": 0.0229927783831954,
      "learning_rate": 2.6569762988232838e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5320
    },
    {
      "completion_length": 45.525,
      "epoch": 1.066,
      "grad_norm": 0.470703125,
      "kl": 12.478434246452526,
      "learning_rate": 2.6482659339399047e-06,
      "loss": 0.0012,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 5330
    },
    {
      "completion_length": 48.5,
      "epoch": 1.068,
      "grad_norm": 0.000637054443359375,
      "kl": 0.04660536227747798,
      "learning_rate": 2.63955376248291e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5340
    },
    {
      "completion_length": 57.5,
      "epoch": 1.07,
      "grad_norm": 0.00103759765625,
      "kl": 0.46991982199251653,
      "learning_rate": 2.6308398906073603e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5350
    },
    {
      "completion_length": 59.9,
      "epoch": 1.072,
      "grad_norm": 0.00051116943359375,
      "kl": 0.05683571686968207,
      "learning_rate": 2.6221244244890336e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5360
    },
    {
      "completion_length": 74.275,
      "epoch": 1.074,
      "grad_norm": 0.00079345703125,
      "kl": 0.044226788356900214,
      "learning_rate": 2.613407470323134e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5370
    },
    {
      "completion_length": 71.075,
      "epoch": 1.076,
      "grad_norm": 0.000576019287109375,
      "kl": 0.014436176512390375,
      "learning_rate": 2.604689134322999e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5380
    },
    {
      "completion_length": 54.925,
      "epoch": 1.078,
      "grad_norm": 0.00112152099609375,
      "kl": 0.06992563903331757,
      "learning_rate": 2.5959695227188e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5390
    },
    {
      "completion_length": 58.65,
      "epoch": 1.08,
      "grad_norm": 0.00103759765625,
      "kl": 0.026312044728547333,
      "learning_rate": 2.587248741756253e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5400
    },
    {
      "completion_length": 61.625,
      "epoch": 1.082,
      "grad_norm": 38.25,
      "kl": 96.55521301142872,
      "learning_rate": 2.578526897695321e-06,
      "loss": 0.0097,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.15,
      "rewards/reward_func": -0.075,
      "step": 5410
    },
    {
      "completion_length": 74.625,
      "epoch": 1.084,
      "grad_norm": 0.0004444122314453125,
      "kl": 0.027571643888950347,
      "learning_rate": 2.569804096808923e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5420
    },
    {
      "completion_length": 63.75,
      "epoch": 1.086,
      "grad_norm": 0.000728607177734375,
      "kl": 0.06449384274892508,
      "learning_rate": 2.5610804453816333e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5430
    },
    {
      "completion_length": 61.75,
      "epoch": 1.088,
      "grad_norm": 0.001007080078125,
      "kl": 0.014204623247496783,
      "learning_rate": 2.5523560497083927e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5440
    },
    {
      "completion_length": 48.175,
      "epoch": 1.09,
      "grad_norm": 41.75,
      "kl": 4.48764673435362,
      "learning_rate": 2.543631016093209e-06,
      "loss": 0.0004,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 5450
    },
    {
      "completion_length": 53.15,
      "epoch": 1.092,
      "grad_norm": 0.00054931640625,
      "kl": 0.04185728752054274,
      "learning_rate": 2.5349054508478636e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5460
    },
    {
      "completion_length": 58.475,
      "epoch": 1.094,
      "grad_norm": 0.000457763671875,
      "kl": 0.058739370107650755,
      "learning_rate": 2.526179460290615e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5470
    },
    {
      "completion_length": 54.5,
      "epoch": 1.096,
      "grad_norm": 0.025634765625,
      "kl": 0.8717142393812537,
      "learning_rate": 2.517453150744904e-06,
      "loss": 0.0001,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5480
    },
    {
      "completion_length": 65.1,
      "epoch": 1.098,
      "grad_norm": 0.000553131103515625,
      "kl": 0.0642022612504661,
      "learning_rate": 2.5087266285380597e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5490
    },
    {
      "completion_length": 55.0,
      "epoch": 1.1,
      "grad_norm": 0.000644683837890625,
      "kl": 0.037831029202789065,
      "learning_rate": 2.5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5500
    },
    {
      "completion_length": 56.85,
      "epoch": 1.102,
      "grad_norm": 0.001007080078125,
      "kl": 0.1622185967862606,
      "learning_rate": 2.4912733714619415e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5510
    },
    {
      "completion_length": 72.775,
      "epoch": 1.104,
      "grad_norm": 0.00127410888671875,
      "kl": 0.014623588742688298,
      "learning_rate": 2.482546849255096e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5520
    },
    {
      "completion_length": 53.525,
      "epoch": 1.106,
      "grad_norm": 0.000782012939453125,
      "kl": 0.21322614937089385,
      "learning_rate": 2.4738205397093863e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5530
    },
    {
      "completion_length": 48.825,
      "epoch": 1.108,
      "grad_norm": 32.0,
      "kl": 3.262874563597143,
      "learning_rate": 2.4650945491521372e-06,
      "loss": 0.0003,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5540
    },
    {
      "completion_length": 58.7,
      "epoch": 1.11,
      "grad_norm": 0.0012664794921875,
      "kl": 0.062091145850718024,
      "learning_rate": 2.4563689839067913e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5550
    },
    {
      "completion_length": 45.5,
      "epoch": 1.112,
      "grad_norm": 0.00070953369140625,
      "kl": 58.79087800290436,
      "learning_rate": 2.447643950291608e-06,
      "loss": 0.0059,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5560
    },
    {
      "completion_length": 60.625,
      "epoch": 1.114,
      "grad_norm": 0.0003070831298828125,
      "kl": 11.288955740490929,
      "learning_rate": 2.4389195546183676e-06,
      "loss": 0.0011,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5570
    },
    {
      "completion_length": 57.375,
      "epoch": 1.116,
      "grad_norm": 0.004669189453125,
      "kl": 0.05885868603363633,
      "learning_rate": 2.4301959031910785e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5580
    },
    {
      "completion_length": 64.825,
      "epoch": 1.1179999999999999,
      "grad_norm": 0.00072479248046875,
      "kl": 0.04936583343660459,
      "learning_rate": 2.4214731023046795e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5590
    },
    {
      "completion_length": 60.0,
      "epoch": 1.12,
      "grad_norm": 0.0009002685546875,
      "kl": 16.106818246748297,
      "learning_rate": 2.4127512582437486e-06,
      "loss": 0.0016,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5600
    },
    {
      "completion_length": 60.425,
      "epoch": 1.1219999999999999,
      "grad_norm": 0.0023193359375,
      "kl": 0.05024177338927984,
      "learning_rate": 2.4040304772812002e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5610
    },
    {
      "completion_length": 54.6,
      "epoch": 1.124,
      "grad_norm": 0.0004558563232421875,
      "kl": 0.3465561534278095,
      "learning_rate": 2.3953108656770018e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5620
    },
    {
      "completion_length": 61.025,
      "epoch": 1.126,
      "grad_norm": 0.00311279296875,
      "kl": 0.019688890036195516,
      "learning_rate": 2.3865925296768658e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5630
    },
    {
      "completion_length": 50.0,
      "epoch": 1.1280000000000001,
      "grad_norm": 0.00066375732421875,
      "kl": 0.38241584403440354,
      "learning_rate": 2.377875575510967e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5640
    },
    {
      "completion_length": 58.7,
      "epoch": 1.13,
      "grad_norm": 0.00112152099609375,
      "kl": 0.11847766758874059,
      "learning_rate": 2.3691601093926406e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5650
    },
    {
      "completion_length": 61.125,
      "epoch": 1.1320000000000001,
      "grad_norm": 0.0023956298828125,
      "kl": 1.8248440870083869,
      "learning_rate": 2.3604462375170905e-06,
      "loss": 0.0002,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 5660
    },
    {
      "completion_length": 57.175,
      "epoch": 1.134,
      "grad_norm": 22.75,
      "kl": 16.894031352642923,
      "learning_rate": 2.3517340660600965e-06,
      "loss": 0.0017,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5670
    },
    {
      "completion_length": 66.275,
      "epoch": 1.1360000000000001,
      "grad_norm": 0.00035858154296875,
      "kl": 0.01967704053968191,
      "learning_rate": 2.3430237011767166e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5680
    },
    {
      "completion_length": 51.75,
      "epoch": 1.138,
      "grad_norm": 69.0,
      "kl": 15.417314376076684,
      "learning_rate": 2.3343152490000004e-06,
      "loss": 0.0015,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5690
    },
    {
      "completion_length": 43.4,
      "epoch": 1.1400000000000001,
      "grad_norm": 0.000396728515625,
      "kl": 9.579606763273478,
      "learning_rate": 2.325608815639687e-06,
      "loss": 0.001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5700
    },
    {
      "completion_length": 63.475,
      "epoch": 1.142,
      "grad_norm": 0.000457763671875,
      "kl": 0.09258651239797473,
      "learning_rate": 2.3169045071809217e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 5710
    },
    {
      "completion_length": 67.575,
      "epoch": 1.144,
      "grad_norm": 0.0004215240478515625,
      "kl": 36.50597060709261,
      "learning_rate": 2.3082024296829538e-06,
      "loss": 0.0037,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5720
    },
    {
      "completion_length": 57.8,
      "epoch": 1.146,
      "grad_norm": 0.0002994537353515625,
      "kl": 0.08120424915105104,
      "learning_rate": 2.2995026891778533e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5730
    },
    {
      "completion_length": 55.275,
      "epoch": 1.148,
      "grad_norm": 0.0027618408203125,
      "kl": 0.9556269285269081,
      "learning_rate": 2.290805391669212e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5740
    },
    {
      "completion_length": 67.5,
      "epoch": 1.15,
      "grad_norm": 0.03662109375,
      "kl": 4.989278326183557,
      "learning_rate": 2.2821106431308546e-06,
      "loss": 0.0005,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5750
    },
    {
      "completion_length": 57.625,
      "epoch": 1.152,
      "grad_norm": 0.0004482269287109375,
      "kl": 0.05195563132874668,
      "learning_rate": 2.2734185495055503e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5760
    },
    {
      "completion_length": 66.875,
      "epoch": 1.154,
      "grad_norm": 0.000614166259765625,
      "kl": 0.04006156194955111,
      "learning_rate": 2.2647292167037143e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5770
    },
    {
      "completion_length": 67.85,
      "epoch": 1.156,
      "grad_norm": 0.000518798828125,
      "kl": 0.017254956741817297,
      "learning_rate": 2.256042750602127e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5780
    },
    {
      "completion_length": 57.9,
      "epoch": 1.158,
      "grad_norm": 0.016357421875,
      "kl": 22.463907711207867,
      "learning_rate": 2.2473592570426343e-06,
      "loss": 0.0022,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5790
    },
    {
      "completion_length": 53.4,
      "epoch": 1.16,
      "grad_norm": 14.9375,
      "kl": 0.47836247340310367,
      "learning_rate": 2.238678841830867e-06,
      "loss": 0.0,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1154700517654419,
      "rewards/reward_func": -0.1,
      "step": 5800
    },
    {
      "completion_length": 52.575,
      "epoch": 1.162,
      "grad_norm": 0.0005340576171875,
      "kl": 1.7170299529330806,
      "learning_rate": 2.230001610734943e-06,
      "loss": 0.0002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5810
    },
    {
      "completion_length": 52.175,
      "epoch": 1.164,
      "grad_norm": 0.000820159912109375,
      "kl": 0.023483294621109964,
      "learning_rate": 2.2213276694841866e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5820
    },
    {
      "completion_length": 66.725,
      "epoch": 1.166,
      "grad_norm": 0.000797271728515625,
      "kl": 0.017644689697772265,
      "learning_rate": 2.212657123767834e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5830
    },
    {
      "completion_length": 46.525,
      "epoch": 1.168,
      "grad_norm": 0.000629425048828125,
      "kl": 0.022305818554013968,
      "learning_rate": 2.2039900792337477e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5840
    },
    {
      "completion_length": 79.55,
      "epoch": 1.17,
      "grad_norm": 0.000514984130859375,
      "kl": 0.045465368404984476,
      "learning_rate": 2.195326641487132e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5850
    },
    {
      "completion_length": 70.875,
      "epoch": 1.172,
      "grad_norm": 0.000728607177734375,
      "kl": 0.026964151486754417,
      "learning_rate": 2.186666916089239e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5860
    },
    {
      "completion_length": 51.9,
      "epoch": 1.174,
      "grad_norm": 0.0006103515625,
      "kl": 0.06193929803557694,
      "learning_rate": 2.1780110085560935e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5870
    },
    {
      "completion_length": 58.975,
      "epoch": 1.176,
      "grad_norm": 0.0003662109375,
      "kl": 96.22263815930928,
      "learning_rate": 2.1693590243571937e-06,
      "loss": 0.0096,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5880
    },
    {
      "completion_length": 50.05,
      "epoch": 1.178,
      "grad_norm": 0.0012054443359375,
      "kl": 0.10491749201901257,
      "learning_rate": 2.1607110689142393e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5890
    },
    {
      "completion_length": 49.3,
      "epoch": 1.18,
      "grad_norm": 0.00064849853515625,
      "kl": 0.09745957013219594,
      "learning_rate": 2.1520672475998374e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5900
    },
    {
      "completion_length": 63.625,
      "epoch": 1.182,
      "grad_norm": 0.00054931640625,
      "kl": 0.0679678438231349,
      "learning_rate": 2.143427665736221e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5910
    },
    {
      "completion_length": 63.7,
      "epoch": 1.184,
      "grad_norm": 0.0005950927734375,
      "kl": 0.03987161219120026,
      "learning_rate": 2.134792428593971e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5920
    },
    {
      "completion_length": 53.3,
      "epoch": 1.186,
      "grad_norm": 0.000591278076171875,
      "kl": 3.143317204480991,
      "learning_rate": 2.1261616413907267e-06,
      "loss": 0.0003,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 5930
    },
    {
      "completion_length": 44.425,
      "epoch": 1.188,
      "grad_norm": 0.01080322265625,
      "kl": 0.0820039251120761,
      "learning_rate": 2.117535409289905e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5940
    },
    {
      "completion_length": 46.825,
      "epoch": 1.19,
      "grad_norm": 0.000396728515625,
      "kl": 0.10649018711410463,
      "learning_rate": 2.1089138373994226e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5950
    },
    {
      "completion_length": 47.475,
      "epoch": 1.192,
      "grad_norm": 0.000499725341796875,
      "kl": 0.01360652674920857,
      "learning_rate": 2.1002970307704134e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5960
    },
    {
      "completion_length": 63.525,
      "epoch": 1.194,
      "grad_norm": 0.00060272216796875,
      "kl": 0.015495409537106753,
      "learning_rate": 2.0916850943959453e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5970
    },
    {
      "completion_length": 47.5,
      "epoch": 1.196,
      "grad_norm": 0.0026702880859375,
      "kl": 0.058427824173122644,
      "learning_rate": 2.0830781332097446e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5980
    },
    {
      "completion_length": 59.525,
      "epoch": 1.198,
      "grad_norm": 0.00106048583984375,
      "kl": 0.07809406300075353,
      "learning_rate": 2.0744762520849193e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 5990
    },
    {
      "completion_length": 68.25,
      "epoch": 1.2,
      "grad_norm": 0.000324249267578125,
      "kl": 5.223696762509644,
      "learning_rate": 2.0658795558326745e-06,
      "loss": 0.0005,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6000
    },
    {
      "completion_length": 52.325,
      "epoch": 1.202,
      "grad_norm": 0.000720977783203125,
      "kl": 0.08351310016587377,
      "learning_rate": 2.0572881492010423e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6010
    },
    {
      "completion_length": 65.15,
      "epoch": 1.204,
      "grad_norm": 0.000499725341796875,
      "kl": 668.4266535042319,
      "learning_rate": 2.0487021368736002e-06,
      "loss": 0.0668,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6020
    },
    {
      "completion_length": 56.75,
      "epoch": 1.206,
      "grad_norm": 0.0004119873046875,
      "kl": 10.605211506178602,
      "learning_rate": 2.0401216234682e-06,
      "loss": 0.0011,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6030
    },
    {
      "completion_length": 69.575,
      "epoch": 1.208,
      "grad_norm": 0.00077056884765625,
      "kl": 0.01607757806777954,
      "learning_rate": 2.031546713535688e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6040
    },
    {
      "completion_length": 66.85,
      "epoch": 1.21,
      "grad_norm": 0.00070953369140625,
      "kl": 0.05791890555992722,
      "learning_rate": 2.022977511558638e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6050
    },
    {
      "completion_length": 55.3,
      "epoch": 1.212,
      "grad_norm": 0.000774383544921875,
      "kl": 301.9731966109015,
      "learning_rate": 2.0144141219500707e-06,
      "loss": 0.0302,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6060
    },
    {
      "completion_length": 53.25,
      "epoch": 1.214,
      "grad_norm": 0.0079345703125,
      "kl": 0.03139863689430058,
      "learning_rate": 2.0058566490521848e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6070
    },
    {
      "completion_length": 61.25,
      "epoch": 1.216,
      "grad_norm": 0.0005340576171875,
      "kl": 0.04796885896939784,
      "learning_rate": 1.997305197135089e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6080
    },
    {
      "completion_length": 47.325,
      "epoch": 1.218,
      "grad_norm": 0.002166748046875,
      "kl": 0.2320690915454179,
      "learning_rate": 1.9887598703955244e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6090
    },
    {
      "completion_length": 69.575,
      "epoch": 1.22,
      "grad_norm": 0.000339508056640625,
      "kl": 0.02641369737684727,
      "learning_rate": 1.9802207729556023e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6100
    },
    {
      "completion_length": 55.0,
      "epoch": 1.222,
      "grad_norm": 0.0003509521484375,
      "kl": 0.07225975301116705,
      "learning_rate": 1.971688008861529e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6110
    },
    {
      "completion_length": 60.75,
      "epoch": 1.224,
      "grad_norm": 0.00022411346435546875,
      "kl": 0.015332509903237224,
      "learning_rate": 1.963161682082342e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6120
    },
    {
      "completion_length": 48.175,
      "epoch": 1.226,
      "grad_norm": 0.002105712890625,
      "kl": 0.05186178609728813,
      "learning_rate": 1.9546418965086444e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6130
    },
    {
      "completion_length": 55.825,
      "epoch": 1.228,
      "grad_norm": 0.00109100341796875,
      "kl": 0.023852512496523558,
      "learning_rate": 1.946128755951332e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6140
    },
    {
      "completion_length": 60.925,
      "epoch": 1.23,
      "grad_norm": 0.0003604888916015625,
      "kl": 0.1579098215326667,
      "learning_rate": 1.937622364140338e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6150
    },
    {
      "completion_length": 54.3,
      "epoch": 1.232,
      "grad_norm": 0.0018463134765625,
      "kl": 0.07494590748101473,
      "learning_rate": 1.9291228247233607e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6160
    },
    {
      "completion_length": 49.825,
      "epoch": 1.234,
      "grad_norm": 0.0006866455078125,
      "kl": 0.019502221944276244,
      "learning_rate": 1.9206302412646074e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6170
    },
    {
      "completion_length": 55.625,
      "epoch": 1.236,
      "grad_norm": 0.0003032684326171875,
      "kl": 0.017997803702019154,
      "learning_rate": 1.912144717243525e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6180
    },
    {
      "completion_length": 70.4,
      "epoch": 1.238,
      "grad_norm": 0.0008697509765625,
      "kl": 0.13488098671659826,
      "learning_rate": 1.9036663560535484e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6190
    },
    {
      "completion_length": 42.95,
      "epoch": 1.24,
      "grad_norm": 0.0008392333984375,
      "kl": 0.05781206511892378,
      "learning_rate": 1.895195261000831e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6200
    },
    {
      "completion_length": 55.925,
      "epoch": 1.242,
      "grad_norm": 0.000698089599609375,
      "kl": 0.018764377292245626,
      "learning_rate": 1.8867315353029937e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6210
    },
    {
      "completion_length": 60.3,
      "epoch": 1.244,
      "grad_norm": 0.000396728515625,
      "kl": 0.061903743352741,
      "learning_rate": 1.8782752820878636e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6220
    },
    {
      "completion_length": 60.525,
      "epoch": 1.246,
      "grad_norm": 0.000698089599609375,
      "kl": 0.06673049959354102,
      "learning_rate": 1.8698266043922159e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6230
    },
    {
      "completion_length": 53.775,
      "epoch": 1.248,
      "grad_norm": 0.0004520416259765625,
      "kl": 0.05146434986963868,
      "learning_rate": 1.8613856051605242e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6240
    },
    {
      "completion_length": 57.675,
      "epoch": 1.25,
      "grad_norm": 0.000431060791015625,
      "kl": 0.05393651574850082,
      "learning_rate": 1.852952387243698e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6250
    },
    {
      "completion_length": 44.85,
      "epoch": 1.252,
      "grad_norm": 0.000461578369140625,
      "kl": 0.13971609035506843,
      "learning_rate": 1.8445270533978387e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6260
    },
    {
      "completion_length": 50.125,
      "epoch": 1.254,
      "grad_norm": 0.000614166259765625,
      "kl": 0.050323914270848036,
      "learning_rate": 1.836109706282978e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6270
    },
    {
      "completion_length": 50.5,
      "epoch": 1.256,
      "grad_norm": 0.00054931640625,
      "kl": 0.13358841557055712,
      "learning_rate": 1.827700448461836e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6280
    },
    {
      "completion_length": 58.675,
      "epoch": 1.258,
      "grad_norm": 0.000518798828125,
      "kl": 0.4532184978015721,
      "learning_rate": 1.8192993823985643e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6290
    },
    {
      "completion_length": 61.5,
      "epoch": 1.26,
      "grad_norm": 0.000621795654296875,
      "kl": 2.259404849074781,
      "learning_rate": 1.8109066104575023e-06,
      "loss": 0.0002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6300
    },
    {
      "completion_length": 47.8,
      "epoch": 1.262,
      "grad_norm": 0.00080108642578125,
      "kl": 0.110232665669173,
      "learning_rate": 1.8025222349019273e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6310
    },
    {
      "completion_length": 46.3,
      "epoch": 1.264,
      "grad_norm": 0.000720977783203125,
      "kl": 0.0984095955034718,
      "learning_rate": 1.7941463578928088e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6320
    },
    {
      "completion_length": 57.525,
      "epoch": 1.266,
      "grad_norm": 17.375,
      "kl": 53.167960462137124,
      "learning_rate": 1.7857790814875665e-06,
      "loss": 0.0053,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 6330
    },
    {
      "completion_length": 56.15,
      "epoch": 1.268,
      "grad_norm": 0.002105712890625,
      "kl": 0.09631253816187382,
      "learning_rate": 1.7774205076388207e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6340
    },
    {
      "completion_length": 50.2,
      "epoch": 1.27,
      "grad_norm": 0.014404296875,
      "kl": 0.185893784603104,
      "learning_rate": 1.7690707381931585e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6350
    },
    {
      "completion_length": 57.775,
      "epoch": 1.272,
      "grad_norm": 0.0006256103515625,
      "kl": 0.032014391385018826,
      "learning_rate": 1.7607298748898844e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6360
    },
    {
      "completion_length": 72.225,
      "epoch": 1.274,
      "grad_norm": 0.0087890625,
      "kl": 0.053022891748696566,
      "learning_rate": 1.7523980193597837e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6370
    },
    {
      "completion_length": 61.275,
      "epoch": 1.276,
      "grad_norm": 0.0004520416259765625,
      "kl": 0.017128141969442366,
      "learning_rate": 1.744075273123889e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6380
    },
    {
      "completion_length": 66.125,
      "epoch": 1.278,
      "grad_norm": 0.00061798095703125,
      "kl": 0.07104477211833,
      "learning_rate": 1.735761737592236e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6390
    },
    {
      "completion_length": 61.725,
      "epoch": 1.28,
      "grad_norm": 0.00168609619140625,
      "kl": 0.022364417230710386,
      "learning_rate": 1.7274575140626318e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6400
    },
    {
      "completion_length": 59.975,
      "epoch": 1.282,
      "grad_norm": 0.0005950927734375,
      "kl": 0.04414304066449404,
      "learning_rate": 1.7191627037194187e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6410
    },
    {
      "completion_length": 58.35,
      "epoch": 1.284,
      "grad_norm": 0.0004100799560546875,
      "kl": 71.30782471811399,
      "learning_rate": 1.7108774076322443e-06,
      "loss": 0.0071,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6420
    },
    {
      "completion_length": 54.925,
      "epoch": 1.286,
      "grad_norm": 25.625,
      "kl": 3.6482051144819705,
      "learning_rate": 1.702601726754825e-06,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6430
    },
    {
      "completion_length": 51.125,
      "epoch": 1.288,
      "grad_norm": 0.00084686279296875,
      "kl": 0.06689287801855244,
      "learning_rate": 1.6943357619237227e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6440
    },
    {
      "completion_length": 60.625,
      "epoch": 1.29,
      "grad_norm": 0.00052642822265625,
      "kl": 0.03276003615465015,
      "learning_rate": 1.686079613857109e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6450
    },
    {
      "completion_length": 49.8,
      "epoch": 1.292,
      "grad_norm": 0.004669189453125,
      "kl": 0.04734173566102982,
      "learning_rate": 1.677833383153542e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6460
    },
    {
      "completion_length": 50.925,
      "epoch": 1.294,
      "grad_norm": 0.00040435791015625,
      "kl": 0.08111863350495696,
      "learning_rate": 1.6695971702907425e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6470
    },
    {
      "completion_length": 51.975,
      "epoch": 1.296,
      "grad_norm": 0.000335693359375,
      "kl": 0.034373713890090585,
      "learning_rate": 1.661371075624363e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6480
    },
    {
      "completion_length": 56.55,
      "epoch": 1.298,
      "grad_norm": 0.0008544921875,
      "kl": 0.05773493410088122,
      "learning_rate": 1.6531551993867717e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6490
    },
    {
      "completion_length": 50.95,
      "epoch": 1.3,
      "grad_norm": 282.0,
      "kl": 178.04388241134583,
      "learning_rate": 1.6449496416858285e-06,
      "loss": 0.0178,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6500
    },
    {
      "completion_length": 58.625,
      "epoch": 1.302,
      "grad_norm": 0.0023651123046875,
      "kl": 5.505481028556824,
      "learning_rate": 1.6367545025036634e-06,
      "loss": 0.0006,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6510
    },
    {
      "completion_length": 67.575,
      "epoch": 1.304,
      "grad_norm": 0.001068115234375,
      "kl": 0.041122534591704604,
      "learning_rate": 1.6285698816954626e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6520
    },
    {
      "completion_length": 49.75,
      "epoch": 1.306,
      "grad_norm": 0.000415802001953125,
      "kl": 0.165414993558079,
      "learning_rate": 1.6203958789882457e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6530
    },
    {
      "completion_length": 56.675,
      "epoch": 1.308,
      "grad_norm": 0.0007476806640625,
      "kl": 0.2703967327717692,
      "learning_rate": 1.612232593979658e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6540
    },
    {
      "completion_length": 56.2,
      "epoch": 1.31,
      "grad_norm": 0.00133514404296875,
      "kl": 0.02644283170811832,
      "learning_rate": 1.6040801261367494e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6550
    },
    {
      "completion_length": 58.05,
      "epoch": 1.312,
      "grad_norm": 0.00030517578125,
      "kl": 0.015307861985638738,
      "learning_rate": 1.5959385747947697e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6560
    },
    {
      "completion_length": 73.35,
      "epoch": 1.314,
      "grad_norm": 0.0004405975341796875,
      "kl": 0.013954693730920554,
      "learning_rate": 1.5878080391559507e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6570
    },
    {
      "completion_length": 51.05,
      "epoch": 1.316,
      "grad_norm": 0.0003566741943359375,
      "kl": 684.7183584340382,
      "learning_rate": 1.5796886182883053e-06,
      "loss": 0.0685,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6580
    },
    {
      "completion_length": 67.15,
      "epoch": 1.318,
      "grad_norm": 0.000598907470703125,
      "kl": 0.024668072490021585,
      "learning_rate": 1.5715804111244138e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6590
    },
    {
      "completion_length": 66.8,
      "epoch": 1.32,
      "grad_norm": 0.00738525390625,
      "kl": 0.047915787994861604,
      "learning_rate": 1.56348351646022e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6600
    },
    {
      "completion_length": 61.85,
      "epoch": 1.322,
      "grad_norm": 0.000537872314453125,
      "kl": 0.05744472313672304,
      "learning_rate": 1.5553980329538326e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6610
    },
    {
      "completion_length": 57.5,
      "epoch": 1.324,
      "grad_norm": 0.0006561279296875,
      "kl": 0.05604997184127569,
      "learning_rate": 1.547324059124315e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6620
    },
    {
      "completion_length": 59.425,
      "epoch": 1.326,
      "grad_norm": 0.00112152099609375,
      "kl": 0.0188056749291718,
      "learning_rate": 1.539261693350491e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6630
    },
    {
      "completion_length": 40.9,
      "epoch": 1.328,
      "grad_norm": 0.00072479248046875,
      "kl": 0.1357155740261078,
      "learning_rate": 1.5312110338697427e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6640
    },
    {
      "completion_length": 61.5,
      "epoch": 1.33,
      "grad_norm": 0.000690460205078125,
      "kl": 0.018099735863506793,
      "learning_rate": 1.5231721787768162e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6650
    },
    {
      "completion_length": 54.125,
      "epoch": 1.332,
      "grad_norm": 0.000720977783203125,
      "kl": 0.014814224326983094,
      "learning_rate": 1.5151452260226224e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6660
    },
    {
      "completion_length": 64.325,
      "epoch": 1.334,
      "grad_norm": 0.00016117095947265625,
      "kl": 0.016494302544742823,
      "learning_rate": 1.5071302734130488e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6670
    },
    {
      "completion_length": 59.2,
      "epoch": 1.336,
      "grad_norm": 0.00194549560546875,
      "kl": 0.06070426572114229,
      "learning_rate": 1.4991274186077632e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6680
    },
    {
      "completion_length": 59.15,
      "epoch": 1.338,
      "grad_norm": 0.0006256103515625,
      "kl": 0.11375871314667166,
      "learning_rate": 1.491136759119025e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6690
    },
    {
      "completion_length": 52.65,
      "epoch": 1.34,
      "grad_norm": 0.000713348388671875,
      "kl": 0.03938477258197963,
      "learning_rate": 1.4831583923105e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6700
    },
    {
      "completion_length": 63.525,
      "epoch": 1.342,
      "grad_norm": 0.00083160400390625,
      "kl": 0.020069646975025536,
      "learning_rate": 1.4751924153960681e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6710
    },
    {
      "completion_length": 53.7,
      "epoch": 1.3439999999999999,
      "grad_norm": 0.06787109375,
      "kl": 0.14404951045289635,
      "learning_rate": 1.467238925438646e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6720
    },
    {
      "completion_length": 55.475,
      "epoch": 1.346,
      "grad_norm": 0.0015411376953125,
      "kl": 180.89379140562377,
      "learning_rate": 1.4592980193489975e-06,
      "loss": 0.0181,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6730
    },
    {
      "completion_length": 53.55,
      "epoch": 1.3479999999999999,
      "grad_norm": 0.00067138671875,
      "kl": 0.05597533159889281,
      "learning_rate": 1.4513697938845571e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6740
    },
    {
      "completion_length": 53.1,
      "epoch": 1.35,
      "grad_norm": 0.002777099609375,
      "kl": 0.3969091270118952,
      "learning_rate": 1.443454345648252e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 6750
    },
    {
      "completion_length": 43.8,
      "epoch": 1.3519999999999999,
      "grad_norm": 0.0003185272216796875,
      "kl": 0.10832225987687708,
      "learning_rate": 1.4355517710873184e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6760
    },
    {
      "completion_length": 68.725,
      "epoch": 1.354,
      "grad_norm": 0.0004730224609375,
      "kl": 0.03166971495375037,
      "learning_rate": 1.4276621664921358e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6770
    },
    {
      "completion_length": 63.625,
      "epoch": 1.3559999999999999,
      "grad_norm": 0.00077056884765625,
      "kl": 0.018040235806256532,
      "learning_rate": 1.419785627995044e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6780
    },
    {
      "completion_length": 56.7,
      "epoch": 1.358,
      "grad_norm": 0.0103759765625,
      "kl": 0.08500627786852419,
      "learning_rate": 1.4119222515691817e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6790
    },
    {
      "completion_length": 60.375,
      "epoch": 1.3599999999999999,
      "grad_norm": 0.000652313232421875,
      "kl": 0.13012904403731226,
      "learning_rate": 1.4040721330273063e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6800
    },
    {
      "completion_length": 51.225,
      "epoch": 1.362,
      "grad_norm": 0.00110626220703125,
      "kl": 0.019943116419017314,
      "learning_rate": 1.3962353680206372e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6810
    },
    {
      "completion_length": 62.1,
      "epoch": 1.3639999999999999,
      "grad_norm": 0.000614166259765625,
      "kl": 0.08246160177513957,
      "learning_rate": 1.388412052037682e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 6820
    },
    {
      "completion_length": 49.75,
      "epoch": 1.366,
      "grad_norm": 0.0008697509765625,
      "kl": 95.11083188317716,
      "learning_rate": 1.380602280403076e-06,
      "loss": 0.0095,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 6830
    },
    {
      "completion_length": 63.875,
      "epoch": 1.3679999999999999,
      "grad_norm": 0.000579833984375,
      "kl": 0.027737328410148622,
      "learning_rate": 1.3728061482764238e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6840
    },
    {
      "completion_length": 56.325,
      "epoch": 1.37,
      "grad_norm": 0.00054168701171875,
      "kl": 0.06817373894155025,
      "learning_rate": 1.3650237506511333e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6850
    },
    {
      "completion_length": 56.875,
      "epoch": 1.3719999999999999,
      "grad_norm": 0.000423431396484375,
      "kl": 0.014378735097125173,
      "learning_rate": 1.3572551823532654e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6860
    },
    {
      "completion_length": 58.425,
      "epoch": 1.374,
      "grad_norm": 0.00054168701171875,
      "kl": 0.43131620325148107,
      "learning_rate": 1.349500538040371e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 6870
    },
    {
      "completion_length": 55.575,
      "epoch": 1.376,
      "grad_norm": 0.00066375732421875,
      "kl": 0.033100543078035116,
      "learning_rate": 1.3417599122003464e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6880
    },
    {
      "completion_length": 54.85,
      "epoch": 1.3780000000000001,
      "grad_norm": 0.002105712890625,
      "kl": 0.04654085249640048,
      "learning_rate": 1.3340333991502723e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6890
    },
    {
      "completion_length": 68.475,
      "epoch": 1.38,
      "grad_norm": 0.0005035400390625,
      "kl": 0.05651907054707408,
      "learning_rate": 1.3263210930352737e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6900
    },
    {
      "completion_length": 61.55,
      "epoch": 1.3820000000000001,
      "grad_norm": 0.00054168701171875,
      "kl": 0.03381169466301799,
      "learning_rate": 1.3186230878273654e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6910
    },
    {
      "completion_length": 57.0,
      "epoch": 1.384,
      "grad_norm": 0.00049591064453125,
      "kl": 0.6785514406859875,
      "learning_rate": 1.3109394773243117e-06,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6920
    },
    {
      "completion_length": 61.8,
      "epoch": 1.3860000000000001,
      "grad_norm": 0.020751953125,
      "kl": 0.08321558614261448,
      "learning_rate": 1.3032703551484832e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6930
    },
    {
      "completion_length": 60.55,
      "epoch": 1.388,
      "grad_norm": 0.000690460205078125,
      "kl": 0.09076761337928474,
      "learning_rate": 1.2956158147457116e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6940
    },
    {
      "completion_length": 49.25,
      "epoch": 1.3900000000000001,
      "grad_norm": 0.000667572021484375,
      "kl": 0.04632122702896595,
      "learning_rate": 1.2879759493841577e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6950
    },
    {
      "completion_length": 66.75,
      "epoch": 1.392,
      "grad_norm": 0.000629425048828125,
      "kl": 3.6676836960949,
      "learning_rate": 1.280350852153168e-06,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 6960
    },
    {
      "completion_length": 58.6,
      "epoch": 1.3940000000000001,
      "grad_norm": 0.0003719329833984375,
      "kl": 0.07259991895407439,
      "learning_rate": 1.272740615962148e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6970
    },
    {
      "completion_length": 62.025,
      "epoch": 1.396,
      "grad_norm": 0.00069427490234375,
      "kl": 0.01472460343502462,
      "learning_rate": 1.2651453335394232e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6980
    },
    {
      "completion_length": 64.175,
      "epoch": 1.3980000000000001,
      "grad_norm": 0.000606536865234375,
      "kl": 0.18156763026490808,
      "learning_rate": 1.2575650974311118e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 6990
    },
    {
      "completion_length": 50.15,
      "epoch": 1.4,
      "grad_norm": 14.125,
      "kl": 10.663465712498873,
      "learning_rate": 1.2500000000000007e-06,
      "loss": 0.0011,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.15,
      "rewards/reward_func": -0.075,
      "step": 7000
    },
    {
      "completion_length": 62.375,
      "epoch": 1.4020000000000001,
      "grad_norm": 0.000507354736328125,
      "kl": 0.14125907123088838,
      "learning_rate": 1.2424501334244124e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7010
    },
    {
      "completion_length": 52.6,
      "epoch": 1.404,
      "grad_norm": 0.0018310546875,
      "kl": 0.13884197538718582,
      "learning_rate": 1.234915589697091e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7020
    },
    {
      "completion_length": 55.575,
      "epoch": 1.4060000000000001,
      "grad_norm": 0.0006103515625,
      "kl": 31778.019179227947,
      "learning_rate": 1.2273964606240718e-06,
      "loss": 3.1778,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7030
    },
    {
      "completion_length": 48.4,
      "epoch": 1.408,
      "grad_norm": 0.0009613037109375,
      "kl": 0.17425558338873087,
      "learning_rate": 1.2198928378235717e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7040
    },
    {
      "completion_length": 47.075,
      "epoch": 1.41,
      "grad_norm": 0.000476837158203125,
      "kl": 0.045156693411991,
      "learning_rate": 1.2124048127248644e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7050
    },
    {
      "completion_length": 62.375,
      "epoch": 1.412,
      "grad_norm": 0.000751495361328125,
      "kl": 0.031479455251246694,
      "learning_rate": 1.204932476567175e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7060
    },
    {
      "completion_length": 55.525,
      "epoch": 1.414,
      "grad_norm": 0.00058746337890625,
      "kl": 0.13692689267918468,
      "learning_rate": 1.19747592039856e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7070
    },
    {
      "completion_length": 47.2,
      "epoch": 1.416,
      "grad_norm": 0.000492095947265625,
      "kl": 0.028804597025737167,
      "learning_rate": 1.1900352350748026e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7080
    },
    {
      "completion_length": 59.075,
      "epoch": 1.418,
      "grad_norm": 0.0003509521484375,
      "kl": 0.04570387415587902,
      "learning_rate": 1.1826105112583061e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7090
    },
    {
      "completion_length": 56.05,
      "epoch": 1.42,
      "grad_norm": 0.0004558563232421875,
      "kl": 0.018374279094859957,
      "learning_rate": 1.1752018394169882e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7100
    },
    {
      "completion_length": 61.95,
      "epoch": 1.422,
      "grad_norm": 0.000583648681640625,
      "kl": 0.03715153355151415,
      "learning_rate": 1.1678093098231748e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7110
    },
    {
      "completion_length": 54.9,
      "epoch": 1.424,
      "grad_norm": 0.000518798828125,
      "kl": 0.016605707909911872,
      "learning_rate": 1.160433012552508e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7120
    },
    {
      "completion_length": 57.55,
      "epoch": 1.426,
      "grad_norm": 0.0004215240478515625,
      "kl": 0.018531074468046426,
      "learning_rate": 1.1530730374828422e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7130
    },
    {
      "completion_length": 58.0,
      "epoch": 1.428,
      "grad_norm": 0.000522613525390625,
      "kl": 0.06935790865682065,
      "learning_rate": 1.1457294742931508e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7140
    },
    {
      "completion_length": 52.5,
      "epoch": 1.43,
      "grad_norm": 0.0010833740234375,
      "kl": 0.09027541326358915,
      "learning_rate": 1.1384024124624324e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7150
    },
    {
      "completion_length": 68.775,
      "epoch": 1.432,
      "grad_norm": 0.00066375732421875,
      "kl": 0.017101448262110353,
      "learning_rate": 1.1310919412686248e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7160
    },
    {
      "completion_length": 66.075,
      "epoch": 1.434,
      "grad_norm": 0.0003528594970703125,
      "kl": 0.10510317548178136,
      "learning_rate": 1.1237981497875112e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 7170
    },
    {
      "completion_length": 58.75,
      "epoch": 1.436,
      "grad_norm": 0.0003719329833984375,
      "kl": 0.07255538417957723,
      "learning_rate": 1.11652112689164e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7180
    },
    {
      "completion_length": 55.575,
      "epoch": 1.438,
      "grad_norm": 0.000713348388671875,
      "kl": 0.04587976224720478,
      "learning_rate": 1.109260961249238e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7190
    },
    {
      "completion_length": 44.3,
      "epoch": 1.44,
      "grad_norm": 0.0007171630859375,
      "kl": 0.026211364893242717,
      "learning_rate": 1.1020177413231334e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7200
    },
    {
      "completion_length": 55.4,
      "epoch": 1.442,
      "grad_norm": 0.00011587142944335938,
      "kl": 1273.178384515643,
      "learning_rate": 1.0947915553696742e-06,
      "loss": 0.1273,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.15773502588272095,
      "rewards/reward_func": -0.1,
      "step": 7210
    },
    {
      "completion_length": 70.35,
      "epoch": 1.444,
      "grad_norm": 0.0002689361572265625,
      "kl": 0.30778478598222136,
      "learning_rate": 1.0875824914376555e-06,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 7220
    },
    {
      "completion_length": 51.35,
      "epoch": 1.446,
      "grad_norm": 0.000743865966796875,
      "kl": 0.11805587047711015,
      "learning_rate": 1.0803906373672477e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7230
    },
    {
      "completion_length": 54.1,
      "epoch": 1.448,
      "grad_norm": 0.00083160400390625,
      "kl": 0.13561045327223836,
      "learning_rate": 1.073216080788921e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7240
    },
    {
      "completion_length": 58.075,
      "epoch": 1.45,
      "grad_norm": 0.0007781982421875,
      "kl": 0.01598156727850437,
      "learning_rate": 1.0660589091223854e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7250
    },
    {
      "completion_length": 55.825,
      "epoch": 1.452,
      "grad_norm": 0.000614166259765625,
      "kl": 0.08759649377316236,
      "learning_rate": 1.0589192095755172e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7260
    },
    {
      "completion_length": 54.95,
      "epoch": 1.454,
      "grad_norm": 0.000942230224609375,
      "kl": 2.808669605664909,
      "learning_rate": 1.0517970691433035e-06,
      "loss": 0.0003,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7270
    },
    {
      "completion_length": 44.8,
      "epoch": 1.456,
      "grad_norm": 0.0010833740234375,
      "kl": 177.8802186036017,
      "learning_rate": 1.0446925746067768e-06,
      "loss": 0.0178,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 7280
    },
    {
      "completion_length": 55.0,
      "epoch": 1.458,
      "grad_norm": 0.001861572265625,
      "kl": 0.25618111025542023,
      "learning_rate": 1.0376058125319614e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7290
    },
    {
      "completion_length": 65.625,
      "epoch": 1.46,
      "grad_norm": 0.01287841796875,
      "kl": 0.1665965816937387,
      "learning_rate": 1.0305368692688175e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7300
    },
    {
      "completion_length": 56.25,
      "epoch": 1.462,
      "grad_norm": 0.00029754638671875,
      "kl": 0.4289894063025713,
      "learning_rate": 1.0234858309501864e-06,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7310
    },
    {
      "completion_length": 48.675,
      "epoch": 1.464,
      "grad_norm": 2.328125,
      "kl": 44.01882844008505,
      "learning_rate": 1.0164527834907468e-06,
      "loss": 0.0044,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7320
    },
    {
      "completion_length": 50.35,
      "epoch": 1.466,
      "grad_norm": 0.0024261474609375,
      "kl": 0.13080412773415445,
      "learning_rate": 1.0094378125859602e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7330
    },
    {
      "completion_length": 55.85,
      "epoch": 1.468,
      "grad_norm": 0.000461578369140625,
      "kl": 0.0169123521191068,
      "learning_rate": 1.0024410037110358e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7340
    },
    {
      "completion_length": 49.05,
      "epoch": 1.47,
      "grad_norm": 0.00058746337890625,
      "kl": 0.13749618739821018,
      "learning_rate": 9.95462442119879e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7350
    },
    {
      "completion_length": 64.4,
      "epoch": 1.472,
      "grad_norm": 0.000614166259765625,
      "kl": 0.025387801649048924,
      "learning_rate": 9.88502212844063e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7360
    },
    {
      "completion_length": 66.95,
      "epoch": 1.474,
      "grad_norm": 0.00026702880859375,
      "kl": 0.03902003513649106,
      "learning_rate": 9.815604006917839e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7370
    },
    {
      "completion_length": 54.0,
      "epoch": 1.476,
      "grad_norm": 0.0159912109375,
      "kl": 0.08616708847694099,
      "learning_rate": 9.746370902468311e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7380
    },
    {
      "completion_length": 61.025,
      "epoch": 1.478,
      "grad_norm": 0.00043487548828125,
      "kl": 0.027717783488333224,
      "learning_rate": 9.677323658675594e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7390
    },
    {
      "completion_length": 47.25,
      "epoch": 1.48,
      "grad_norm": 0.00274658203125,
      "kl": 0.09193211463280022,
      "learning_rate": 9.608463116858544e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7400
    },
    {
      "completion_length": 61.15,
      "epoch": 1.482,
      "grad_norm": 0.00194549560546875,
      "kl": 0.09381414433009923,
      "learning_rate": 9.53979011606115e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7410
    },
    {
      "completion_length": 55.875,
      "epoch": 1.484,
      "grad_norm": 0.000873565673828125,
      "kl": 0.06577477985993027,
      "learning_rate": 9.471305493042243e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7420
    },
    {
      "completion_length": 55.325,
      "epoch": 1.486,
      "grad_norm": 0.0006866455078125,
      "kl": 0.05357563262805343,
      "learning_rate": 9.403010082265351e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7430
    },
    {
      "completion_length": 55.225,
      "epoch": 1.488,
      "grad_norm": 0.000606536865234375,
      "kl": 0.043947093037422745,
      "learning_rate": 9.334904715888496e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7440
    },
    {
      "completion_length": 47.125,
      "epoch": 1.49,
      "grad_norm": 0.0005035400390625,
      "kl": 0.020906020514667036,
      "learning_rate": 9.266990223754069e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7450
    },
    {
      "completion_length": 75.375,
      "epoch": 1.492,
      "grad_norm": 0.0007476806640625,
      "kl": 0.0432497413828969,
      "learning_rate": 9.199267433378728e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7460
    },
    {
      "completion_length": 66.75,
      "epoch": 1.494,
      "grad_norm": 0.000682830810546875,
      "kl": 0.01475386363454163,
      "learning_rate": 9.131737169943314e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7470
    },
    {
      "completion_length": 53.25,
      "epoch": 1.496,
      "grad_norm": 0.000530242919921875,
      "kl": 0.034680284932255744,
      "learning_rate": 9.064400256282757e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7480
    },
    {
      "completion_length": 66.5,
      "epoch": 1.498,
      "grad_norm": 0.0012664794921875,
      "kl": 0.020779677666723728,
      "learning_rate": 8.99725751287611e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7490
    },
    {
      "completion_length": 69.5,
      "epoch": 1.5,
      "grad_norm": 35.5,
      "kl": 10.81919735018164,
      "learning_rate": 8.930309757836517e-07,
      "loss": 0.0011,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 7500
    },
    {
      "completion_length": 63.825,
      "epoch": 1.502,
      "grad_norm": 0.000682830810546875,
      "kl": 0.018158415833022447,
      "learning_rate": 8.863557806901233e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7510
    },
    {
      "completion_length": 46.225,
      "epoch": 1.504,
      "grad_norm": 0.0003185272216796875,
      "kl": 0.04219387628836557,
      "learning_rate": 8.797002473421729e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7520
    },
    {
      "completion_length": 70.0,
      "epoch": 1.506,
      "grad_norm": 0.00041961669921875,
      "kl": 0.03907957626506686,
      "learning_rate": 8.73064456835373e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7530
    },
    {
      "completion_length": 60.8,
      "epoch": 1.508,
      "grad_norm": 0.000659942626953125,
      "kl": 152.24121750062332,
      "learning_rate": 8.664484900247363e-07,
      "loss": 0.0152,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.0,
      "rewards/reward_func": -0.1,
      "step": 7540
    },
    {
      "completion_length": 46.375,
      "epoch": 1.51,
      "grad_norm": 0.001556396484375,
      "kl": 0.2765787610784173,
      "learning_rate": 8.598524275237321e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7550
    },
    {
      "completion_length": 68.675,
      "epoch": 1.512,
      "grad_norm": 0.0027008056640625,
      "kl": 0.028881799709051848,
      "learning_rate": 8.532763497032987e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7560
    },
    {
      "completion_length": 63.8,
      "epoch": 1.514,
      "grad_norm": 0.000469207763671875,
      "kl": 0.04086120091378689,
      "learning_rate": 8.467203366908708e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7570
    },
    {
      "completion_length": 49.95,
      "epoch": 1.516,
      "grad_norm": 0.0037689208984375,
      "kl": 0.12793728783726693,
      "learning_rate": 8.40184468369396e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7580
    },
    {
      "completion_length": 53.5,
      "epoch": 1.518,
      "grad_norm": 0.00121307373046875,
      "kl": 42.36696035126224,
      "learning_rate": 8.336688243763691e-07,
      "loss": 0.0042,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 7590
    },
    {
      "completion_length": 47.325,
      "epoch": 1.52,
      "grad_norm": 0.0006103515625,
      "kl": 14.073435558238998,
      "learning_rate": 8.271734841028553e-07,
      "loss": 0.0014,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 7600
    },
    {
      "completion_length": 65.0,
      "epoch": 1.522,
      "grad_norm": 0.000720977783203125,
      "kl": 0.04525826433673501,
      "learning_rate": 8.206985266925249e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7610
    },
    {
      "completion_length": 59.525,
      "epoch": 1.524,
      "grad_norm": 0.000843048095703125,
      "kl": 0.04048813302069902,
      "learning_rate": 8.142440310406923e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7620
    },
    {
      "completion_length": 46.45,
      "epoch": 1.526,
      "grad_norm": 0.0004329681396484375,
      "kl": 8.161725069396198,
      "learning_rate": 8.078100757933486e-07,
      "loss": 0.0008,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7630
    },
    {
      "completion_length": 57.225,
      "epoch": 1.528,
      "grad_norm": 0.000698089599609375,
      "kl": 0.1071649724675808,
      "learning_rate": 8.013967393462094e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7640
    },
    {
      "completion_length": 52.975,
      "epoch": 1.53,
      "grad_norm": 0.000431060791015625,
      "kl": 0.12129491865634918,
      "learning_rate": 7.950040998437541e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7650
    },
    {
      "completion_length": 43.225,
      "epoch": 1.532,
      "grad_norm": 0.0026702880859375,
      "kl": 0.1067592917010188,
      "learning_rate": 7.886322351782782e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7660
    },
    {
      "completion_length": 59.9,
      "epoch": 1.534,
      "grad_norm": 0.00093841552734375,
      "kl": 0.05229797107167542,
      "learning_rate": 7.822812229889429e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7670
    },
    {
      "completion_length": 65.525,
      "epoch": 1.536,
      "grad_norm": 0.0004405975341796875,
      "kl": 0.010464739426970482,
      "learning_rate": 7.759511406608255e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7680
    },
    {
      "completion_length": 66.625,
      "epoch": 1.538,
      "grad_norm": 0.0004825592041015625,
      "kl": 0.045739847654476765,
      "learning_rate": 7.696420653239834e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7690
    },
    {
      "completion_length": 73.575,
      "epoch": 1.54,
      "grad_norm": 0.00058746337890625,
      "kl": 0.031135138869285584,
      "learning_rate": 7.633540738525066e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7700
    },
    {
      "completion_length": 48.175,
      "epoch": 1.542,
      "grad_norm": 51.25,
      "kl": 80.07849281346425,
      "learning_rate": 7.57087242863589e-07,
      "loss": 0.008,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7710
    },
    {
      "completion_length": 46.375,
      "epoch": 1.544,
      "grad_norm": 0.00138092041015625,
      "kl": 0.0285742097068578,
      "learning_rate": 7.508416487165862e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7720
    },
    {
      "completion_length": 57.2,
      "epoch": 1.546,
      "grad_norm": 0.000492095947265625,
      "kl": 0.07636187486350536,
      "learning_rate": 7.44617367512094e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7730
    },
    {
      "completion_length": 56.15,
      "epoch": 1.548,
      "grad_norm": 0.00045013427734375,
      "kl": 0.03193683328572661,
      "learning_rate": 7.384144750910133e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7740
    },
    {
      "completion_length": 59.65,
      "epoch": 1.55,
      "grad_norm": 0.000370025634765625,
      "kl": 0.09212675780290738,
      "learning_rate": 7.322330470336314e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7750
    },
    {
      "completion_length": 53.225,
      "epoch": 1.552,
      "grad_norm": 0.000499725341796875,
      "kl": 0.0655658102594316,
      "learning_rate": 7.260731586586983e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7760
    },
    {
      "completion_length": 50.3,
      "epoch": 1.554,
      "grad_norm": 0.00186920166015625,
      "kl": 0.07870109416544438,
      "learning_rate": 7.199348850225091e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7770
    },
    {
      "completion_length": 53.8,
      "epoch": 1.556,
      "grad_norm": 0.003204345703125,
      "kl": 146.1276578912046,
      "learning_rate": 7.138183009179922e-07,
      "loss": 0.0146,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7780
    },
    {
      "completion_length": 69.675,
      "epoch": 1.558,
      "grad_norm": 0.0005645751953125,
      "kl": 0.028883875254541634,
      "learning_rate": 7.077234808737932e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7790
    },
    {
      "completion_length": 67.7,
      "epoch": 1.56,
      "grad_norm": 0.00102996826171875,
      "kl": 0.0383193613961339,
      "learning_rate": 7.016504991533727e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7800
    },
    {
      "completion_length": 59.3,
      "epoch": 1.562,
      "grad_norm": 0.001800537109375,
      "kl": 0.07388523239642382,
      "learning_rate": 6.955994297540947e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7810
    },
    {
      "completion_length": 56.75,
      "epoch": 1.564,
      "grad_norm": 0.000614166259765625,
      "kl": 0.08295210748910904,
      "learning_rate": 6.895703464063319e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7820
    },
    {
      "completion_length": 60.15,
      "epoch": 1.5659999999999998,
      "grad_norm": 0.00628662109375,
      "kl": 0.03666973649524152,
      "learning_rate": 6.835633225725604e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7830
    },
    {
      "completion_length": 67.05,
      "epoch": 1.568,
      "grad_norm": 0.000774383544921875,
      "kl": 0.017159267235547303,
      "learning_rate": 6.775784314464717e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7840
    },
    {
      "completion_length": 63.4,
      "epoch": 1.5699999999999998,
      "grad_norm": 0.00055694580078125,
      "kl": 0.01824809005483985,
      "learning_rate": 6.716157459520739e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7850
    },
    {
      "completion_length": 54.525,
      "epoch": 1.572,
      "grad_norm": 0.0005035400390625,
      "kl": 0.03218274647369981,
      "learning_rate": 6.656753387428089e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7860
    },
    {
      "completion_length": 70.525,
      "epoch": 1.5739999999999998,
      "grad_norm": 0.000705718994140625,
      "kl": 0.0729364191647619,
      "learning_rate": 6.597572822006643e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7870
    },
    {
      "completion_length": 69.875,
      "epoch": 1.576,
      "grad_norm": 0.0003833770751953125,
      "kl": 0.07040122235193849,
      "learning_rate": 6.538616484352902e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7880
    },
    {
      "completion_length": 59.375,
      "epoch": 1.5779999999999998,
      "grad_norm": 0.001129150390625,
      "kl": 0.04789869613014162,
      "learning_rate": 6.479885092831251e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7890
    },
    {
      "completion_length": 56.575,
      "epoch": 1.58,
      "grad_norm": 0.00049591064453125,
      "kl": 0.05556117547675967,
      "learning_rate": 6.421379363065142e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7900
    },
    {
      "completion_length": 50.85,
      "epoch": 1.5819999999999999,
      "grad_norm": 0.000957489013671875,
      "kl": 0.07133134175091982,
      "learning_rate": 6.363100007928447e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7910
    },
    {
      "completion_length": 74.025,
      "epoch": 1.584,
      "grad_norm": 0.0003108978271484375,
      "kl": 0.10760618806816638,
      "learning_rate": 6.305047737536707e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7920
    },
    {
      "completion_length": 56.6,
      "epoch": 1.5859999999999999,
      "grad_norm": 0.00092315673828125,
      "kl": 2.8826652359217406,
      "learning_rate": 6.247223259238511e-07,
      "loss": 0.0003,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 7930
    },
    {
      "completion_length": 55.475,
      "epoch": 1.588,
      "grad_norm": 0.0006103515625,
      "kl": 2.284485016670078,
      "learning_rate": 6.189627277606894e-07,
      "loss": 0.0002,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7940
    },
    {
      "completion_length": 46.125,
      "epoch": 1.5899999999999999,
      "grad_norm": 0.000972747802734375,
      "kl": 0.12067738296464085,
      "learning_rate": 6.1322604944307e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7950
    },
    {
      "completion_length": 59.225,
      "epoch": 1.592,
      "grad_norm": 0.00066375732421875,
      "kl": 0.14497559778392316,
      "learning_rate": 6.075123608706093e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7960
    },
    {
      "completion_length": 60.925,
      "epoch": 1.5939999999999999,
      "grad_norm": 0.0024566650390625,
      "kl": 258.11860719914546,
      "learning_rate": 6.01821731662798e-07,
      "loss": 0.0258,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 7970
    },
    {
      "completion_length": 62.225,
      "epoch": 1.596,
      "grad_norm": 0.00153350830078125,
      "kl": 0.024051298201084138,
      "learning_rate": 5.961542311581586e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7980
    },
    {
      "completion_length": 50.25,
      "epoch": 1.5979999999999999,
      "grad_norm": 0.0004482269287109375,
      "kl": 0.10747648775577545,
      "learning_rate": 5.905099284133953e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 7990
    },
    {
      "completion_length": 52.1,
      "epoch": 1.6,
      "grad_norm": 0.00038909912109375,
      "kl": 4.900969664240256,
      "learning_rate": 5.848888922025553e-07,
      "loss": 0.0005,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 8000
    },
    {
      "completion_length": 46.85,
      "epoch": 1.6019999999999999,
      "grad_norm": 0.000553131103515625,
      "kl": 0.06989260124973953,
      "learning_rate": 5.792911910161922e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8010
    },
    {
      "completion_length": 64.45,
      "epoch": 1.604,
      "grad_norm": 0.0006561279296875,
      "kl": 0.054861510870978236,
      "learning_rate": 5.737168930605272e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8020
    },
    {
      "completion_length": 69.5,
      "epoch": 1.6059999999999999,
      "grad_norm": 0.0003757476806640625,
      "kl": 0.01320057879202068,
      "learning_rate": 5.681660662566225e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8030
    },
    {
      "completion_length": 48.75,
      "epoch": 1.608,
      "grad_norm": 0.000743865966796875,
      "kl": 0.5451168741099537,
      "learning_rate": 5.626387782395512e-07,
      "loss": 0.0001,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.15,
      "rewards/reward_func": -0.075,
      "step": 8040
    },
    {
      "completion_length": 57.45,
      "epoch": 1.6099999999999999,
      "grad_norm": 0.0004177093505859375,
      "kl": 0.06206353167071939,
      "learning_rate": 5.571350963575728e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8050
    },
    {
      "completion_length": 54.175,
      "epoch": 1.612,
      "grad_norm": 0.000614166259765625,
      "kl": 2.013091558404267,
      "learning_rate": 5.516550876713142e-07,
      "loss": 0.0002,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 8060
    },
    {
      "completion_length": 61.025,
      "epoch": 1.6139999999999999,
      "grad_norm": 0.000652313232421875,
      "kl": 0.06326800542883575,
      "learning_rate": 5.461988189529529e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8070
    },
    {
      "completion_length": 58.825,
      "epoch": 1.616,
      "grad_norm": 0.00104522705078125,
      "kl": 0.014891783054918051,
      "learning_rate": 5.407663566854008e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8080
    },
    {
      "completion_length": 54.875,
      "epoch": 1.6179999999999999,
      "grad_norm": 0.0020904541015625,
      "kl": 0.025240180967375635,
      "learning_rate": 5.353577670614951e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8090
    },
    {
      "completion_length": 61.5,
      "epoch": 1.62,
      "grad_norm": 0.000614166259765625,
      "kl": 0.04227957231923938,
      "learning_rate": 5.299731159831953e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8100
    },
    {
      "completion_length": 65.5,
      "epoch": 1.6219999999999999,
      "grad_norm": 0.0004177093505859375,
      "kl": 0.19428066378459335,
      "learning_rate": 5.24612469060774e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8110
    },
    {
      "completion_length": 53.05,
      "epoch": 1.624,
      "grad_norm": 0.0010986328125,
      "kl": 0.042151403008028866,
      "learning_rate": 5.192758916120236e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8120
    },
    {
      "completion_length": 55.55,
      "epoch": 1.626,
      "grad_norm": 0.00029754638671875,
      "kl": 0.02924617677927017,
      "learning_rate": 5.139634486614544e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8130
    },
    {
      "completion_length": 76.4,
      "epoch": 1.6280000000000001,
      "grad_norm": 0.01080322265625,
      "kl": 0.042456808709539474,
      "learning_rate": 5.086752049395094e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8140
    },
    {
      "completion_length": 60.325,
      "epoch": 1.63,
      "grad_norm": 0.01495361328125,
      "kl": 0.025815209513530134,
      "learning_rate": 5.034112248817685e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8150
    },
    {
      "completion_length": 71.75,
      "epoch": 1.6320000000000001,
      "grad_norm": 0.000865936279296875,
      "kl": 0.0641026332974434,
      "learning_rate": 4.981715726281666e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8160
    },
    {
      "completion_length": 57.4,
      "epoch": 1.634,
      "grad_norm": 0.000530242919921875,
      "kl": 0.045072671584784986,
      "learning_rate": 4.929563120222142e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8170
    },
    {
      "completion_length": 52.075,
      "epoch": 1.6360000000000001,
      "grad_norm": 0.00011157989501953125,
      "kl": 0.1052944268565625,
      "learning_rate": 4.87765506610215e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8180
    },
    {
      "completion_length": 54.375,
      "epoch": 1.638,
      "grad_norm": 0.0003986358642578125,
      "kl": 0.03237830828875303,
      "learning_rate": 4.825992196404958e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8190
    },
    {
      "completion_length": 56.3,
      "epoch": 1.6400000000000001,
      "grad_norm": 0.000835418701171875,
      "kl": 0.01498257415369153,
      "learning_rate": 4.774575140626317e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8200
    },
    {
      "completion_length": 66.55,
      "epoch": 1.642,
      "grad_norm": 0.0003414154052734375,
      "kl": 0.011996694607660174,
      "learning_rate": 4.7234045252668393e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8210
    },
    {
      "completion_length": 53.05,
      "epoch": 1.6440000000000001,
      "grad_norm": 0.0003986358642578125,
      "kl": 0.1085278536658734,
      "learning_rate": 4.672480973824312e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8220
    },
    {
      "completion_length": 58.675,
      "epoch": 1.646,
      "grad_norm": 0.00080108642578125,
      "kl": 0.04904728039400652,
      "learning_rate": 4.6218051067861423e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8230
    },
    {
      "completion_length": 50.825,
      "epoch": 1.6480000000000001,
      "grad_norm": 0.000759124755859375,
      "kl": 0.014587640948593616,
      "learning_rate": 4.5713775416217884e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8240
    },
    {
      "completion_length": 45.15,
      "epoch": 1.65,
      "grad_norm": 0.0003795623779296875,
      "kl": 0.09401618214324117,
      "learning_rate": 4.5211988927752026e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8250
    },
    {
      "completion_length": 64.925,
      "epoch": 1.6520000000000001,
      "grad_norm": 0.0026702880859375,
      "kl": 0.026571149285882712,
      "learning_rate": 4.4712697716573994e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8260
    },
    {
      "completion_length": 68.8,
      "epoch": 1.654,
      "grad_norm": 0.000606536865234375,
      "kl": 0.09327265082392841,
      "learning_rate": 4.421590786638952e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8270
    },
    {
      "completion_length": 58.775,
      "epoch": 1.6560000000000001,
      "grad_norm": 0.000858306884765625,
      "kl": 0.06238628029823303,
      "learning_rate": 4.372162543042624e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8280
    },
    {
      "completion_length": 56.575,
      "epoch": 1.658,
      "grad_norm": 0.00030517578125,
      "kl": 0.06320808534510433,
      "learning_rate": 4.3229856431359516e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8290
    },
    {
      "completion_length": 59.625,
      "epoch": 1.6600000000000001,
      "grad_norm": 0.000576019287109375,
      "kl": 0.06072661457583308,
      "learning_rate": 4.27406068612396e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8300
    },
    {
      "completion_length": 54.5,
      "epoch": 1.662,
      "grad_norm": 0.000698089599609375,
      "kl": 0.0206127600511536,
      "learning_rate": 4.225388268141797e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8310
    },
    {
      "completion_length": 64.5,
      "epoch": 1.6640000000000001,
      "grad_norm": 0.000766754150390625,
      "kl": 1.2397997039370239,
      "learning_rate": 4.1769689822475147e-07,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8320
    },
    {
      "completion_length": 63.85,
      "epoch": 1.666,
      "grad_norm": 0.00113677978515625,
      "kl": 0.04746299120597541,
      "learning_rate": 4.12880341841484e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8330
    },
    {
      "completion_length": 65.3,
      "epoch": 1.6680000000000001,
      "grad_norm": 0.000629425048828125,
      "kl": 0.07974740182980895,
      "learning_rate": 4.0808921635259595e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8340
    },
    {
      "completion_length": 54.1,
      "epoch": 1.67,
      "grad_norm": 0.000759124755859375,
      "kl": 350.0721945284866,
      "learning_rate": 4.033235801364402e-07,
      "loss": 0.035,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 8350
    },
    {
      "completion_length": 50.45,
      "epoch": 1.6720000000000002,
      "grad_norm": 0.00054168701171875,
      "kl": 7.292029631882906,
      "learning_rate": 3.9858349126078945e-07,
      "loss": 0.0007,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 8360
    },
    {
      "completion_length": 58.775,
      "epoch": 1.674,
      "grad_norm": 0.012939453125,
      "kl": 0.07852717223577202,
      "learning_rate": 3.938690074821314e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8370
    },
    {
      "completion_length": 54.425,
      "epoch": 1.6760000000000002,
      "grad_norm": 0.00070953369140625,
      "kl": 0.01418459378182888,
      "learning_rate": 3.891801862449629e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8380
    },
    {
      "completion_length": 50.175,
      "epoch": 1.678,
      "grad_norm": 0.000560760498046875,
      "kl": 0.03785524540580809,
      "learning_rate": 3.8451708468109026e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8390
    },
    {
      "completion_length": 43.075,
      "epoch": 1.6800000000000002,
      "grad_norm": 0.00136566162109375,
      "kl": 0.08654712834395469,
      "learning_rate": 3.798797596089351e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8400
    },
    {
      "completion_length": 59.775,
      "epoch": 1.682,
      "grad_norm": 0.0005950927734375,
      "kl": 0.2558928931131959,
      "learning_rate": 3.7526826753284065e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8410
    },
    {
      "completion_length": 58.375,
      "epoch": 1.6840000000000002,
      "grad_norm": 0.00067901611328125,
      "kl": 1.3383296761894599,
      "learning_rate": 3.7068266464238085e-07,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8420
    },
    {
      "completion_length": 64.6,
      "epoch": 1.686,
      "grad_norm": 0.0003833770751953125,
      "kl": 0.0377775629516691,
      "learning_rate": 3.661230068116811e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8430
    },
    {
      "completion_length": 58.925,
      "epoch": 1.688,
      "grad_norm": 0.00092315673828125,
      "kl": 0.05282154800370335,
      "learning_rate": 3.615893495987335e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8440
    },
    {
      "completion_length": 61.375,
      "epoch": 1.69,
      "grad_norm": 0.00079345703125,
      "kl": 104.9448153554462,
      "learning_rate": 3.5708174824471947e-07,
      "loss": 0.0105,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 8450
    },
    {
      "completion_length": 66.675,
      "epoch": 1.692,
      "grad_norm": 0.0004119873046875,
      "kl": 0.040050674229860306,
      "learning_rate": 3.5260025767333894e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8460
    },
    {
      "completion_length": 69.475,
      "epoch": 1.694,
      "grad_norm": 0.000682830810546875,
      "kl": 0.020584713015705348,
      "learning_rate": 3.481449324901412e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8470
    },
    {
      "completion_length": 66.525,
      "epoch": 1.696,
      "grad_norm": 0.0002193450927734375,
      "kl": 0.05711883215699345,
      "learning_rate": 3.4371582698185636e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8480
    },
    {
      "completion_length": 68.075,
      "epoch": 1.698,
      "grad_norm": 0.0002460479736328125,
      "kl": 0.05435404470190406,
      "learning_rate": 3.393129951157384e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8490
    },
    {
      "completion_length": 63.9,
      "epoch": 1.7,
      "grad_norm": 0.00174713134765625,
      "kl": 0.06182208526879549,
      "learning_rate": 3.3493649053890325e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8500
    },
    {
      "completion_length": 68.7,
      "epoch": 1.702,
      "grad_norm": 167.0,
      "kl": 88.8611083610449,
      "learning_rate": 3.3058636657767927e-07,
      "loss": 0.0089,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8510
    },
    {
      "completion_length": 76.675,
      "epoch": 1.704,
      "grad_norm": 0.0038909912109375,
      "kl": 0.05937002245336771,
      "learning_rate": 3.262626762369525e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8520
    },
    {
      "completion_length": 57.825,
      "epoch": 1.706,
      "grad_norm": 0.000690460205078125,
      "kl": 0.05280606346204877,
      "learning_rate": 3.219654721995266e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8530
    },
    {
      "completion_length": 48.85,
      "epoch": 1.708,
      "grad_norm": 0.000560760498046875,
      "kl": 188.36169426795095,
      "learning_rate": 3.176948068254762e-07,
      "loss": 0.0188,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8540
    },
    {
      "completion_length": 64.1,
      "epoch": 1.71,
      "grad_norm": 0.0003757476806640625,
      "kl": 0.01651700264774263,
      "learning_rate": 3.134507321515107e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8550
    },
    {
      "completion_length": 43.95,
      "epoch": 1.712,
      "grad_norm": 0.0003681182861328125,
      "kl": 5.833402361674234,
      "learning_rate": 3.092332998903416e-07,
      "loss": 0.0006,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 8560
    },
    {
      "completion_length": 60.85,
      "epoch": 1.714,
      "grad_norm": 0.005889892578125,
      "kl": 0.057405439857393506,
      "learning_rate": 3.050425614300487e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8570
    },
    {
      "completion_length": 70.15,
      "epoch": 1.716,
      "grad_norm": 0.000476837158203125,
      "kl": 0.01461967695504427,
      "learning_rate": 3.0087856783345916e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8580
    },
    {
      "completion_length": 48.075,
      "epoch": 1.718,
      "grad_norm": 0.000438690185546875,
      "kl": 0.0398553837556392,
      "learning_rate": 2.967413698375196e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8590
    },
    {
      "completion_length": 57.75,
      "epoch": 1.72,
      "grad_norm": 0.000518798828125,
      "kl": 0.02238648202328477,
      "learning_rate": 2.9263101785268253e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8600
    },
    {
      "completion_length": 66.25,
      "epoch": 1.722,
      "grad_norm": 0.0015716552734375,
      "kl": 0.04364732797257602,
      "learning_rate": 2.8854756196229017e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8610
    },
    {
      "completion_length": 43.7,
      "epoch": 1.724,
      "grad_norm": 0.0003299713134765625,
      "kl": 0.11783089116215706,
      "learning_rate": 2.844910519219632e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8620
    },
    {
      "completion_length": 49.45,
      "epoch": 1.726,
      "grad_norm": 0.000640869140625,
      "kl": 0.13984102117829025,
      "learning_rate": 2.8046153715899695e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8630
    },
    {
      "completion_length": 48.175,
      "epoch": 1.728,
      "grad_norm": 0.00054931640625,
      "kl": 0.40176068069413307,
      "learning_rate": 2.764590667717562e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8640
    },
    {
      "completion_length": 63.95,
      "epoch": 1.73,
      "grad_norm": 0.0004711151123046875,
      "kl": 0.8224213434383273,
      "learning_rate": 2.7248368952908055e-07,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8650
    },
    {
      "completion_length": 44.175,
      "epoch": 1.732,
      "grad_norm": 0.000644683837890625,
      "kl": 0.04362150589004159,
      "learning_rate": 2.6853545386968607e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8660
    },
    {
      "completion_length": 67.65,
      "epoch": 1.734,
      "grad_norm": 0.000659942626953125,
      "kl": 0.048378444463014605,
      "learning_rate": 2.6461440790157974e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8670
    },
    {
      "completion_length": 61.075,
      "epoch": 1.736,
      "grad_norm": 0.0022430419921875,
      "kl": 0.019372216332703827,
      "learning_rate": 2.6072059940146775e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8680
    },
    {
      "completion_length": 44.075,
      "epoch": 1.738,
      "grad_norm": 0.0004119873046875,
      "kl": 8.146503202756866,
      "learning_rate": 2.568540758141791e-07,
      "loss": 0.0008,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8690
    },
    {
      "completion_length": 53.575,
      "epoch": 1.74,
      "grad_norm": 0.0004138946533203125,
      "kl": 49.64364205431193,
      "learning_rate": 2.53014884252083e-07,
      "loss": 0.005,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 8700
    },
    {
      "completion_length": 65.325,
      "epoch": 1.742,
      "grad_norm": 0.000579833984375,
      "kl": 0.4298483125632629,
      "learning_rate": 2.492030714945162e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8710
    },
    {
      "completion_length": 52.625,
      "epoch": 1.744,
      "grad_norm": 0.0016326904296875,
      "kl": 0.07851723725907504,
      "learning_rate": 2.454186839872158e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8720
    },
    {
      "completion_length": 62.65,
      "epoch": 1.746,
      "grad_norm": 0.001373291015625,
      "kl": 23.3441758136265,
      "learning_rate": 2.4166176784174795e-07,
      "loss": 0.0023,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8730
    },
    {
      "completion_length": 74.575,
      "epoch": 1.748,
      "grad_norm": 0.000637054443359375,
      "kl": 11.16068452913314,
      "learning_rate": 2.3793236883495164e-07,
      "loss": 0.0011,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8740
    },
    {
      "completion_length": 49.475,
      "epoch": 1.75,
      "grad_norm": 0.000820159912109375,
      "kl": 0.075881730299443,
      "learning_rate": 2.3423053240837518e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8750
    },
    {
      "completion_length": 56.025,
      "epoch": 1.752,
      "grad_norm": 0.000476837158203125,
      "kl": 0.01870635347440839,
      "learning_rate": 2.3055630366772857e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8760
    },
    {
      "completion_length": 47.775,
      "epoch": 1.754,
      "grad_norm": 0.000675201416015625,
      "kl": 0.07872601179406047,
      "learning_rate": 2.269097273823287e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8770
    },
    {
      "completion_length": 55.075,
      "epoch": 1.756,
      "grad_norm": 0.002685546875,
      "kl": 2.795431226864457,
      "learning_rate": 2.2329084798455747e-07,
      "loss": 0.0003,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8780
    },
    {
      "completion_length": 57.725,
      "epoch": 1.758,
      "grad_norm": 0.0003795623779296875,
      "kl": 0.03155038901604712,
      "learning_rate": 2.1969970956931762e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8790
    },
    {
      "completion_length": 56.35,
      "epoch": 1.76,
      "grad_norm": 0.000591278076171875,
      "kl": 48.86689073387534,
      "learning_rate": 2.1613635589349756e-07,
      "loss": 0.0049,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8800
    },
    {
      "completion_length": 52.05,
      "epoch": 1.762,
      "grad_norm": 0.00151824951171875,
      "kl": 0.05452495804056525,
      "learning_rate": 2.1260083037543817e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8810
    },
    {
      "completion_length": 47.3,
      "epoch": 1.764,
      "grad_norm": 0.0004444122314453125,
      "kl": 312.406746559497,
      "learning_rate": 2.0909317609440093e-07,
      "loss": 0.0312,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8820
    },
    {
      "completion_length": 52.475,
      "epoch": 1.766,
      "grad_norm": 0.0005950927734375,
      "kl": 0.19929129825904965,
      "learning_rate": 2.0561343579004716e-07,
      "loss": 0.0,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 8830
    },
    {
      "completion_length": 74.525,
      "epoch": 1.768,
      "grad_norm": 0.0003871917724609375,
      "kl": 0.022377661243081094,
      "learning_rate": 2.0216165186191406e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8840
    },
    {
      "completion_length": 47.8,
      "epoch": 1.77,
      "grad_norm": 0.00055694580078125,
      "kl": 0.018075392534956335,
      "learning_rate": 1.9873786636889908e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8850
    },
    {
      "completion_length": 66.875,
      "epoch": 1.772,
      "grad_norm": 0.00037384033203125,
      "kl": 0.0537069259211421,
      "learning_rate": 1.95342121028749e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8860
    },
    {
      "completion_length": 50.35,
      "epoch": 1.774,
      "grad_norm": 0.000408172607421875,
      "kl": 0.16273712795227765,
      "learning_rate": 1.9197445721754777e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8870
    },
    {
      "completion_length": 41.1,
      "epoch": 1.776,
      "grad_norm": 0.00162506103515625,
      "kl": 0.0835498913191259,
      "learning_rate": 1.8863491596921745e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8880
    },
    {
      "completion_length": 56.375,
      "epoch": 1.778,
      "grad_norm": 0.0021514892578125,
      "kl": 3.5430075244046746,
      "learning_rate": 1.8532353797501318e-07,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8890
    },
    {
      "completion_length": 49.0,
      "epoch": 1.78,
      "grad_norm": 0.00153350830078125,
      "kl": 0.09430858921259641,
      "learning_rate": 1.8204036358303173e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8900
    },
    {
      "completion_length": 56.175,
      "epoch": 1.782,
      "grad_norm": 0.00121307373046875,
      "kl": 0.04068310302682221,
      "learning_rate": 1.787854327977162e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8910
    },
    {
      "completion_length": 59.075,
      "epoch": 1.784,
      "grad_norm": 0.0003814697265625,
      "kl": 0.04385726461187005,
      "learning_rate": 1.7555878527937164e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8920
    },
    {
      "completion_length": 55.05,
      "epoch": 1.786,
      "grad_norm": 0.0030517578125,
      "kl": 0.018657160410657524,
      "learning_rate": 1.7236046034367959e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8930
    },
    {
      "completion_length": 47.975,
      "epoch": 1.788,
      "grad_norm": 0.0004711151123046875,
      "kl": 1.90866837259382,
      "learning_rate": 1.6919049696121957e-07,
      "loss": 0.0002,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8940
    },
    {
      "completion_length": 61.925,
      "epoch": 1.79,
      "grad_norm": 0.0030517578125,
      "kl": 0.7081083978526295,
      "learning_rate": 1.6604893375699594e-07,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8950
    },
    {
      "completion_length": 59.725,
      "epoch": 1.792,
      "grad_norm": 0.0004444122314453125,
      "kl": 4.325691572204232,
      "learning_rate": 1.629358090099639e-07,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8960
    },
    {
      "completion_length": 54.6,
      "epoch": 1.794,
      "grad_norm": 0.0009918212890625,
      "kl": 0.11546620442532003,
      "learning_rate": 1.5985116065256683e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8970
    },
    {
      "completion_length": 58.7,
      "epoch": 1.796,
      "grad_norm": 0.000591278076171875,
      "kl": 1.2548286508535966,
      "learning_rate": 1.567950262702714e-07,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 8980
    },
    {
      "completion_length": 56.925,
      "epoch": 1.798,
      "grad_norm": 0.0003376007080078125,
      "kl": 0.07534236300271005,
      "learning_rate": 1.5376744310111019e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 8990
    },
    {
      "completion_length": 59.7,
      "epoch": 1.8,
      "grad_norm": 0.000576019287109375,
      "kl": 79.93720495556481,
      "learning_rate": 1.507684480352292e-07,
      "loss": 0.008,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 9000
    },
    {
      "completion_length": 47.55,
      "epoch": 1.802,
      "grad_norm": 0.000606536865234375,
      "kl": 0.07122775209136308,
      "learning_rate": 1.4779807761443638e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9010
    },
    {
      "completion_length": 56.475,
      "epoch": 1.804,
      "grad_norm": 0.0019378662109375,
      "kl": 31.246724256686868,
      "learning_rate": 1.4485636803175828e-07,
      "loss": 0.0031,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9020
    },
    {
      "completion_length": 56.8,
      "epoch": 1.806,
      "grad_norm": 0.0007476806640625,
      "kl": 0.01624395214021206,
      "learning_rate": 1.419433551309976e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9030
    },
    {
      "completion_length": 52.925,
      "epoch": 1.808,
      "grad_norm": 0.00052642822265625,
      "kl": 0.03512433131691069,
      "learning_rate": 1.3905907440629752e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9040
    },
    {
      "completion_length": 63.475,
      "epoch": 1.81,
      "grad_norm": 0.0008087158203125,
      "kl": 0.05812466649804264,
      "learning_rate": 1.362035610017079e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9050
    },
    {
      "completion_length": 67.2,
      "epoch": 1.812,
      "grad_norm": 0.0002841949462890625,
      "kl": 0.053207884868606926,
      "learning_rate": 1.3337684971075932e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9060
    },
    {
      "completion_length": 63.025,
      "epoch": 1.814,
      "grad_norm": 0.00083160400390625,
      "kl": 0.017276625451631843,
      "learning_rate": 1.305789749760361e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9070
    },
    {
      "completion_length": 53.85,
      "epoch": 1.8159999999999998,
      "grad_norm": 0.0014495849609375,
      "kl": 0.1759139670059085,
      "learning_rate": 1.278099708887587e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9080
    },
    {
      "completion_length": 49.575,
      "epoch": 1.818,
      "grad_norm": 0.00054168701171875,
      "kl": 0.06552611859515309,
      "learning_rate": 1.2506987118836912e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9090
    },
    {
      "completion_length": 57.775,
      "epoch": 1.8199999999999998,
      "grad_norm": 0.0005035400390625,
      "kl": 0.14279152313247323,
      "learning_rate": 1.223587092621162e-07,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9100
    },
    {
      "completion_length": 66.725,
      "epoch": 1.822,
      "grad_norm": 0.000827789306640625,
      "kl": 26.51692173536867,
      "learning_rate": 1.1967651814465353e-07,
      "loss": 0.0027,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9110
    },
    {
      "completion_length": 45.675,
      "epoch": 1.8239999999999998,
      "grad_norm": 24.0,
      "kl": 21.41680323826149,
      "learning_rate": 1.1702333051763271e-07,
      "loss": 0.0021,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 9120
    },
    {
      "completion_length": 50.95,
      "epoch": 1.826,
      "grad_norm": 0.0005035400390625,
      "kl": 0.031037054676562547,
      "learning_rate": 1.1439917870930795e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9130
    },
    {
      "completion_length": 49.925,
      "epoch": 1.8279999999999998,
      "grad_norm": 0.000782012939453125,
      "kl": 0.06788429841399193,
      "learning_rate": 1.1180409469414094e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9140
    },
    {
      "completion_length": 54.575,
      "epoch": 1.83,
      "grad_norm": 0.00067901611328125,
      "kl": 0.02977508623152971,
      "learning_rate": 1.0923811009241142e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9150
    },
    {
      "completion_length": 62.55,
      "epoch": 1.8319999999999999,
      "grad_norm": 0.005157470703125,
      "kl": 0.03778183825779706,
      "learning_rate": 1.067012561698319e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9160
    },
    {
      "completion_length": 55.6,
      "epoch": 1.834,
      "grad_norm": 0.0003528594970703125,
      "kl": 0.033238646434620024,
      "learning_rate": 1.041935638371669e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9170
    },
    {
      "completion_length": 60.35,
      "epoch": 1.8359999999999999,
      "grad_norm": 0.00052642822265625,
      "kl": 0.02701822677627206,
      "learning_rate": 1.0171506364985622e-07,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9180
    },
    {
      "completion_length": 53.775,
      "epoch": 1.838,
      "grad_norm": 0.0002994537353515625,
      "kl": 1243.2045701113996,
      "learning_rate": 9.926578580764234e-08,
      "loss": 0.1243,
      "match_ratio": 0.9,
      "reward": -0.1,
      "reward_std": 0.1154700517654419,
      "rewards/reward_func": -0.1,
      "step": 9190
    },
    {
      "completion_length": 76.5,
      "epoch": 1.8399999999999999,
      "grad_norm": 0.000469207763671875,
      "kl": 0.017305072862654924,
      "learning_rate": 9.684576015420277e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9200
    },
    {
      "completion_length": 50.85,
      "epoch": 1.842,
      "grad_norm": 0.00045013427734375,
      "kl": 13.52835137634538,
      "learning_rate": 9.445501617678654e-08,
      "loss": 0.0014,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9210
    },
    {
      "completion_length": 55.425,
      "epoch": 1.8439999999999999,
      "grad_norm": 0.00072479248046875,
      "kl": 0.03520208708941937,
      "learning_rate": 9.209358300585474e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9220
    },
    {
      "completion_length": 66.8,
      "epoch": 1.846,
      "grad_norm": 0.000614166259765625,
      "kl": 0.02983384854160249,
      "learning_rate": 8.9761489414725e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9230
    },
    {
      "completion_length": 49.075,
      "epoch": 1.8479999999999999,
      "grad_norm": 0.000537872314453125,
      "kl": 0.032278594188392164,
      "learning_rate": 8.745876381922147e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9240
    },
    {
      "completion_length": 60.0,
      "epoch": 1.85,
      "grad_norm": 0.000659942626953125,
      "kl": 0.02954811817035079,
      "learning_rate": 8.518543427732951e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9250
    },
    {
      "completion_length": 58.05,
      "epoch": 1.8519999999999999,
      "grad_norm": 0.000522613525390625,
      "kl": 0.020372640853747726,
      "learning_rate": 8.294152848885156e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9260
    },
    {
      "completion_length": 57.65,
      "epoch": 1.854,
      "grad_norm": 0.000514984130859375,
      "kl": 48.13295641997829,
      "learning_rate": 8.072707379507217e-08,
      "loss": 0.0048,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 9270
    },
    {
      "completion_length": 73.275,
      "epoch": 1.8559999999999999,
      "grad_norm": 0.000583648681640625,
      "kl": 259.27141086012125,
      "learning_rate": 7.854209717842231e-08,
      "loss": 0.0259,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 9280
    },
    {
      "completion_length": 62.75,
      "epoch": 1.858,
      "grad_norm": 0.0002765655517578125,
      "kl": 0.0620627264957875,
      "learning_rate": 7.638662526215284e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9290
    },
    {
      "completion_length": 60.6,
      "epoch": 1.8599999999999999,
      "grad_norm": 0.000881195068359375,
      "kl": 0.0414402786642313,
      "learning_rate": 7.426068431000883e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9300
    },
    {
      "completion_length": 65.575,
      "epoch": 1.862,
      "grad_norm": 0.00058746337890625,
      "kl": 0.08443178189918399,
      "learning_rate": 7.216430022591009e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9310
    },
    {
      "completion_length": 64.35,
      "epoch": 1.8639999999999999,
      "grad_norm": 0.00634765625,
      "kl": 0.12132438533008098,
      "learning_rate": 7.009749855363457e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9320
    },
    {
      "completion_length": 59.775,
      "epoch": 1.866,
      "grad_norm": 0.000392913818359375,
      "kl": 7.24802761040628,
      "learning_rate": 6.806030447650879e-08,
      "loss": 0.0007,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.05,
      "rewards/reward_func": -0.075,
      "step": 9330
    },
    {
      "completion_length": 51.325,
      "epoch": 1.8679999999999999,
      "grad_norm": 0.000522613525390625,
      "kl": 12.393874236382544,
      "learning_rate": 6.605274281709929e-08,
      "loss": 0.0012,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9340
    },
    {
      "completion_length": 54.575,
      "epoch": 1.87,
      "grad_norm": 0.000598907470703125,
      "kl": 0.20043480526655913,
      "learning_rate": 6.407483803691216e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9350
    },
    {
      "completion_length": 65.05,
      "epoch": 1.8719999999999999,
      "grad_norm": 0.000621795654296875,
      "kl": 0.038857326842844486,
      "learning_rate": 6.212661423609184e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9360
    },
    {
      "completion_length": 59.3,
      "epoch": 1.874,
      "grad_norm": 0.000453948974609375,
      "kl": 0.1275158784352243,
      "learning_rate": 6.020809515313141e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9370
    },
    {
      "completion_length": 55.175,
      "epoch": 1.876,
      "grad_norm": 0.001220703125,
      "kl": 0.6000383426435292,
      "learning_rate": 5.83193041645802e-08,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9380
    },
    {
      "completion_length": 57.375,
      "epoch": 1.8780000000000001,
      "grad_norm": 0.00177001953125,
      "kl": 0.06478001358918846,
      "learning_rate": 5.6460264284760316e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9390
    },
    {
      "completion_length": 51.175,
      "epoch": 1.88,
      "grad_norm": 0.006195068359375,
      "kl": 0.08602785079274326,
      "learning_rate": 5.463099816548578e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9400
    },
    {
      "completion_length": 57.75,
      "epoch": 1.8820000000000001,
      "grad_norm": 0.00092315673828125,
      "kl": 0.028340872889384628,
      "learning_rate": 5.283152809578751e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9410
    },
    {
      "completion_length": 61.275,
      "epoch": 1.884,
      "grad_norm": 0.0004024505615234375,
      "kl": 0.089741973252967,
      "learning_rate": 5.106187600163987e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9420
    },
    {
      "completion_length": 65.625,
      "epoch": 1.8860000000000001,
      "grad_norm": 0.0025634765625,
      "kl": 0.060642439499497415,
      "learning_rate": 4.932206344569562e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9430
    },
    {
      "completion_length": 58.875,
      "epoch": 1.888,
      "grad_norm": 0.00067901611328125,
      "kl": 0.06356988861225546,
      "learning_rate": 4.761211162702117e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9440
    },
    {
      "completion_length": 57.25,
      "epoch": 1.8900000000000001,
      "grad_norm": 0.00072479248046875,
      "kl": 16.864195838803425,
      "learning_rate": 4.593204138084006e-08,
      "loss": 0.0017,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.1,
      "rewards/reward_func": -0.05,
      "step": 9450
    },
    {
      "completion_length": 51.025,
      "epoch": 1.892,
      "grad_norm": 0.00037384033203125,
      "kl": 0.047673306241631505,
      "learning_rate": 4.428187317827848e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9460
    },
    {
      "completion_length": 66.45,
      "epoch": 1.8940000000000001,
      "grad_norm": 0.000469207763671875,
      "kl": 0.035626521334052086,
      "learning_rate": 4.26616271261146e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9470
    },
    {
      "completion_length": 47.25,
      "epoch": 1.896,
      "grad_norm": 0.00045013427734375,
      "kl": 0.09364478723146022,
      "learning_rate": 4.1071322966535487e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9480
    },
    {
      "completion_length": 61.45,
      "epoch": 1.8980000000000001,
      "grad_norm": 0.0004100799560546875,
      "kl": 0.02402509720996022,
      "learning_rate": 3.95109800768953e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9490
    },
    {
      "completion_length": 52.2,
      "epoch": 1.9,
      "grad_norm": 0.00049591064453125,
      "kl": 0.08021967611275613,
      "learning_rate": 3.798061746947995e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9500
    },
    {
      "completion_length": 54.175,
      "epoch": 1.9020000000000001,
      "grad_norm": 0.0003814697265625,
      "kl": 0.08938063569366932,
      "learning_rate": 3.648025379127479e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9510
    },
    {
      "completion_length": 48.6,
      "epoch": 1.904,
      "grad_norm": 0.000553131103515625,
      "kl": 0.03359618247486651,
      "learning_rate": 3.5009907323737826e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9520
    },
    {
      "completion_length": 49.65,
      "epoch": 1.9060000000000001,
      "grad_norm": 0.00131988525390625,
      "kl": 0.10454095806926489,
      "learning_rate": 3.3569595982576584e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9530
    },
    {
      "completion_length": 42.4,
      "epoch": 1.908,
      "grad_norm": 0.000667572021484375,
      "kl": 0.18224592534825207,
      "learning_rate": 3.2159337317530234e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9540
    },
    {
      "completion_length": 51.125,
      "epoch": 1.9100000000000001,
      "grad_norm": 0.0009613037109375,
      "kl": 0.2186179363168776,
      "learning_rate": 3.077914851215585e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9550
    },
    {
      "completion_length": 60.75,
      "epoch": 1.912,
      "grad_norm": 0.000701904296875,
      "kl": 0.090417854860425,
      "learning_rate": 2.9429046383618042e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9560
    },
    {
      "completion_length": 41.175,
      "epoch": 1.9140000000000001,
      "grad_norm": 0.00130462646484375,
      "kl": 0.07170910434797406,
      "learning_rate": 2.810904738248549e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9570
    },
    {
      "completion_length": 43.875,
      "epoch": 1.916,
      "grad_norm": 0.0018310546875,
      "kl": 0.12651289403438568,
      "learning_rate": 2.681916759252917e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9580
    },
    {
      "completion_length": 67.95,
      "epoch": 1.9180000000000001,
      "grad_norm": 0.0009613037109375,
      "kl": 0.046817721845582125,
      "learning_rate": 2.555942273052753e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9590
    },
    {
      "completion_length": 44.825,
      "epoch": 1.92,
      "grad_norm": 0.002197265625,
      "kl": 1.0959480846766383,
      "learning_rate": 2.4329828146074096e-08,
      "loss": 0.0001,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9600
    },
    {
      "completion_length": 40.15,
      "epoch": 1.9220000000000002,
      "grad_norm": 0.0015869140625,
      "kl": 0.05991814769804478,
      "learning_rate": 2.313039882139101e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9610
    },
    {
      "completion_length": 56.9,
      "epoch": 1.924,
      "grad_norm": 0.000377655029296875,
      "kl": 0.019165601092390717,
      "learning_rate": 2.1961149371145795e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9620
    },
    {
      "completion_length": 61.1,
      "epoch": 1.9260000000000002,
      "grad_norm": 0.000263214111328125,
      "kl": 0.05205519350711256,
      "learning_rate": 2.082209404227403e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9630
    },
    {
      "completion_length": 46.925,
      "epoch": 1.928,
      "grad_norm": 0.0004367828369140625,
      "kl": 0.08729816749691963,
      "learning_rate": 1.9713246713805588e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9640
    },
    {
      "completion_length": 60.35,
      "epoch": 1.9300000000000002,
      "grad_norm": 0.0017852783203125,
      "kl": 0.09572115261107683,
      "learning_rate": 1.8634620896695044e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9650
    },
    {
      "completion_length": 62.975,
      "epoch": 1.932,
      "grad_norm": 10.0625,
      "kl": 0.08949833824299276,
      "learning_rate": 1.7586229733657646e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9660
    },
    {
      "completion_length": 60.05,
      "epoch": 1.9340000000000002,
      "grad_norm": 0.0003566741943359375,
      "kl": 0.051867073588073256,
      "learning_rate": 1.6568085999008886e-08,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9670
    },
    {
      "completion_length": 56.425,
      "epoch": 1.936,
      "grad_norm": 0.0006866455078125,
      "kl": 0.05798132345080376,
      "learning_rate": 1.5580202098509078e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9680
    },
    {
      "completion_length": 60.575,
      "epoch": 1.938,
      "grad_norm": 29.75,
      "kl": 22.995475397538392,
      "learning_rate": 1.4622590069211517e-08,
      "loss": 0.0023,
      "match_ratio": 0.95,
      "reward": -0.05,
      "reward_std": 0.05773502588272095,
      "rewards/reward_func": -0.05,
      "step": 9690
    },
    {
      "completion_length": 52.575,
      "epoch": 1.94,
      "grad_norm": 0.00121307373046875,
      "kl": 0.05500190043821931,
      "learning_rate": 1.3695261579316776e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9700
    },
    {
      "completion_length": 51.8,
      "epoch": 1.942,
      "grad_norm": 0.000308990478515625,
      "kl": 0.07781615569256246,
      "learning_rate": 1.2798227928029483e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9710
    },
    {
      "completion_length": 56.15,
      "epoch": 1.944,
      "grad_norm": 0.0003795623779296875,
      "kl": 0.08795451316982508,
      "learning_rate": 1.193150004542204e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9720
    },
    {
      "completion_length": 52.4,
      "epoch": 1.946,
      "grad_norm": 0.002899169921875,
      "kl": 0.047139992006123066,
      "learning_rate": 1.109508849230001e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9730
    },
    {
      "completion_length": 45.45,
      "epoch": 1.948,
      "grad_norm": 0.0003662109375,
      "kl": 4.14183980775997,
      "learning_rate": 1.0289003460074165e-08,
      "loss": 0.0004,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9740
    },
    {
      "completion_length": 56.375,
      "epoch": 1.95,
      "grad_norm": 0.0031890869140625,
      "kl": 0.060475172754377124,
      "learning_rate": 9.513254770636138e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9750
    },
    {
      "completion_length": 48.85,
      "epoch": 1.952,
      "grad_norm": 0.0008087158203125,
      "kl": 0.01811651182360947,
      "learning_rate": 8.767851876239075e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9760
    },
    {
      "completion_length": 63.625,
      "epoch": 1.954,
      "grad_norm": 0.000522613525390625,
      "kl": 0.04865064946934581,
      "learning_rate": 8.052803859382174e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9770
    },
    {
      "completion_length": 66.825,
      "epoch": 1.956,
      "grad_norm": 0.0003662109375,
      "kl": 0.017140331957489253,
      "learning_rate": 7.368119432699383e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9780
    },
    {
      "completion_length": 49.275,
      "epoch": 1.958,
      "grad_norm": 0.000598907470703125,
      "kl": 0.05813699197024107,
      "learning_rate": 6.7138069388547614e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9790
    },
    {
      "completion_length": 53.425,
      "epoch": 1.96,
      "grad_norm": 0.00069427490234375,
      "kl": 1490.5032024047337,
      "learning_rate": 6.089874350439507e-09,
      "loss": 0.1491,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 9800
    },
    {
      "completion_length": 43.9,
      "epoch": 1.962,
      "grad_norm": 0.0005645751953125,
      "kl": 31.488269805023446,
      "learning_rate": 5.4963292698750896e-09,
      "loss": 0.0031,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9810
    },
    {
      "completion_length": 54.95,
      "epoch": 1.964,
      "grad_norm": 0.0184326171875,
      "kl": 0.13211959092877806,
      "learning_rate": 4.933178929321103e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9820
    },
    {
      "completion_length": 48.75,
      "epoch": 1.966,
      "grad_norm": 0.000850677490234375,
      "kl": 0.10980427814647556,
      "learning_rate": 4.400430190586724e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9830
    },
    {
      "completion_length": 57.525,
      "epoch": 1.968,
      "grad_norm": 14.625,
      "kl": 0.031209711637347936,
      "learning_rate": 3.8980895450474455e-09,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9840
    },
    {
      "completion_length": 49.9,
      "epoch": 1.97,
      "grad_norm": 0.000507354736328125,
      "kl": 16.15725321341306,
      "learning_rate": 3.4261631135654174e-09,
      "loss": 0.0016,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9850
    },
    {
      "completion_length": 54.075,
      "epoch": 1.972,
      "grad_norm": 0.001739501953125,
      "kl": 0.04826322416774929,
      "learning_rate": 2.984656646415063e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9860
    },
    {
      "completion_length": 66.375,
      "epoch": 1.974,
      "grad_norm": 0.000919342041015625,
      "kl": 0.035056399274617435,
      "learning_rate": 2.573575523213412e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9870
    },
    {
      "completion_length": 72.6,
      "epoch": 1.976,
      "grad_norm": 0.0007781982421875,
      "kl": 0.05609772065654397,
      "learning_rate": 2.192924752854042e-09,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9880
    },
    {
      "completion_length": 72.45,
      "epoch": 1.978,
      "grad_norm": 0.0004405975341796875,
      "kl": 0.26721446458250286,
      "learning_rate": 1.842708973447127e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9890
    },
    {
      "completion_length": 62.25,
      "epoch": 1.98,
      "grad_norm": 0.0003643035888671875,
      "kl": 0.08341183541342616,
      "learning_rate": 1.5229324522605949e-09,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9900
    },
    {
      "completion_length": 58.65,
      "epoch": 1.982,
      "grad_norm": 0.00063323974609375,
      "kl": 2.203670488623902,
      "learning_rate": 1.2335990856710001e-09,
      "loss": 0.0002,
      "match_ratio": 0.925,
      "reward": -0.075,
      "reward_std": 0.10773502588272095,
      "rewards/reward_func": -0.075,
      "step": 9910
    },
    {
      "completion_length": 45.05,
      "epoch": 1.984,
      "grad_norm": 0.00061798095703125,
      "kl": 0.06615068479441107,
      "learning_rate": 9.747123991141193e-10,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9920
    },
    {
      "completion_length": 57.625,
      "epoch": 1.986,
      "grad_norm": 0.000545501708984375,
      "kl": 0.0431473188335076,
      "learning_rate": 7.462755470422078e-10,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9930
    },
    {
      "completion_length": 65.05,
      "epoch": 1.988,
      "grad_norm": 0.0004634857177734375,
      "kl": 0.049975822074338795,
      "learning_rate": 5.48291312886251e-10,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9940
    },
    {
      "completion_length": 67.95,
      "epoch": 1.99,
      "grad_norm": 0.00127410888671875,
      "kl": 0.033399745682254435,
      "learning_rate": 3.8076210902182607e-10,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9950
    },
    {
      "completion_length": 56.175,
      "epoch": 1.992,
      "grad_norm": 0.0004825592041015625,
      "kl": 0.11492122933268548,
      "learning_rate": 2.43689976739403e-10,
      "loss": 0.0,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9960
    },
    {
      "completion_length": 58.85,
      "epoch": 1.994,
      "grad_norm": 0.0040283203125,
      "kl": 0.04082223805598915,
      "learning_rate": 1.3707658621964216e-10,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9970
    },
    {
      "completion_length": 64.95,
      "epoch": 1.996,
      "grad_norm": 0.00299072265625,
      "kl": 0.055863088183104995,
      "learning_rate": 6.092323651313293e-11,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 9980
    },
    {
      "completion_length": 64.9,
      "epoch": 1.998,
      "grad_norm": 0.00091552734375,
      "kl": 40.98459475683048,
      "learning_rate": 1.5230855524017708e-11,
      "loss": 0.0041,
      "match_ratio": 0.975,
      "reward": -0.025,
      "reward_std": 0.05,
      "rewards/reward_func": -0.025,
      "step": 9990
    },
    {
      "completion_length": 65.95,
      "epoch": 2.0,
      "grad_norm": 0.0003757476806640625,
      "kl": 0.02109892386943102,
      "learning_rate": 0.0,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/reward_func": 0.0,
      "step": 10000
    }
  ],
  "logging_steps": 10,
  "max_steps": 10000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}