{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.746227709190672,
  "eval_steps": 500,
  "global_step": 1000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 85.09375,
      "epoch": 0.0027434842249657062,
      "grad_norm": 22.875594605913278,
      "kl": 0.0,
      "learning_rate": 9.990909090909091e-07,
      "loss": 0.0,
      "reward": 0.5414062440395355,
      "reward_std": 0.4481600522994995,
      "rewards/choice_reward": 0.28125,
      "rewards/roam_reward": 0.2601562440395355,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.328125,
      "epoch": 0.0054869684499314125,
      "grad_norm": 23.368213544115484,
      "kl": 0.0011501312255859375,
      "learning_rate": 9.98181818181818e-07,
      "loss": 0.0,
      "reward": 0.7562499940395355,
      "reward_std": 0.7361946105957031,
      "rewards/choice_reward": 0.40625,
      "rewards/roam_reward": 0.34999997913837433,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.5625,
      "epoch": 0.00823045267489712,
      "grad_norm": 6.439502461054405,
      "kl": 0.001247406005859375,
      "learning_rate": 9.972727272727273e-07,
      "loss": 0.0,
      "reward": 0.7953124940395355,
      "reward_std": 0.8322779834270477,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.3421874940395355,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.21875,
      "epoch": 0.010973936899862825,
      "grad_norm": 5.055634258308147,
      "kl": 0.002346038818359375,
      "learning_rate": 9.963636363636362e-07,
      "loss": 0.0001,
      "reward": 0.813281238079071,
      "reward_std": 0.6915583610534668,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.36015625298023224,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.046875,
      "epoch": 0.013717421124828532,
      "grad_norm": 5.4128284043265396,
      "kl": 0.002582550048828125,
      "learning_rate": 9.954545454545454e-07,
      "loss": 0.0001,
      "reward": 0.571093738079071,
      "reward_std": 0.6553446054458618,
      "rewards/choice_reward": 0.328125,
      "rewards/roam_reward": 0.24296873062849045,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.9375,
      "epoch": 0.01646090534979424,
      "grad_norm": 4.376063841540647,
      "kl": 0.00384521484375,
      "learning_rate": 9.945454545454544e-07,
      "loss": 0.0002,
      "reward": 0.8460937142372131,
      "reward_std": 0.6018270701169968,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.3773437440395355,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.390625,
      "epoch": 0.019204389574759947,
      "grad_norm": 7.101229751865706,
      "kl": 0.01727294921875,
      "learning_rate": 9.936363636363636e-07,
      "loss": 0.0007,
      "reward": 1.126562476158142,
      "reward_std": 0.7818476557731628,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.5015625208616257,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.78125,
      "epoch": 0.02194787379972565,
      "grad_norm": 6.022296712805634,
      "kl": 0.0061187744140625,
      "learning_rate": 9.927272727272727e-07,
      "loss": 0.0002,
      "reward": 1.0984374284744263,
      "reward_std": 0.8049026429653168,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.48906251788139343,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.3125,
      "epoch": 0.024691358024691357,
      "grad_norm": 4.245932293968646,
      "kl": 0.01190185546875,
      "learning_rate": 9.918181818181817e-07,
      "loss": 0.0005,
      "reward": 0.6945312321186066,
      "reward_std": 0.3619903400540352,
      "rewards/choice_reward": 0.359375,
      "rewards/roam_reward": 0.33515624701976776,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.484375,
      "epoch": 0.027434842249657063,
      "grad_norm": 8.484504056440915,
      "kl": 0.0076446533203125,
      "learning_rate": 9.909090909090909e-07,
      "loss": 0.0003,
      "reward": 0.90625,
      "reward_std": 0.45835646986961365,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.3906249850988388,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.03125,
      "epoch": 0.03017832647462277,
      "grad_norm": 7.742548302467443,
      "kl": 0.0139007568359375,
      "learning_rate": 9.9e-07,
      "loss": 0.0006,
      "reward": 0.8593749701976776,
      "reward_std": 0.43932054936885834,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.390625,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.84375,
      "epoch": 0.03292181069958848,
      "grad_norm": 3.983053793753651,
      "kl": 0.011077880859375,
      "learning_rate": 9.89090909090909e-07,
      "loss": 0.0004,
      "reward": 1.0476561784744263,
      "reward_std": 0.789838433265686,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.46953126788139343,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.25,
      "epoch": 0.03566529492455418,
      "grad_norm": 5.130214531045759,
      "kl": 0.017059326171875,
      "learning_rate": 9.881818181818182e-07,
      "loss": 0.0007,
      "reward": 0.9124999940395355,
      "reward_std": 0.6435409486293793,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.3656250089406967,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.859375,
      "epoch": 0.038408779149519894,
      "grad_norm": 6.98501961751934,
      "kl": 0.01129150390625,
      "learning_rate": 9.872727272727272e-07,
      "loss": 0.0005,
      "reward": 0.9640624523162842,
      "reward_std": 0.645575761795044,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.41718751192092896,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.046875,
      "epoch": 0.0411522633744856,
      "grad_norm": 7.147359479240712,
      "kl": 0.0162200927734375,
      "learning_rate": 9.863636363636363e-07,
      "loss": 0.0006,
      "reward": 1.0695312023162842,
      "reward_std": 0.820113867521286,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.46015626192092896,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.859375,
      "epoch": 0.0438957475994513,
      "grad_norm": 5.664219699067135,
      "kl": 0.015106201171875,
      "learning_rate": 9.854545454545455e-07,
      "loss": 0.0006,
      "reward": 1.0304687321186066,
      "reward_std": 0.6156916320323944,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.43671874701976776,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.296875,
      "epoch": 0.04663923182441701,
      "grad_norm": 4.598762294986751,
      "kl": 0.028656005859375,
      "learning_rate": 9.845454545454545e-07,
      "loss": 0.0011,
      "reward": 1.1671874523162842,
      "reward_std": 0.717985987663269,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.49531251192092896,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.125,
      "epoch": 0.04938271604938271,
      "grad_norm": 14.244035690799146,
      "kl": 0.016326904296875,
      "learning_rate": 9.836363636363637e-07,
      "loss": 0.0007,
      "reward": 0.9195311665534973,
      "reward_std": 0.6659009456634521,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.3882812261581421,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.71875,
      "epoch": 0.05212620027434842,
      "grad_norm": 10.540110657265467,
      "kl": 0.1883544921875,
      "learning_rate": 9.827272727272728e-07,
      "loss": 0.0075,
      "reward": 0.7343749701976776,
      "reward_std": 0.4525277465581894,
      "rewards/choice_reward": 0.4375,
      "rewards/roam_reward": 0.2968749701976776,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.5,
      "epoch": 0.05486968449931413,
      "grad_norm": 7.327457221273048,
      "kl": 0.0169677734375,
      "learning_rate": 9.818181818181818e-07,
      "loss": 0.0007,
      "reward": 0.9648436903953552,
      "reward_std": 0.7674175798892975,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.4335937350988388,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.875,
      "epoch": 0.05761316872427984,
      "grad_norm": 22.457323451419064,
      "kl": 0.02655029296875,
      "learning_rate": 9.80909090909091e-07,
      "loss": 0.0011,
      "reward": 0.9695312082767487,
      "reward_std": 0.8863083720207214,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.43828125298023224,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.375,
      "epoch": 0.06035665294924554,
      "grad_norm": 5.7571881626029295,
      "kl": 0.0379638671875,
      "learning_rate": 9.8e-07,
      "loss": 0.0015,
      "reward": 1.309374988079071,
      "reward_std": 0.54523104429245,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.543749988079071,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.859375,
      "epoch": 0.06310013717421124,
      "grad_norm": 11.09390596331488,
      "kl": 0.0340576171875,
      "learning_rate": 9.790909090909091e-07,
      "loss": 0.0014,
      "reward": 0.9507811963558197,
      "reward_std": 0.604695588350296,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.4195312559604645,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.796875,
      "epoch": 0.06584362139917696,
      "grad_norm": 31.58108805295004,
      "kl": 0.0621337890625,
      "learning_rate": 9.78181818181818e-07,
      "loss": 0.0025,
      "reward": 1.024999976158142,
      "reward_std": 0.6390542387962341,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.4625000059604645,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.46875,
      "epoch": 0.06858710562414266,
      "grad_norm": 11.336338033820835,
      "kl": 0.034423828125,
      "learning_rate": 9.772727272727273e-07,
      "loss": 0.0014,
      "reward": 0.6968749463558197,
      "reward_std": 0.5415091067552567,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.3062499910593033,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.71875,
      "epoch": 0.07133058984910837,
      "grad_norm": 8.12783631250481,
      "kl": 0.042724609375,
      "learning_rate": 9.763636363636362e-07,
      "loss": 0.0017,
      "reward": 0.9734374582767487,
      "reward_std": 0.7181236445903778,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.42656248807907104,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.8125,
      "epoch": 0.07407407407407407,
      "grad_norm": 4.5322020786066854,
      "kl": 0.03717041015625,
      "learning_rate": 9.754545454545454e-07,
      "loss": 0.0015,
      "reward": 0.8960937261581421,
      "reward_std": 0.6352558135986328,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.3960937261581421,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.875,
      "epoch": 0.07681755829903979,
      "grad_norm": 6.2617401945392,
      "kl": 0.0208740234375,
      "learning_rate": 9.745454545454544e-07,
      "loss": 0.0008,
      "reward": 1.333593726158142,
      "reward_std": 0.668065220117569,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.5523437559604645,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.734375,
      "epoch": 0.07956104252400549,
      "grad_norm": 4.8442832275116245,
      "kl": 0.042236328125,
      "learning_rate": 9.736363636363635e-07,
      "loss": 0.0017,
      "reward": 1.3671874403953552,
      "reward_std": 0.7418243885040283,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.6015625,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.75,
      "epoch": 0.0823045267489712,
      "grad_norm": 4.996777022694269,
      "kl": 0.01519775390625,
      "learning_rate": 9.727272727272727e-07,
      "loss": 0.0006,
      "reward": 0.9710937142372131,
      "reward_std": 0.5648946315050125,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.4085937589406967,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.796875,
      "epoch": 0.0850480109739369,
      "grad_norm": 6.927741488328218,
      "kl": 0.0390625,
      "learning_rate": 9.718181818181817e-07,
      "loss": 0.0016,
      "reward": 0.8914062678813934,
      "reward_std": 0.7767362892627716,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.40703123807907104,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.96875,
      "epoch": 0.0877914951989026,
      "grad_norm": 21.37164743991267,
      "kl": 0.02935791015625,
      "learning_rate": 9.709090909090909e-07,
      "loss": 0.0012,
      "reward": 1.427343726158142,
      "reward_std": 0.5377233773469925,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.6148437559604645,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.890625,
      "epoch": 0.09053497942386832,
      "grad_norm": 6.229072965781938,
      "kl": 0.0120697021484375,
      "learning_rate": 9.7e-07,
      "loss": 0.0005,
      "reward": 1.2453124523162842,
      "reward_std": 0.5992177128791809,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.557812511920929,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.71875,
      "epoch": 0.09327846364883402,
      "grad_norm": 19.471019529635235,
      "kl": 0.00970458984375,
      "learning_rate": 9.69090909090909e-07,
      "loss": 0.0004,
      "reward": 0.9453125,
      "reward_std": 0.5610494464635849,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.3984375,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.25,
      "epoch": 0.09602194787379972,
      "grad_norm": 3.510572314601594,
      "kl": 0.015655517578125,
      "learning_rate": 9.681818181818182e-07,
      "loss": 0.0006,
      "reward": 1.1757811903953552,
      "reward_std": 0.34083548188209534,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.4882812649011612,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.921875,
      "epoch": 0.09876543209876543,
      "grad_norm": 3.6755713799066623,
      "kl": 0.02471923828125,
      "learning_rate": 9.672727272727271e-07,
      "loss": 0.001,
      "reward": 1.1796874403953552,
      "reward_std": 0.4743943512439728,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.5234375149011612,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.1875,
      "epoch": 0.10150891632373114,
      "grad_norm": 5.910609278279588,
      "kl": 0.0098876953125,
      "learning_rate": 9.663636363636363e-07,
      "loss": 0.0004,
      "reward": 1.034375011920929,
      "reward_std": 0.7065398991107941,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.45624999701976776,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.953125,
      "epoch": 0.10425240054869685,
      "grad_norm": 5.3764744844672006,
      "kl": 0.02191162109375,
      "learning_rate": 9.654545454545455e-07,
      "loss": 0.0009,
      "reward": 1.2164061665534973,
      "reward_std": 0.5084110051393509,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.5289062708616257,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.28125,
      "epoch": 0.10699588477366255,
      "grad_norm": 4.693501392661649,
      "kl": 0.071197509765625,
      "learning_rate": 9.645454545454545e-07,
      "loss": 0.0029,
      "reward": 0.9593749642372131,
      "reward_std": 0.2084776610136032,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.4437499940395355,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.203125,
      "epoch": 0.10973936899862825,
      "grad_norm": 5.331160500435366,
      "kl": 0.062744140625,
      "learning_rate": 9.636363636363636e-07,
      "loss": 0.0025,
      "reward": 0.9445311427116394,
      "reward_std": 0.6358062028884888,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.42890623211860657,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.03125,
      "epoch": 0.11248285322359397,
      "grad_norm": 6.95362416315892,
      "kl": 0.019287109375,
      "learning_rate": 9.627272727272728e-07,
      "loss": 0.0008,
      "reward": 0.9070312082767487,
      "reward_std": 0.6886500120162964,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.39140623807907104,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.703125,
      "epoch": 0.11522633744855967,
      "grad_norm": 3.3675354564501534,
      "kl": 0.010711669921875,
      "learning_rate": 9.618181818181818e-07,
      "loss": 0.0004,
      "reward": 1.071874976158142,
      "reward_std": 0.29209593310952187,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.4624999910593033,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.671875,
      "epoch": 0.11796982167352538,
      "grad_norm": 3.9867453434532205,
      "kl": 0.0181884765625,
      "learning_rate": 9.60909090909091e-07,
      "loss": 0.0007,
      "reward": 0.7265624403953552,
      "reward_std": 0.589001476764679,
      "rewards/choice_reward": 0.40625,
      "rewards/roam_reward": 0.3203124776482582,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.015625,
      "epoch": 0.12071330589849108,
      "grad_norm": 4.903322287174992,
      "kl": 0.0218505859375,
      "learning_rate": 9.6e-07,
      "loss": 0.0009,
      "reward": 1.0820311903953552,
      "reward_std": 0.5496466904878616,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.48828125,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.390625,
      "epoch": 0.12345679012345678,
      "grad_norm": 8.089264703469837,
      "kl": 0.018707275390625,
      "learning_rate": 9.59090909090909e-07,
      "loss": 0.0007,
      "reward": 0.8726561665534973,
      "reward_std": 0.47763732075691223,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.3726562410593033,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.015625,
      "epoch": 0.1262002743484225,
      "grad_norm": 9.349089691780627,
      "kl": 0.02105712890625,
      "learning_rate": 9.58181818181818e-07,
      "loss": 0.0008,
      "reward": 1.414843738079071,
      "reward_std": 0.604925662279129,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.6179687678813934,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.015625,
      "epoch": 0.1289437585733882,
      "grad_norm": 308.00424029568256,
      "kl": 0.018798828125,
      "learning_rate": 9.572727272727272e-07,
      "loss": 0.0008,
      "reward": 1.1007811725139618,
      "reward_std": 0.3999259024858475,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.49140626192092896,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.0,
      "epoch": 0.13168724279835392,
      "grad_norm": 4.176206112067573,
      "kl": 0.01495361328125,
      "learning_rate": 9.563636363636364e-07,
      "loss": 0.0006,
      "reward": 1.0499999523162842,
      "reward_std": 0.45452696830034256,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.48749999701976776,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.34375,
      "epoch": 0.13443072702331962,
      "grad_norm": 8.442730409761806,
      "kl": 0.0172119140625,
      "learning_rate": 9.554545454545454e-07,
      "loss": 0.0007,
      "reward": 1.2593749165534973,
      "reward_std": 0.5093583762645721,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.5562500208616257,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.3125,
      "epoch": 0.13717421124828533,
      "grad_norm": 7.629286049055737,
      "kl": 0.016357421875,
      "learning_rate": 9.545454545454546e-07,
      "loss": 0.0007,
      "reward": 1.1859374642372131,
      "reward_std": 0.5876812934875488,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.5296874791383743,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.734375,
      "epoch": 0.13991769547325103,
      "grad_norm": 4.772489927214594,
      "kl": 0.02093505859375,
      "learning_rate": 9.536363636363636e-07,
      "loss": 0.0008,
      "reward": 1.1367187201976776,
      "reward_std": 0.3796752989292145,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.4804687350988388,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.28125,
      "epoch": 0.14266117969821673,
      "grad_norm": 5.50996340780448,
      "kl": 0.01727294921875,
      "learning_rate": 9.527272727272727e-07,
      "loss": 0.0007,
      "reward": 0.9468749761581421,
      "reward_std": 0.7070161998271942,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.4000000059604645,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.578125,
      "epoch": 0.14540466392318244,
      "grad_norm": 12.007296704335586,
      "kl": 0.041259765625,
      "learning_rate": 9.518181818181818e-07,
      "loss": 0.0016,
      "reward": 1.2179687023162842,
      "reward_std": 0.5313120782375336,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.5617187768220901,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.375,
      "epoch": 0.14814814814814814,
      "grad_norm": 22.29430069726113,
      "kl": 0.0205078125,
      "learning_rate": 9.509090909090908e-07,
      "loss": 0.0008,
      "reward": 1.108593761920929,
      "reward_std": 0.5563602149486542,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.48359373211860657,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.875,
      "epoch": 0.15089163237311384,
      "grad_norm": 7.735008497656213,
      "kl": 0.013916015625,
      "learning_rate": 9.499999999999999e-07,
      "loss": 0.0006,
      "reward": 0.9749999940395355,
      "reward_std": 0.6277340650558472,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.4281249940395355,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.78125,
      "epoch": 0.15363511659807957,
      "grad_norm": 6.050105779999899,
      "kl": 0.0223388671875,
      "learning_rate": 9.490909090909091e-07,
      "loss": 0.0009,
      "reward": 0.9632812291383743,
      "reward_std": 0.2669573575258255,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.3695312552154064,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.609375,
      "epoch": 0.15637860082304528,
      "grad_norm": 4.091151241083613,
      "kl": 0.0247802734375,
      "learning_rate": 9.481818181818182e-07,
      "loss": 0.001,
      "reward": 0.9515624642372131,
      "reward_std": 0.4127080887556076,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.45156247913837433,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.0,
      "epoch": 0.15912208504801098,
      "grad_norm": 3.6710308570080694,
      "kl": 0.0223388671875,
      "learning_rate": 9.472727272727272e-07,
      "loss": 0.0009,
      "reward": 1.506250023841858,
      "reward_std": 0.5399478673934937,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.6625000536441803,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.3125,
      "epoch": 0.16186556927297668,
      "grad_norm": 4.478479008678768,
      "kl": 0.0220947265625,
      "learning_rate": 9.463636363636364e-07,
      "loss": 0.0009,
      "reward": 0.8124999403953552,
      "reward_std": 0.522526353597641,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.3593749850988388,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.6875,
      "epoch": 0.1646090534979424,
      "grad_norm": 5.956354120549674,
      "kl": 0.0487060546875,
      "learning_rate": 9.454545454545454e-07,
      "loss": 0.0019,
      "reward": 1.021093726158142,
      "reward_std": 0.68604376912117,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.4429687559604645,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 99.0625,
      "epoch": 0.1673525377229081,
      "grad_norm": 4.817665871751061,
      "kl": 0.02001953125,
      "learning_rate": 9.445454545454545e-07,
      "loss": 0.0008,
      "reward": 1.3484374284744263,
      "reward_std": 0.7283227443695068,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.5828125178813934,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 96.09375,
      "epoch": 0.1700960219478738,
      "grad_norm": 3.6242596404503176,
      "kl": 0.0205078125,
      "learning_rate": 9.436363636363636e-07,
      "loss": 0.0008,
      "reward": 1.244531273841858,
      "reward_std": 0.46894803643226624,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.4789062440395355,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.53125,
      "epoch": 0.1728395061728395,
      "grad_norm": 3.7126985433932367,
      "kl": 0.03619384765625,
      "learning_rate": 9.427272727272727e-07,
      "loss": 0.0014,
      "reward": 0.8460937142372131,
      "reward_std": 0.6942168772220612,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.37734372913837433,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.578125,
      "epoch": 0.1755829903978052,
      "grad_norm": 5.404480273297857,
      "kl": 0.020751953125,
      "learning_rate": 9.418181818181818e-07,
      "loss": 0.0008,
      "reward": 1.407031238079071,
      "reward_std": 0.3813091591000557,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.5945312678813934,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.5625,
      "epoch": 0.17832647462277093,
      "grad_norm": 11.106716403601697,
      "kl": 0.0284423828125,
      "learning_rate": 9.409090909090908e-07,
      "loss": 0.0011,
      "reward": 1.239843726158142,
      "reward_std": 0.2835831791162491,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.5523437559604645,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.671875,
      "epoch": 0.18106995884773663,
      "grad_norm": 5.339972405871843,
      "kl": 0.02227783203125,
      "learning_rate": 9.399999999999999e-07,
      "loss": 0.0009,
      "reward": 1.2054687142372131,
      "reward_std": 0.4735162556171417,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.5648437142372131,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 105.078125,
      "epoch": 0.18381344307270234,
      "grad_norm": 6.59725887491622,
      "kl": 0.026123046875,
      "learning_rate": 9.390909090909091e-07,
      "loss": 0.001,
      "reward": 1.0601562857627869,
      "reward_std": 0.5607233345508575,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.4351562559604645,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.171875,
      "epoch": 0.18655692729766804,
      "grad_norm": 5.148995458977506,
      "kl": 0.0205078125,
      "learning_rate": 9.381818181818181e-07,
      "loss": 0.0008,
      "reward": 1.079687476158142,
      "reward_std": 0.67804354429245,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.4859375059604645,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.125,
      "epoch": 0.18930041152263374,
      "grad_norm": 5.44976353091056,
      "kl": 0.02325439453125,
      "learning_rate": 9.372727272727272e-07,
      "loss": 0.0009,
      "reward": 1.5710936784744263,
      "reward_std": 0.525007963180542,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.680468738079071,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.5625,
      "epoch": 0.19204389574759945,
      "grad_norm": 4.727339965319695,
      "kl": 0.0242919921875,
      "learning_rate": 9.363636363636364e-07,
      "loss": 0.001,
      "reward": 1.239843726158142,
      "reward_std": 0.559609979391098,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.5523437559604645,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.953125,
      "epoch": 0.19478737997256515,
      "grad_norm": 6.868511552604511,
      "kl": 0.0240478515625,
      "learning_rate": 9.354545454545454e-07,
      "loss": 0.001,
      "reward": 1.1156249642372131,
      "reward_std": 0.7248922288417816,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.5062500387430191,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.625,
      "epoch": 0.19753086419753085,
      "grad_norm": 5.111425144500442,
      "kl": 0.033935546875,
      "learning_rate": 9.345454545454545e-07,
      "loss": 0.0014,
      "reward": 0.8828124701976776,
      "reward_std": 0.5237676799297333,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.4140624701976776,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.84375,
      "epoch": 0.20027434842249658,
      "grad_norm": 6.375847552027292,
      "kl": 0.0338134765625,
      "learning_rate": 9.336363636363636e-07,
      "loss": 0.0014,
      "reward": 0.9273437261581421,
      "reward_std": 0.6422051191329956,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.4273437112569809,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.59375,
      "epoch": 0.2030178326474623,
      "grad_norm": 6.3864872962971,
      "kl": 0.034423828125,
      "learning_rate": 9.327272727272727e-07,
      "loss": 0.0014,
      "reward": 1.5023437142372131,
      "reward_std": 0.42026302218437195,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.6742187738418579,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 99.921875,
      "epoch": 0.205761316872428,
      "grad_norm": 4.324614987231985,
      "kl": 0.02606201171875,
      "learning_rate": 9.318181818181817e-07,
      "loss": 0.001,
      "reward": 0.7874999642372131,
      "reward_std": 0.34173646569252014,
      "rewards/choice_reward": 0.421875,
      "rewards/roam_reward": 0.36562497913837433,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.46875,
      "epoch": 0.2085048010973937,
      "grad_norm": 7.876492304090726,
      "kl": 0.02252197265625,
      "learning_rate": 9.309090909090909e-07,
      "loss": 0.0009,
      "reward": 1.3250000476837158,
      "reward_std": 0.6011227965354919,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.5593750178813934,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.9375,
      "epoch": 0.2112482853223594,
      "grad_norm": 8.904234886894526,
      "kl": 0.04833984375,
      "learning_rate": 9.3e-07,
      "loss": 0.0019,
      "reward": 1.142968773841858,
      "reward_std": 0.41374167799949646,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.5335937440395355,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.234375,
      "epoch": 0.2139917695473251,
      "grad_norm": 5.0500367034566285,
      "kl": 0.0286865234375,
      "learning_rate": 9.290909090909091e-07,
      "loss": 0.0011,
      "reward": 1.395312488079071,
      "reward_std": 0.5761922597885132,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.645312488079071,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.734375,
      "epoch": 0.2167352537722908,
      "grad_norm": 5.17109089115606,
      "kl": 0.02337646484375,
      "learning_rate": 9.281818181818181e-07,
      "loss": 0.0009,
      "reward": 1.2734374403953552,
      "reward_std": 0.4153551012277603,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.5859375149011612,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.34375,
      "epoch": 0.2194787379972565,
      "grad_norm": 5.490569803700363,
      "kl": 0.025634765625,
      "learning_rate": 9.272727272727272e-07,
      "loss": 0.001,
      "reward": 0.8453124463558197,
      "reward_std": 0.5321288704872131,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.37656253576278687,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.734375,
      "epoch": 0.2222222222222222,
      "grad_norm": 5.0754221806707305,
      "kl": 0.04443359375,
      "learning_rate": 9.263636363636364e-07,
      "loss": 0.0018,
      "reward": 1.7281250357627869,
      "reward_std": 0.29361481964588165,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.7593750357627869,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.25,
      "epoch": 0.22496570644718794,
      "grad_norm": 3.7285408578284724,
      "kl": 0.02777099609375,
      "learning_rate": 9.254545454545453e-07,
      "loss": 0.0011,
      "reward": 1.30078125,
      "reward_std": 0.35986968874931335,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.58203125,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.5625,
      "epoch": 0.22770919067215364,
      "grad_norm": 5.535866076818073,
      "kl": 0.02117919921875,
      "learning_rate": 9.245454545454545e-07,
      "loss": 0.0008,
      "reward": 1.393750011920929,
      "reward_std": 0.2868022173643112,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.643750011920929,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.453125,
      "epoch": 0.23045267489711935,
      "grad_norm": 5.021842818824957,
      "kl": 0.0277099609375,
      "learning_rate": 9.236363636363636e-07,
      "loss": 0.0011,
      "reward": 1.263281226158142,
      "reward_std": 0.47014766931533813,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.5601562559604645,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.34375,
      "epoch": 0.23319615912208505,
      "grad_norm": 5.028085915013138,
      "kl": 0.02374267578125,
      "learning_rate": 9.227272727272727e-07,
      "loss": 0.001,
      "reward": 0.977343738079071,
      "reward_std": 0.3106689900159836,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.47734376788139343,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.203125,
      "epoch": 0.23593964334705075,
      "grad_norm": 14.449547520531839,
      "kl": 0.03857421875,
      "learning_rate": 9.218181818181818e-07,
      "loss": 0.0015,
      "reward": 0.8820312321186066,
      "reward_std": 0.5157787501811981,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.42890623211860657,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.6875,
      "epoch": 0.23868312757201646,
      "grad_norm": 6.240824814588787,
      "kl": 0.056884765625,
      "learning_rate": 9.209090909090909e-07,
      "loss": 0.0023,
      "reward": 1.243749976158142,
      "reward_std": 0.7348380982875824,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6031249761581421,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.0625,
      "epoch": 0.24142661179698216,
      "grad_norm": 5.99769488738683,
      "kl": 0.0648193359375,
      "learning_rate": 9.2e-07,
      "loss": 0.0026,
      "reward": 1.2578125,
      "reward_std": 0.7988573312759399,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6015624701976776,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.890625,
      "epoch": 0.24417009602194786,
      "grad_norm": 4.495550953149352,
      "kl": 0.0609130859375,
      "learning_rate": 9.19090909090909e-07,
      "loss": 0.0024,
      "reward": 1.310937523841858,
      "reward_std": 0.6118793487548828,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6234374940395355,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.203125,
      "epoch": 0.24691358024691357,
      "grad_norm": 14.959203165834163,
      "kl": 0.0543212890625,
      "learning_rate": 9.181818181818181e-07,
      "loss": 0.0022,
      "reward": 0.96875,
      "reward_std": 0.5600316077470779,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.4843749701976776,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.140625,
      "epoch": 0.2496570644718793,
      "grad_norm": 8.605452241909484,
      "kl": 0.0543212890625,
      "learning_rate": 9.172727272727272e-07,
      "loss": 0.0022,
      "reward": 1.167187511920929,
      "reward_std": 0.6275071799755096,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.5578124821186066,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.484375,
      "epoch": 0.252400548696845,
      "grad_norm": 12.223314100403433,
      "kl": 0.0606689453125,
      "learning_rate": 9.163636363636364e-07,
      "loss": 0.0024,
      "reward": 1.2749999165534973,
      "reward_std": 0.6670045852661133,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6187500059604645,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.4375,
      "epoch": 0.2551440329218107,
      "grad_norm": 4.38653790036809,
      "kl": 0.098876953125,
      "learning_rate": 9.154545454545453e-07,
      "loss": 0.004,
      "reward": 1.211718738079071,
      "reward_std": 0.4608733206987381,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.586718738079071,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.03125,
      "epoch": 0.2578875171467764,
      "grad_norm": 6.757753284366764,
      "kl": 0.063232421875,
      "learning_rate": 9.145454545454545e-07,
      "loss": 0.0025,
      "reward": 1.547656238079071,
      "reward_std": 0.5729297697544098,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7507812082767487,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.5625,
      "epoch": 0.2606310013717421,
      "grad_norm": 5.077342693634887,
      "kl": 0.0767822265625,
      "learning_rate": 9.136363636363637e-07,
      "loss": 0.0031,
      "reward": 0.9843749701976776,
      "reward_std": 0.7534768581390381,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.4843749701976776,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.34375,
      "epoch": 0.26337448559670784,
      "grad_norm": 4.650632802200209,
      "kl": 0.0548095703125,
      "learning_rate": 9.127272727272727e-07,
      "loss": 0.0022,
      "reward": 1.3523436784744263,
      "reward_std": 0.809281975030899,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.617968738079071,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.46875,
      "epoch": 0.2661179698216735,
      "grad_norm": 10.070287917248892,
      "kl": 0.06982421875,
      "learning_rate": 9.118181818181818e-07,
      "loss": 0.0028,
      "reward": 1.1124999523162842,
      "reward_std": 0.4468783587217331,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.5343749970197678,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.59375,
      "epoch": 0.26886145404663925,
      "grad_norm": 14.869262822828924,
      "kl": 0.10107421875,
      "learning_rate": 9.109090909090909e-07,
      "loss": 0.004,
      "reward": 1.078906238079071,
      "reward_std": 0.8109665513038635,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5320312678813934,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.09375,
      "epoch": 0.2716049382716049,
      "grad_norm": 8.524354382622187,
      "kl": 0.078857421875,
      "learning_rate": 9.1e-07,
      "loss": 0.0032,
      "reward": 1.3359375,
      "reward_std": 0.8216530680656433,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6640624701976776,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.390625,
      "epoch": 0.27434842249657065,
      "grad_norm": 4.866005700071,
      "kl": 0.0703125,
      "learning_rate": 9.09090909090909e-07,
      "loss": 0.0028,
      "reward": 1.3390624523162842,
      "reward_std": 0.40313820540905,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6671874523162842,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.390625,
      "epoch": 0.27709190672153633,
      "grad_norm": 11.598271374848927,
      "kl": 0.077880859375,
      "learning_rate": 9.081818181818181e-07,
      "loss": 0.0031,
      "reward": 1.548437476158142,
      "reward_std": 0.6047469079494476,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7359375357627869,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.796875,
      "epoch": 0.27983539094650206,
      "grad_norm": 4.238458907315475,
      "kl": 0.0662841796875,
      "learning_rate": 9.072727272727272e-07,
      "loss": 0.0027,
      "reward": 1.456250011920929,
      "reward_std": 0.6197087466716766,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.706250011920929,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.484375,
      "epoch": 0.2825788751714678,
      "grad_norm": 4.893207308232037,
      "kl": 0.0733642578125,
      "learning_rate": 9.063636363636364e-07,
      "loss": 0.0029,
      "reward": 1.110937476158142,
      "reward_std": 0.36215464770793915,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5484375059604645,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.453125,
      "epoch": 0.28532235939643347,
      "grad_norm": 3.444057641618053,
      "kl": 0.056640625,
      "learning_rate": 9.054545454545453e-07,
      "loss": 0.0023,
      "reward": 1.409375011920929,
      "reward_std": 0.43059414625167847,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7062499821186066,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.0625,
      "epoch": 0.2880658436213992,
      "grad_norm": 6.563920276782772,
      "kl": 0.05712890625,
      "learning_rate": 9.045454545454545e-07,
      "loss": 0.0023,
      "reward": 1.3742187023162842,
      "reward_std": 0.6756561994552612,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6867187321186066,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.109375,
      "epoch": 0.2908093278463649,
      "grad_norm": 5.004219209273814,
      "kl": 0.0635986328125,
      "learning_rate": 9.036363636363637e-07,
      "loss": 0.0025,
      "reward": 1.0578124970197678,
      "reward_std": 0.543773353099823,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5421874970197678,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.84375,
      "epoch": 0.2935528120713306,
      "grad_norm": 5.006064034352587,
      "kl": 0.0760498046875,
      "learning_rate": 9.027272727272726e-07,
      "loss": 0.003,
      "reward": 1.364062488079071,
      "reward_std": 0.4392424523830414,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6765624582767487,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.828125,
      "epoch": 0.2962962962962963,
      "grad_norm": 6.230846226086625,
      "kl": 0.07080078125,
      "learning_rate": 9.018181818181818e-07,
      "loss": 0.0028,
      "reward": 1.2906250357627869,
      "reward_std": 0.7027235329151154,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6343750059604645,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.078125,
      "epoch": 0.299039780521262,
      "grad_norm": 17.840305616448358,
      "kl": 0.0703125,
      "learning_rate": 9.009090909090909e-07,
      "loss": 0.0028,
      "reward": 1.500781238079071,
      "reward_std": 0.6326878219842911,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.735156238079071,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.46875,
      "epoch": 0.3017832647462277,
      "grad_norm": 4.0753012242949564,
      "kl": 0.0653076171875,
      "learning_rate": 9e-07,
      "loss": 0.0026,
      "reward": 0.8124999701976776,
      "reward_std": 0.8824117183685303,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.421875,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.328125,
      "epoch": 0.3045267489711934,
      "grad_norm": 3.228177073766033,
      "kl": 0.06787109375,
      "learning_rate": 8.99090909090909e-07,
      "loss": 0.0027,
      "reward": 1.228906273841858,
      "reward_std": 0.5324103534221649,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6195312738418579,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.828125,
      "epoch": 0.30727023319615915,
      "grad_norm": 4.50841039714633,
      "kl": 0.06982421875,
      "learning_rate": 8.981818181818181e-07,
      "loss": 0.0028,
      "reward": 1.30859375,
      "reward_std": 0.7017756104469299,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6523437201976776,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.5,
      "epoch": 0.3100137174211248,
      "grad_norm": 4.325246595315257,
      "kl": 0.09228515625,
      "learning_rate": 8.972727272727272e-07,
      "loss": 0.0037,
      "reward": 1.007031261920929,
      "reward_std": 0.5978388488292694,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.49140623211860657,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.34375,
      "epoch": 0.31275720164609055,
      "grad_norm": 4.994040116821786,
      "kl": 0.080078125,
      "learning_rate": 8.963636363636363e-07,
      "loss": 0.0032,
      "reward": 1.610156238079071,
      "reward_std": 0.3702356815338135,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.797656238079071,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.765625,
      "epoch": 0.31550068587105623,
      "grad_norm": 3.2675458624595204,
      "kl": 0.087890625,
      "learning_rate": 8.954545454545454e-07,
      "loss": 0.0035,
      "reward": 1.7054687142372131,
      "reward_std": 0.3386332616209984,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8460937440395355,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.21875,
      "epoch": 0.31824417009602196,
      "grad_norm": 6.459049917145571,
      "kl": 0.0908203125,
      "learning_rate": 8.945454545454545e-07,
      "loss": 0.0036,
      "reward": 0.7921874821186066,
      "reward_std": 0.6447290182113647,
      "rewards/choice_reward": 0.375,
      "rewards/roam_reward": 0.41718748211860657,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.359375,
      "epoch": 0.32098765432098764,
      "grad_norm": 3.464024839590293,
      "kl": 0.07177734375,
      "learning_rate": 8.936363636363637e-07,
      "loss": 0.0029,
      "reward": 1.1210937798023224,
      "reward_std": 0.46119415760040283,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5742186903953552,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 98.84375,
      "epoch": 0.32373113854595337,
      "grad_norm": 3.787838016059412,
      "kl": 0.0732421875,
      "learning_rate": 8.927272727272726e-07,
      "loss": 0.0029,
      "reward": 1.0484374463558197,
      "reward_std": 0.6449593305587769,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5328124761581421,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.796875,
      "epoch": 0.32647462277091904,
      "grad_norm": 4.158322704150245,
      "kl": 0.10693359375,
      "learning_rate": 8.918181818181818e-07,
      "loss": 0.0043,
      "reward": 0.9898437261581421,
      "reward_std": 0.6245564520359039,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.5054687261581421,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.953125,
      "epoch": 0.3292181069958848,
      "grad_norm": 5.328842138867986,
      "kl": 0.082275390625,
      "learning_rate": 8.909090909090909e-07,
      "loss": 0.0033,
      "reward": 1.060937523841858,
      "reward_std": 0.774547666311264,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5140624493360519,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.25,
      "epoch": 0.3319615912208505,
      "grad_norm": 4.398455266312539,
      "kl": 0.074462890625,
      "learning_rate": 8.9e-07,
      "loss": 0.003,
      "reward": 1.529687523841858,
      "reward_std": 0.6608569622039795,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7484374642372131,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.09375,
      "epoch": 0.3347050754458162,
      "grad_norm": 3.6986348942117826,
      "kl": 0.0673828125,
      "learning_rate": 8.89090909090909e-07,
      "loss": 0.0027,
      "reward": 1.6007813215255737,
      "reward_std": 0.6274507939815521,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7882812321186066,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.109375,
      "epoch": 0.3374485596707819,
      "grad_norm": 5.404402499390665,
      "kl": 0.0672607421875,
      "learning_rate": 8.881818181818181e-07,
      "loss": 0.0027,
      "reward": 1.2890624403953552,
      "reward_std": 0.6910946071147919,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6484375,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.359375,
      "epoch": 0.3401920438957476,
      "grad_norm": 6.0013254416888335,
      "kl": 0.07373046875,
      "learning_rate": 8.872727272727273e-07,
      "loss": 0.003,
      "reward": 0.9984374940395355,
      "reward_std": 0.695489376783371,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.5140624642372131,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.09375,
      "epoch": 0.3429355281207133,
      "grad_norm": 3.900033333400682,
      "kl": 0.06494140625,
      "learning_rate": 8.863636363636363e-07,
      "loss": 0.0026,
      "reward": 1.207812488079071,
      "reward_std": 0.29828448593616486,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.582812488079071,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.203125,
      "epoch": 0.345679012345679,
      "grad_norm": 4.849697720686882,
      "kl": 0.0731201171875,
      "learning_rate": 8.854545454545454e-07,
      "loss": 0.0029,
      "reward": 1.1765624284744263,
      "reward_std": 0.7504218816757202,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.5828124582767487,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.546875,
      "epoch": 0.3484224965706447,
      "grad_norm": 3.7293956851296635,
      "kl": 0.080810546875,
      "learning_rate": 8.845454545454545e-07,
      "loss": 0.0032,
      "reward": 1.224218726158142,
      "reward_std": 0.45756466686725616,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6148437559604645,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.640625,
      "epoch": 0.3511659807956104,
      "grad_norm": 4.20915141452958,
      "kl": 0.0579833984375,
      "learning_rate": 8.836363636363637e-07,
      "loss": 0.0023,
      "reward": 1.764062523841858,
      "reward_std": 0.4473951756954193,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8734375238418579,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.71875,
      "epoch": 0.35390946502057613,
      "grad_norm": 3.3123008804712497,
      "kl": 0.0567626953125,
      "learning_rate": 8.827272727272726e-07,
      "loss": 0.0023,
      "reward": 1.3398438096046448,
      "reward_std": 0.5563405156135559,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6679687201976776,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.578125,
      "epoch": 0.35665294924554186,
      "grad_norm": 4.994418127246765,
      "kl": 0.0604248046875,
      "learning_rate": 8.818181818181818e-07,
      "loss": 0.0024,
      "reward": 1.596093773841858,
      "reward_std": 0.5808419585227966,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7992187440395355,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.28125,
      "epoch": 0.35939643347050754,
      "grad_norm": 3.2360790785258327,
      "kl": 0.063720703125,
      "learning_rate": 8.809090909090909e-07,
      "loss": 0.0026,
      "reward": 1.8734374642372131,
      "reward_std": 0.25403610337525606,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9203124642372131,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.21875,
      "epoch": 0.36213991769547327,
      "grad_norm": 7.248695098065959,
      "kl": 0.078125,
      "learning_rate": 8.799999999999999e-07,
      "loss": 0.0031,
      "reward": 1.436718761920929,
      "reward_std": 0.6495934128761292,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.717968761920929,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.265625,
      "epoch": 0.36488340192043894,
      "grad_norm": 8.378430132685443,
      "kl": 0.064697265625,
      "learning_rate": 8.790909090909091e-07,
      "loss": 0.0026,
      "reward": 1.046093761920929,
      "reward_std": 0.9277720153331757,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5304687172174454,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.78125,
      "epoch": 0.3676268861454047,
      "grad_norm": 3.1589997148498195,
      "kl": 0.0614013671875,
      "learning_rate": 8.781818181818182e-07,
      "loss": 0.0025,
      "reward": 1.125,
      "reward_std": 0.5783344358205795,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5781249701976776,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.171875,
      "epoch": 0.37037037037037035,
      "grad_norm": 5.042679339537896,
      "kl": 0.0548095703125,
      "learning_rate": 8.772727272727273e-07,
      "loss": 0.0022,
      "reward": 1.153124988079071,
      "reward_std": 0.7899528741836548,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.590624988079071,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.015625,
      "epoch": 0.3731138545953361,
      "grad_norm": 4.251656339164905,
      "kl": 0.055908203125,
      "learning_rate": 8.763636363636363e-07,
      "loss": 0.0022,
      "reward": 1.579687476158142,
      "reward_std": 0.4745950400829315,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7828125357627869,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.671875,
      "epoch": 0.37585733882030176,
      "grad_norm": 4.776801386893953,
      "kl": 0.073486328125,
      "learning_rate": 8.754545454545454e-07,
      "loss": 0.0029,
      "reward": 1.482812523841858,
      "reward_std": 0.49166353046894073,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7328124940395355,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.53125,
      "epoch": 0.3786008230452675,
      "grad_norm": 5.061327970873458,
      "kl": 0.066162109375,
      "learning_rate": 8.745454545454545e-07,
      "loss": 0.0027,
      "reward": 1.675000011920929,
      "reward_std": 0.44719746708869934,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.831250011920929,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.75,
      "epoch": 0.3813443072702332,
      "grad_norm": 3.6735769907735163,
      "kl": 0.0628662109375,
      "learning_rate": 8.736363636363636e-07,
      "loss": 0.0025,
      "reward": 1.249218761920929,
      "reward_std": 0.6481688320636749,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6398437321186066,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 101.796875,
      "epoch": 0.3840877914951989,
      "grad_norm": 2.6505790004359864,
      "kl": 0.060546875,
      "learning_rate": 8.727272727272726e-07,
      "loss": 0.0024,
      "reward": 1.449999988079071,
      "reward_std": 0.4319717437028885,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7156250178813934,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.03125,
      "epoch": 0.3868312757201646,
      "grad_norm": 4.147421746031423,
      "kl": 0.0592041015625,
      "learning_rate": 8.718181818181818e-07,
      "loss": 0.0024,
      "reward": 1.5,
      "reward_std": 0.3329978957772255,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.75,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.5625,
      "epoch": 0.3895747599451303,
      "grad_norm": 10.931878070060389,
      "kl": 0.07763671875,
      "learning_rate": 8.709090909090909e-07,
      "loss": 0.0031,
      "reward": 1.0132812559604645,
      "reward_std": 0.6477212905883789,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.5289062112569809,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.390625,
      "epoch": 0.39231824417009603,
      "grad_norm": 4.543133727421236,
      "kl": 0.078125,
      "learning_rate": 8.699999999999999e-07,
      "loss": 0.0031,
      "reward": 1.245312511920929,
      "reward_std": 0.5095339268445969,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6359374821186066,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 99.5,
      "epoch": 0.3950617283950617,
      "grad_norm": 6.799154945627812,
      "kl": 0.06591796875,
      "learning_rate": 8.690909090909091e-07,
      "loss": 0.0026,
      "reward": 1.2093749642372131,
      "reward_std": 0.7542276382446289,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6156249940395355,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 96.890625,
      "epoch": 0.39780521262002744,
      "grad_norm": 21.830417679362668,
      "kl": 0.067626953125,
      "learning_rate": 8.681818181818182e-07,
      "loss": 0.0027,
      "reward": 0.8671875,
      "reward_std": 0.6834112703800201,
      "rewards/choice_reward": 0.4375,
      "rewards/roam_reward": 0.4296874850988388,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.5,
      "epoch": 0.40054869684499317,
      "grad_norm": 4.962962840054633,
      "kl": 0.090087890625,
      "learning_rate": 8.672727272727272e-07,
      "loss": 0.0036,
      "reward": 1.069531261920929,
      "reward_std": 0.42628343403339386,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5539062321186066,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.59375,
      "epoch": 0.40329218106995884,
      "grad_norm": 3.1575892724965473,
      "kl": 0.066650390625,
      "learning_rate": 8.663636363636363e-07,
      "loss": 0.0027,
      "reward": 1.569531261920929,
      "reward_std": 0.49635447561740875,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.772656261920929,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.828125,
      "epoch": 0.4060356652949246,
      "grad_norm": 6.6179935670184085,
      "kl": 0.069580078125,
      "learning_rate": 8.654545454545454e-07,
      "loss": 0.0028,
      "reward": 1.360937476158142,
      "reward_std": 0.4443059712648392,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6890624463558197,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.734375,
      "epoch": 0.40877914951989025,
      "grad_norm": 5.892963936248546,
      "kl": 0.070556640625,
      "learning_rate": 8.645454545454545e-07,
      "loss": 0.0028,
      "reward": 1.543749988079071,
      "reward_std": 0.3481606990098953,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7625000178813934,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.328125,
      "epoch": 0.411522633744856,
      "grad_norm": 6.263730759309839,
      "kl": 0.081787109375,
      "learning_rate": 8.636363636363636e-07,
      "loss": 0.0033,
      "reward": 1.300000011920929,
      "reward_std": 0.7723948359489441,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6437499821186066,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.96875,
      "epoch": 0.41426611796982166,
      "grad_norm": 4.024542032878697,
      "kl": 0.147705078125,
      "learning_rate": 8.627272727272726e-07,
      "loss": 0.0059,
      "reward": 1.3515625,
      "reward_std": 0.3730863481760025,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6796874701976776,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.359375,
      "epoch": 0.4170096021947874,
      "grad_norm": 5.399045676631981,
      "kl": 0.05712890625,
      "learning_rate": 8.618181818181818e-07,
      "loss": 0.0023,
      "reward": 0.90234375,
      "reward_std": 0.39868150651454926,
      "rewards/choice_reward": 0.4375,
      "rewards/roam_reward": 0.46484375,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.03125,
      "epoch": 0.41975308641975306,
      "grad_norm": 5.7191847832606,
      "kl": 0.06103515625,
      "learning_rate": 8.60909090909091e-07,
      "loss": 0.0024,
      "reward": 1.348437488079071,
      "reward_std": 0.35833945870399475,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6765624582767487,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.1875,
      "epoch": 0.4224965706447188,
      "grad_norm": 2.661374077804329,
      "kl": 0.06787109375,
      "learning_rate": 8.599999999999999e-07,
      "loss": 0.0027,
      "reward": 1.7312500476837158,
      "reward_std": 0.04797553177922964,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8562500178813934,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.015625,
      "epoch": 0.4252400548696845,
      "grad_norm": 3.9622328522717303,
      "kl": 0.060302734375,
      "learning_rate": 8.590909090909091e-07,
      "loss": 0.0024,
      "reward": 1.4890624284744263,
      "reward_std": 0.5478314012289047,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7390625178813934,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.671875,
      "epoch": 0.4279835390946502,
      "grad_norm": 3.273955933162255,
      "kl": 0.0582275390625,
      "learning_rate": 8.581818181818182e-07,
      "loss": 0.0023,
      "reward": 1.2171874642372131,
      "reward_std": 0.6891123056411743,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6078124940395355,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.984375,
      "epoch": 0.43072702331961593,
      "grad_norm": 4.7893666691359345,
      "kl": 0.0482177734375,
      "learning_rate": 8.572727272727272e-07,
      "loss": 0.0019,
      "reward": 1.54296875,
      "reward_std": 0.5729685127735138,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.73046875,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.75,
      "epoch": 0.4334705075445816,
      "grad_norm": 5.119948994612539,
      "kl": 0.087890625,
      "learning_rate": 8.563636363636363e-07,
      "loss": 0.0035,
      "reward": 1.5859375,
      "reward_std": 0.35503947734832764,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7734374701976776,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.140625,
      "epoch": 0.43621399176954734,
      "grad_norm": 5.3369385279264705,
      "kl": 0.052978515625,
      "learning_rate": 8.554545454545454e-07,
      "loss": 0.0021,
      "reward": 1.34375,
      "reward_std": 0.7487562596797943,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.671875,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.578125,
      "epoch": 0.438957475994513,
      "grad_norm": 6.963922269114714,
      "kl": 0.0548095703125,
      "learning_rate": 8.545454545454544e-07,
      "loss": 0.0022,
      "reward": 1.1132811903953552,
      "reward_std": 0.5710795968770981,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.51953125,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.125,
      "epoch": 0.44170096021947874,
      "grad_norm": 5.0024696069552075,
      "kl": 0.059326171875,
      "learning_rate": 8.536363636363636e-07,
      "loss": 0.0024,
      "reward": 1.1171875596046448,
      "reward_std": 0.5813678801059723,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5546875,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.765625,
      "epoch": 0.4444444444444444,
      "grad_norm": 6.863926677541896,
      "kl": 0.072265625,
      "learning_rate": 8.527272727272727e-07,
      "loss": 0.0029,
      "reward": 1.3398436903953552,
      "reward_std": 0.45532071590423584,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.65234375,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.34375,
      "epoch": 0.44718792866941015,
      "grad_norm": 4.309835209354842,
      "kl": 0.062255859375,
      "learning_rate": 8.518181818181818e-07,
      "loss": 0.0025,
      "reward": 1.2171874642372131,
      "reward_std": 0.5791269540786743,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6078124940395355,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.21875,
      "epoch": 0.4499314128943759,
      "grad_norm": 4.814305223557731,
      "kl": 0.0640869140625,
      "learning_rate": 8.509090909090909e-07,
      "loss": 0.0026,
      "reward": 1.4398437142372131,
      "reward_std": 0.19878476113080978,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7210937440395355,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.03125,
      "epoch": 0.45267489711934156,
      "grad_norm": 4.324305104508143,
      "kl": 0.066650390625,
      "learning_rate": 8.499999999999999e-07,
      "loss": 0.0027,
      "reward": 1.706250011920929,
      "reward_std": 0.45024340599775314,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.831250011920929,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.25,
      "epoch": 0.4554183813443073,
      "grad_norm": 12.515244809329793,
      "kl": 0.081298828125,
      "learning_rate": 8.490909090909091e-07,
      "loss": 0.0033,
      "reward": 1.435156226158142,
      "reward_std": 0.5398507863283157,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7007812261581421,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.1875,
      "epoch": 0.45816186556927296,
      "grad_norm": 8.8304789346465,
      "kl": 0.062255859375,
      "learning_rate": 8.481818181818182e-07,
      "loss": 0.0025,
      "reward": 1.694531261920929,
      "reward_std": 0.6171333193778992,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8195312321186066,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.53125,
      "epoch": 0.4609053497942387,
      "grad_norm": 5.0298882453907705,
      "kl": 0.064697265625,
      "learning_rate": 8.472727272727272e-07,
      "loss": 0.0026,
      "reward": 1.306249976158142,
      "reward_std": 0.5636239349842072,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6343749463558197,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.1875,
      "epoch": 0.46364883401920437,
      "grad_norm": 6.918591227186971,
      "kl": 0.053955078125,
      "learning_rate": 8.463636363636363e-07,
      "loss": 0.0022,
      "reward": 1.1968750655651093,
      "reward_std": 0.3086411654949188,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6031249761581421,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.078125,
      "epoch": 0.4663923182441701,
      "grad_norm": 10.229800465554694,
      "kl": 0.0631103515625,
      "learning_rate": 8.454545454545454e-07,
      "loss": 0.0025,
      "reward": 1.167187511920929,
      "reward_std": 0.40548501908779144,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.557812511920929,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.109375,
      "epoch": 0.4691358024691358,
      "grad_norm": 4.895522138743598,
      "kl": 0.0555419921875,
      "learning_rate": 8.445454545454545e-07,
      "loss": 0.0022,
      "reward": 0.9687499701976776,
      "reward_std": 0.36868916451931,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.4843749850988388,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.703125,
      "epoch": 0.4718792866941015,
      "grad_norm": 8.387517289112681,
      "kl": 0.05908203125,
      "learning_rate": 8.436363636363636e-07,
      "loss": 0.0024,
      "reward": 1.2609374523162842,
      "reward_std": 0.47968146204948425,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.620312511920929,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.03125,
      "epoch": 0.47462277091906724,
      "grad_norm": 8.177680301301846,
      "kl": 0.0472412109375,
      "learning_rate": 8.427272727272727e-07,
      "loss": 0.0019,
      "reward": 1.29296875,
      "reward_std": 0.6639520227909088,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6523436903953552,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.03125,
      "epoch": 0.4773662551440329,
      "grad_norm": 4.238237025234008,
      "kl": 0.06640625,
      "learning_rate": 8.418181818181818e-07,
      "loss": 0.0027,
      "reward": 1.607031226158142,
      "reward_std": 0.2946796417236328,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7789062559604645,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.515625,
      "epoch": 0.48010973936899864,
      "grad_norm": 9.88349820542714,
      "kl": 0.0687255859375,
      "learning_rate": 8.409090909090909e-07,
      "loss": 0.0027,
      "reward": 1.362500011920929,
      "reward_std": 0.4368693381547928,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.675000011920929,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.59375,
      "epoch": 0.4828532235939643,
      "grad_norm": 4.996971343930396,
      "kl": 0.064453125,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.0026,
      "reward": 1.498437523841858,
      "reward_std": 0.5075877904891968,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7484375238418579,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.171875,
      "epoch": 0.48559670781893005,
      "grad_norm": 3.9600821078570516,
      "kl": 0.0634765625,
      "learning_rate": 8.390909090909091e-07,
      "loss": 0.0025,
      "reward": 1.4000000357627869,
      "reward_std": 0.5038798004388809,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6812499910593033,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.171875,
      "epoch": 0.4883401920438957,
      "grad_norm": 3.1362251817973954,
      "kl": 0.06396484375,
      "learning_rate": 8.381818181818181e-07,
      "loss": 0.0026,
      "reward": 1.4093750715255737,
      "reward_std": 0.38460302352905273,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6906249821186066,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.265625,
      "epoch": 0.49108367626886146,
      "grad_norm": 25.078130616204984,
      "kl": 0.062744140625,
      "learning_rate": 8.372727272727272e-07,
      "loss": 0.0025,
      "reward": 1.28515625,
      "reward_std": 0.808452844619751,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.62890625,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.265625,
      "epoch": 0.49382716049382713,
      "grad_norm": 4.035103162287522,
      "kl": 0.06103515625,
      "learning_rate": 8.363636363636363e-07,
      "loss": 0.0024,
      "reward": 1.5062499642372131,
      "reward_std": 0.5272868573665619,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7406249642372131,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.78125,
      "epoch": 0.49657064471879286,
      "grad_norm": 7.1345410663907,
      "kl": 0.0576171875,
      "learning_rate": 8.354545454545455e-07,
      "loss": 0.0023,
      "reward": 1.501562476158142,
      "reward_std": 0.4868186265230179,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7515625059604645,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.234375,
      "epoch": 0.4993141289437586,
      "grad_norm": 5.1332471380641245,
      "kl": 0.06201171875,
      "learning_rate": 8.345454545454545e-07,
      "loss": 0.0025,
      "reward": 1.438281238079071,
      "reward_std": 0.5540306866168976,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.703906238079071,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.625,
      "epoch": 0.5020576131687243,
      "grad_norm": 3.6923216925479827,
      "kl": 0.063232421875,
      "learning_rate": 8.336363636363636e-07,
      "loss": 0.0025,
      "reward": 1.921875,
      "reward_std": 0.1922881193459034,
      "rewards/choice_reward": 0.984375,
      "rewards/roam_reward": 0.9375,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.96875,
      "epoch": 0.50480109739369,
      "grad_norm": 10.401860032506013,
      "kl": 0.081298828125,
      "learning_rate": 8.327272727272727e-07,
      "loss": 0.0033,
      "reward": 1.055468738079071,
      "reward_std": 0.506570965051651,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5398437082767487,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.5625,
      "epoch": 0.5075445816186557,
      "grad_norm": 11.418018723924762,
      "kl": 0.0673828125,
      "learning_rate": 8.318181818181817e-07,
      "loss": 0.0027,
      "reward": 0.9195312261581421,
      "reward_std": 0.7840481698513031,
      "rewards/choice_reward": 0.4375,
      "rewards/roam_reward": 0.4820312112569809,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.546875,
      "epoch": 0.5102880658436214,
      "grad_norm": 8.196839368231336,
      "kl": 0.0618896484375,
      "learning_rate": 8.309090909090909e-07,
      "loss": 0.0025,
      "reward": 1.266406238079071,
      "reward_std": 0.6124914884567261,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.641406238079071,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.4375,
      "epoch": 0.5130315500685871,
      "grad_norm": 5.765281950846985,
      "kl": 0.07470703125,
      "learning_rate": 8.299999999999999e-07,
      "loss": 0.003,
      "reward": 1.354687511920929,
      "reward_std": 0.3992236405611038,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.682812511920929,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.28125,
      "epoch": 0.5157750342935528,
      "grad_norm": 3.7172993929568325,
      "kl": 0.0555419921875,
      "learning_rate": 8.290909090909091e-07,
      "loss": 0.0022,
      "reward": 1.3773437142372131,
      "reward_std": 0.5645609945058823,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6898437142372131,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.109375,
      "epoch": 0.5185185185185185,
      "grad_norm": 16.786414154053386,
      "kl": 0.068359375,
      "learning_rate": 8.281818181818181e-07,
      "loss": 0.0027,
      "reward": 1.662500023841858,
      "reward_std": 0.5854551792144775,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8343749940395355,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.421875,
      "epoch": 0.5212620027434842,
      "grad_norm": 4.276593973859327,
      "kl": 0.08642578125,
      "learning_rate": 8.272727272727272e-07,
      "loss": 0.0035,
      "reward": 1.5609374642372131,
      "reward_std": 0.4817267805337906,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7796874940395355,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.640625,
      "epoch": 0.52400548696845,
      "grad_norm": 11.234841144738262,
      "kl": 0.074462890625,
      "learning_rate": 8.263636363636364e-07,
      "loss": 0.003,
      "reward": 1.4281249642372131,
      "reward_std": 0.13276604190468788,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7249999642372131,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.96875,
      "epoch": 0.5267489711934157,
      "grad_norm": 4.108221314020624,
      "kl": 0.0673828125,
      "learning_rate": 8.254545454545455e-07,
      "loss": 0.0027,
      "reward": 1.456250011920929,
      "reward_std": 0.35238832235336304,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7218749821186066,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.671875,
      "epoch": 0.5294924554183813,
      "grad_norm": 16.0986145516457,
      "kl": 0.106201171875,
      "learning_rate": 8.245454545454545e-07,
      "loss": 0.0042,
      "reward": 1.378125011920929,
      "reward_std": 0.44103650748729706,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.690625011920929,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.671875,
      "epoch": 0.532235939643347,
      "grad_norm": 5.763755062215557,
      "kl": 0.077392578125,
      "learning_rate": 8.236363636363636e-07,
      "loss": 0.0031,
      "reward": 1.1351562142372131,
      "reward_std": 0.5356723964214325,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5882812142372131,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.78125,
      "epoch": 0.5349794238683128,
      "grad_norm": 3.782326754008591,
      "kl": 0.091064453125,
      "learning_rate": 8.227272727272727e-07,
      "loss": 0.0036,
      "reward": 1.3101562857627869,
      "reward_std": 0.5293387621641159,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6382812261581421,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.5625,
      "epoch": 0.5377229080932785,
      "grad_norm": 39.2966810629021,
      "kl": 0.077880859375,
      "learning_rate": 8.218181818181817e-07,
      "loss": 0.0031,
      "reward": 1.4812500476837158,
      "reward_std": 0.5564833879470825,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.715624988079071,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.203125,
      "epoch": 0.5404663923182441,
      "grad_norm": 4.055896823642352,
      "kl": 0.09765625,
      "learning_rate": 8.209090909090909e-07,
      "loss": 0.0039,
      "reward": 1.590624988079071,
      "reward_std": 0.7486997246742249,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7937500178813934,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.4375,
      "epoch": 0.5432098765432098,
      "grad_norm": 4.254385588670383,
      "kl": 0.1015625,
      "learning_rate": 8.199999999999999e-07,
      "loss": 0.0041,
      "reward": 1.4296874403953552,
      "reward_std": 0.54799535125494,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7265624701976776,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.265625,
      "epoch": 0.5459533607681756,
      "grad_norm": 5.646190691397499,
      "kl": 0.096435546875,
      "learning_rate": 8.190909090909091e-07,
      "loss": 0.0039,
      "reward": 1.3124999403953552,
      "reward_std": 0.7107203602790833,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.65625,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.515625,
      "epoch": 0.5486968449931413,
      "grad_norm": 12.874617507070692,
      "kl": 0.07421875,
      "learning_rate": 8.181818181818182e-07,
      "loss": 0.003,
      "reward": 1.3499999642372131,
      "reward_std": 0.7221298813819885,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6468750238418579,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.6875,
      "epoch": 0.551440329218107,
      "grad_norm": 3.1856851559121817,
      "kl": 0.075439453125,
      "learning_rate": 8.172727272727272e-07,
      "loss": 0.003,
      "reward": 1.6726562976837158,
      "reward_std": 0.47229228913784027,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.844531238079071,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.546875,
      "epoch": 0.5541838134430727,
      "grad_norm": 2.3982091402370185,
      "kl": 0.085205078125,
      "learning_rate": 8.163636363636364e-07,
      "loss": 0.0034,
      "reward": 1.5078125,
      "reward_std": 0.3067757934331894,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7578125,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.34375,
      "epoch": 0.5569272976680384,
      "grad_norm": 3.0016047967579027,
      "kl": 0.08203125,
      "learning_rate": 8.154545454545454e-07,
      "loss": 0.0033,
      "reward": 1.48828125,
      "reward_std": 0.4044075235724449,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.75390625,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.125,
      "epoch": 0.5596707818930041,
      "grad_norm": 11.362915172474983,
      "kl": 0.08251953125,
      "learning_rate": 8.145454545454545e-07,
      "loss": 0.0033,
      "reward": 1.375781238079071,
      "reward_std": 0.6488424837589264,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.703906238079071,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.09375,
      "epoch": 0.5624142661179699,
      "grad_norm": 10.498621862531559,
      "kl": 0.072265625,
      "learning_rate": 8.136363636363636e-07,
      "loss": 0.0029,
      "reward": 1.2671875357627869,
      "reward_std": 0.7070984542369843,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6421874761581421,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.234375,
      "epoch": 0.5651577503429356,
      "grad_norm": 3.5297036527459893,
      "kl": 0.0771484375,
      "learning_rate": 8.127272727272727e-07,
      "loss": 0.0031,
      "reward": 1.071874976158142,
      "reward_std": 0.7598352134227753,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5562500059604645,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.890625,
      "epoch": 0.5679012345679012,
      "grad_norm": 6.090925733370347,
      "kl": 0.0631103515625,
      "learning_rate": 8.118181818181817e-07,
      "loss": 0.0025,
      "reward": 1.0101562440395355,
      "reward_std": 0.5675522089004517,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5101562589406967,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.53125,
      "epoch": 0.5706447187928669,
      "grad_norm": 3.4062820304676844,
      "kl": 0.085205078125,
      "learning_rate": 8.109090909090909e-07,
      "loss": 0.0034,
      "reward": 1.087499976158142,
      "reward_std": 0.6073361337184906,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5562499761581421,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.078125,
      "epoch": 0.5733882030178327,
      "grad_norm": 5.021924382672697,
      "kl": 0.07666015625,
      "learning_rate": 8.1e-07,
      "loss": 0.0031,
      "reward": 1.4632812142372131,
      "reward_std": 0.5444166660308838,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7132812142372131,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.59375,
      "epoch": 0.5761316872427984,
      "grad_norm": 4.83366950976083,
      "kl": 0.0662841796875,
      "learning_rate": 8.09090909090909e-07,
      "loss": 0.0027,
      "reward": 1.278124988079071,
      "reward_std": 0.4413192719221115,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6531249582767487,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.65625,
      "epoch": 0.578875171467764,
      "grad_norm": 7.426211553447796,
      "kl": 0.072509765625,
      "learning_rate": 8.081818181818182e-07,
      "loss": 0.0029,
      "reward": 1.670312523841858,
      "reward_std": 0.5201479196548462,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8265624940395355,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.421875,
      "epoch": 0.5816186556927297,
      "grad_norm": 6.151006851159569,
      "kl": 0.070068359375,
      "learning_rate": 8.072727272727272e-07,
      "loss": 0.0028,
      "reward": 1.505468726158142,
      "reward_std": 0.7296687662601471,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7554687261581421,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.546875,
      "epoch": 0.5843621399176955,
      "grad_norm": 4.997037481883234,
      "kl": 0.0791015625,
      "learning_rate": 8.063636363636364e-07,
      "loss": 0.0032,
      "reward": 1.4304687976837158,
      "reward_std": 0.3970513939857483,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.6960937082767487,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.703125,
      "epoch": 0.5871056241426612,
      "grad_norm": 9.389905559594292,
      "kl": 0.07861328125,
      "learning_rate": 8.054545454545454e-07,
      "loss": 0.0032,
      "reward": 1.30859375,
      "reward_std": 0.5475015193223953,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.65234375,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.421875,
      "epoch": 0.5898491083676269,
      "grad_norm": 9.63225445426572,
      "kl": 0.072998046875,
      "learning_rate": 8.045454545454545e-07,
      "loss": 0.0029,
      "reward": 1.8234374523162842,
      "reward_std": 0.4097355008125305,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.901562511920929,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.578125,
      "epoch": 0.5925925925925926,
      "grad_norm": 2.9208152120219633,
      "kl": 0.09912109375,
      "learning_rate": 8.036363636363636e-07,
      "loss": 0.004,
      "reward": 0.934374988079071,
      "reward_std": 0.3858245126903057,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.48124995827674866,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.0625,
      "epoch": 0.5953360768175583,
      "grad_norm": 4.505664414649176,
      "kl": 0.06494140625,
      "learning_rate": 8.027272727272726e-07,
      "loss": 0.0026,
      "reward": 1.4281249642372131,
      "reward_std": 0.508673369884491,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7249999642372131,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.171875,
      "epoch": 0.598079561042524,
      "grad_norm": 9.316782090306322,
      "kl": 0.095947265625,
      "learning_rate": 8.018181818181817e-07,
      "loss": 0.0038,
      "reward": 0.9671874642372131,
      "reward_std": 0.5713367760181427,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.48281247913837433,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.609375,
      "epoch": 0.6008230452674898,
      "grad_norm": 2.855058328976504,
      "kl": 0.071533203125,
      "learning_rate": 8.009090909090909e-07,
      "loss": 0.0029,
      "reward": 1.116406261920929,
      "reward_std": 0.344180703163147,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5695312172174454,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.03125,
      "epoch": 0.6035665294924554,
      "grad_norm": 6.719437428317107,
      "kl": 0.0653076171875,
      "learning_rate": 8e-07,
      "loss": 0.0026,
      "reward": 1.3828125,
      "reward_std": 0.6445798277854919,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6796874701976776,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.453125,
      "epoch": 0.6063100137174211,
      "grad_norm": 71.48990096623994,
      "kl": 0.074462890625,
      "learning_rate": 7.99090909090909e-07,
      "loss": 0.003,
      "reward": 1.6539062857627869,
      "reward_std": 0.3329023942351341,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8257812559604645,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.625,
      "epoch": 0.6090534979423868,
      "grad_norm": 3.0987243394875694,
      "kl": 0.0572509765625,
      "learning_rate": 7.981818181818182e-07,
      "loss": 0.0023,
      "reward": 1.564843773841858,
      "reward_std": 0.5028156638145447,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7679687440395355,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.90625,
      "epoch": 0.6117969821673526,
      "grad_norm": 4.11857447430343,
      "kl": 0.0654296875,
      "learning_rate": 7.972727272727272e-07,
      "loss": 0.0026,
      "reward": 0.9781249761581421,
      "reward_std": 0.12847728352062404,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.5093749761581421,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.765625,
      "epoch": 0.6145404663923183,
      "grad_norm": 6.078673620884672,
      "kl": 0.078369140625,
      "learning_rate": 7.963636363636364e-07,
      "loss": 0.0031,
      "reward": 1.6046874523162842,
      "reward_std": 0.3992674648761749,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8078124821186066,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.890625,
      "epoch": 0.6172839506172839,
      "grad_norm": 8.644719384891074,
      "kl": 0.071533203125,
      "learning_rate": 7.954545454545454e-07,
      "loss": 0.0029,
      "reward": 1.5695313215255737,
      "reward_std": 0.49713802337646484,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.772656261920929,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.140625,
      "epoch": 0.6200274348422496,
      "grad_norm": 3.2040195903210864,
      "kl": 0.07763671875,
      "learning_rate": 7.945454545454545e-07,
      "loss": 0.0031,
      "reward": 1.139843761920929,
      "reward_std": 0.5067263841629028,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.561718761920929,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.890625,
      "epoch": 0.6227709190672154,
      "grad_norm": 4.010879810986008,
      "kl": 0.0733642578125,
      "learning_rate": 7.936363636363636e-07,
      "loss": 0.0029,
      "reward": 0.8140625059604645,
      "reward_std": 0.5690027624368668,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.4234374761581421,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.734375,
      "epoch": 0.6255144032921811,
      "grad_norm": 9.581974697201934,
      "kl": 0.08203125,
      "learning_rate": 7.927272727272727e-07,
      "loss": 0.0033,
      "reward": 1.751562476158142,
      "reward_std": 0.33712854981422424,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8765625059604645,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.171875,
      "epoch": 0.6282578875171467,
      "grad_norm": 4.489954093125781,
      "kl": 0.084716796875,
      "learning_rate": 7.918181818181818e-07,
      "loss": 0.0034,
      "reward": 1.2937500476837158,
      "reward_std": 0.5399462282657623,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6374999582767487,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.25,
      "epoch": 0.6310013717421125,
      "grad_norm": 10.636316970075013,
      "kl": 0.12744140625,
      "learning_rate": 7.909090909090909e-07,
      "loss": 0.0051,
      "reward": 1.693750023841858,
      "reward_std": 0.5591640174388885,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8343750238418579,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.421875,
      "epoch": 0.6337448559670782,
      "grad_norm": 2.8163130027865257,
      "kl": 0.0859375,
      "learning_rate": 7.9e-07,
      "loss": 0.0034,
      "reward": 1.4468750357627869,
      "reward_std": 0.28992322087287903,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7281249761581421,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.46875,
      "epoch": 0.6364883401920439,
      "grad_norm": 5.0001878798824695,
      "kl": 0.077392578125,
      "learning_rate": 7.89090909090909e-07,
      "loss": 0.0031,
      "reward": 0.835156261920929,
      "reward_std": 0.5924297571182251,
      "rewards/choice_reward": 0.40625,
      "rewards/roam_reward": 0.4289062172174454,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.65625,
      "epoch": 0.6392318244170097,
      "grad_norm": 3.9331776114517534,
      "kl": 0.104248046875,
      "learning_rate": 7.881818181818182e-07,
      "loss": 0.0042,
      "reward": 0.967968761920929,
      "reward_std": 0.537762463092804,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.49921873211860657,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.53125,
      "epoch": 0.6419753086419753,
      "grad_norm": 12.025679525832018,
      "kl": 0.076416015625,
      "learning_rate": 7.872727272727272e-07,
      "loss": 0.0031,
      "reward": 1.5515625476837158,
      "reward_std": 0.6109479367733002,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7859375178813934,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.484375,
      "epoch": 0.644718792866941,
      "grad_norm": 4.602797841328001,
      "kl": 0.092529296875,
      "learning_rate": 7.863636363636363e-07,
      "loss": 0.0037,
      "reward": 1.405468761920929,
      "reward_std": 0.6358656883239746,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7179687321186066,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.234375,
      "epoch": 0.6474622770919067,
      "grad_norm": 16.49968920373472,
      "kl": 0.096923828125,
      "learning_rate": 7.854545454545454e-07,
      "loss": 0.0039,
      "reward": 1.0156249403953552,
      "reward_std": 0.3995538502931595,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.53125,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.78125,
      "epoch": 0.6502057613168725,
      "grad_norm": 5.31859764210797,
      "kl": 0.102294921875,
      "learning_rate": 7.845454545454545e-07,
      "loss": 0.0041,
      "reward": 1.419531226158142,
      "reward_std": 0.7316969633102417,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7164062261581421,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.265625,
      "epoch": 0.6529492455418381,
      "grad_norm": 3.9773992042537007,
      "kl": 0.09814453125,
      "learning_rate": 7.836363636363637e-07,
      "loss": 0.0039,
      "reward": 1.442187488079071,
      "reward_std": 0.3996770977973938,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.723437488079071,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.625,
      "epoch": 0.6556927297668038,
      "grad_norm": 22.07054390449565,
      "kl": 0.13623046875,
      "learning_rate": 7.827272727272727e-07,
      "loss": 0.0055,
      "reward": 1.612500011920929,
      "reward_std": 0.40521635860204697,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.815625011920929,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.25,
      "epoch": 0.6584362139917695,
      "grad_norm": 5.980129650176006,
      "kl": 0.097412109375,
      "learning_rate": 7.818181818181818e-07,
      "loss": 0.0039,
      "reward": 1.268750011920929,
      "reward_std": 0.6887725591659546,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6437499821186066,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.921875,
      "epoch": 0.6611796982167353,
      "grad_norm": 8.202068432584896,
      "kl": 0.0947265625,
      "learning_rate": 7.809090909090909e-07,
      "loss": 0.0038,
      "reward": 0.9382812082767487,
      "reward_std": 0.7795107066631317,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.48515623807907104,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.484375,
      "epoch": 0.663923182441701,
      "grad_norm": 8.414940184713855,
      "kl": 0.09765625,
      "learning_rate": 7.799999999999999e-07,
      "loss": 0.0039,
      "reward": 1.387499988079071,
      "reward_std": 0.5833183825016022,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.699999988079071,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.078125,
      "epoch": 0.6666666666666666,
      "grad_norm": 2.0305731063352193,
      "kl": 0.1015625,
      "learning_rate": 7.79090909090909e-07,
      "loss": 0.0041,
      "reward": 1.864062488079071,
      "reward_std": 0.28691305220127106,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9265625178813934,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.34375,
      "epoch": 0.6694101508916324,
      "grad_norm": 2.7281908126720817,
      "kl": 0.10302734375,
      "learning_rate": 7.781818181818182e-07,
      "loss": 0.0041,
      "reward": 1.1234374642372131,
      "reward_std": 0.5269819796085358,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5765624642372131,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.375,
      "epoch": 0.6721536351165981,
      "grad_norm": 25.08389462493231,
      "kl": 0.098388671875,
      "learning_rate": 7.772727272727272e-07,
      "loss": 0.0039,
      "reward": 1.0125000178813934,
      "reward_std": 0.6459892988204956,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.528124988079071,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.875,
      "epoch": 0.6748971193415638,
      "grad_norm": 9.640690247273248,
      "kl": 0.109619140625,
      "learning_rate": 7.763636363636363e-07,
      "loss": 0.0044,
      "reward": 0.7359375059604645,
      "reward_std": 0.7127074599266052,
      "rewards/choice_reward": 0.34375,
      "rewards/roam_reward": 0.3921874761581421,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.28125,
      "epoch": 0.6776406035665294,
      "grad_norm": 9.572287534215459,
      "kl": 0.14697265625,
      "learning_rate": 7.754545454545455e-07,
      "loss": 0.0059,
      "reward": 1.31640625,
      "reward_std": 0.5501902848482132,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6757812798023224,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.765625,
      "epoch": 0.6803840877914952,
      "grad_norm": 9.930117524851914,
      "kl": 0.1484375,
      "learning_rate": 7.745454545454545e-07,
      "loss": 0.0059,
      "reward": 1.3507812023162842,
      "reward_std": 0.5230868756771088,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6789062321186066,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.453125,
      "epoch": 0.6831275720164609,
      "grad_norm": 2.8552077332389056,
      "kl": 0.121826171875,
      "learning_rate": 7.736363636363637e-07,
      "loss": 0.0049,
      "reward": 1.6515625715255737,
      "reward_std": 0.5274887830018997,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.807812511920929,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.140625,
      "epoch": 0.6858710562414266,
      "grad_norm": 3.978831679062336,
      "kl": 0.10205078125,
      "learning_rate": 7.727272727272727e-07,
      "loss": 0.0041,
      "reward": 1.3421874642372131,
      "reward_std": 0.54762202501297,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.484375,
      "epoch": 0.6886145404663924,
      "grad_norm": 7.637263581573902,
      "kl": 0.091552734375,
      "learning_rate": 7.718181818181818e-07,
      "loss": 0.0037,
      "reward": 1.490625023841858,
      "reward_std": 0.6624627113342285,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7406249940395355,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.25,
      "epoch": 0.691358024691358,
      "grad_norm": 3.9888732110964122,
      "kl": 0.08740234375,
      "learning_rate": 7.709090909090909e-07,
      "loss": 0.0035,
      "reward": 1.66796875,
      "reward_std": 0.42886100709438324,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.83984375,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.40625,
      "epoch": 0.6941015089163237,
      "grad_norm": 4.590621836385763,
      "kl": 0.0966796875,
      "learning_rate": 7.699999999999999e-07,
      "loss": 0.0039,
      "reward": 1.596875011920929,
      "reward_std": 0.47170980274677277,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7999999821186066,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.890625,
      "epoch": 0.6968449931412894,
      "grad_norm": 4.347474133639677,
      "kl": 0.100341796875,
      "learning_rate": 7.69090909090909e-07,
      "loss": 0.004,
      "reward": 1.478124976158142,
      "reward_std": 0.593116819858551,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7437499761581421,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.171875,
      "epoch": 0.6995884773662552,
      "grad_norm": 8.503027719373268,
      "kl": 0.082763671875,
      "learning_rate": 7.681818181818182e-07,
      "loss": 0.0033,
      "reward": 1.448437511920929,
      "reward_std": 0.47390592098236084,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7296874821186066,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.28125,
      "epoch": 0.7023319615912208,
      "grad_norm": 4.592452577929141,
      "kl": 0.09130859375,
      "learning_rate": 7.672727272727271e-07,
      "loss": 0.0036,
      "reward": 1.4421875476837158,
      "reward_std": 0.1951931044459343,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.723437488079071,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.875,
      "epoch": 0.7050754458161865,
      "grad_norm": 6.177062965711192,
      "kl": 0.0947265625,
      "learning_rate": 7.663636363636363e-07,
      "loss": 0.0038,
      "reward": 1.3125,
      "reward_std": 0.33411741256713867,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6718749701976776,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.171875,
      "epoch": 0.7078189300411523,
      "grad_norm": 3.819488973591032,
      "kl": 0.09521484375,
      "learning_rate": 7.654545454545455e-07,
      "loss": 0.0038,
      "reward": 1.234375,
      "reward_std": 0.3691229335963726,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.625,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.75,
      "epoch": 0.710562414266118,
      "grad_norm": 8.505930688018147,
      "kl": 0.09716796875,
      "learning_rate": 7.645454545454545e-07,
      "loss": 0.0039,
      "reward": 0.8343750238418579,
      "reward_std": 0.43087735772132874,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.4437499940395355,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.15625,
      "epoch": 0.7133058984910837,
      "grad_norm": 2.619379040722451,
      "kl": 0.081298828125,
      "learning_rate": 7.636363636363636e-07,
      "loss": 0.0032,
      "reward": 1.250781238079071,
      "reward_std": 0.09419287834316492,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6414062082767487,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.078125,
      "epoch": 0.7160493827160493,
      "grad_norm": 24.31183400021532,
      "kl": 0.086181640625,
      "learning_rate": 7.627272727272727e-07,
      "loss": 0.0035,
      "reward": 1.541406273841858,
      "reward_std": 0.3669806718826294,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7601561844348907,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.25,
      "epoch": 0.7187928669410151,
      "grad_norm": 4.330377980133479,
      "kl": 0.085205078125,
      "learning_rate": 7.618181818181818e-07,
      "loss": 0.0034,
      "reward": 1.778124988079071,
      "reward_std": 0.42067764699459076,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8875000178813934,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.4375,
      "epoch": 0.7215363511659808,
      "grad_norm": 2.8368583838801182,
      "kl": 0.067626953125,
      "learning_rate": 7.609090909090909e-07,
      "loss": 0.0027,
      "reward": 1.1468749642372131,
      "reward_std": 0.4227535352110863,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5843749642372131,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.640625,
      "epoch": 0.7242798353909465,
      "grad_norm": 5.131099302078508,
      "kl": 0.078369140625,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0031,
      "reward": 1.178906261920929,
      "reward_std": 0.49694086611270905,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.6007812470197678,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.703125,
      "epoch": 0.7270233196159122,
      "grad_norm": 7.5594308912964765,
      "kl": 0.093505859375,
      "learning_rate": 7.59090909090909e-07,
      "loss": 0.0037,
      "reward": 1.3468750715255737,
      "reward_std": 0.42676912248134613,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6906249225139618,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.359375,
      "epoch": 0.7297668038408779,
      "grad_norm": 3.699739470898038,
      "kl": 0.081298828125,
      "learning_rate": 7.581818181818182e-07,
      "loss": 0.0033,
      "reward": 1.2773436903953552,
      "reward_std": 0.6463219225406647,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6523437798023224,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.75,
      "epoch": 0.7325102880658436,
      "grad_norm": 5.612634899949188,
      "kl": 0.096923828125,
      "learning_rate": 7.572727272727272e-07,
      "loss": 0.0039,
      "reward": 0.8046875,
      "reward_std": 0.4974658638238907,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.4140624403953552,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.34375,
      "epoch": 0.7352537722908093,
      "grad_norm": 3.9276511293328675,
      "kl": 0.1044921875,
      "learning_rate": 7.563636363636363e-07,
      "loss": 0.0042,
      "reward": 1.5101561546325684,
      "reward_std": 0.4631221741437912,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7132812440395355,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.390625,
      "epoch": 0.7379972565157751,
      "grad_norm": 7.060213562987752,
      "kl": 0.090087890625,
      "learning_rate": 7.554545454545455e-07,
      "loss": 0.0036,
      "reward": 1.3593750596046448,
      "reward_std": 0.586755782365799,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6874999403953552,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.9375,
      "epoch": 0.7407407407407407,
      "grad_norm": 5.23495205214961,
      "kl": 0.0888671875,
      "learning_rate": 7.545454545454545e-07,
      "loss": 0.0036,
      "reward": 1.020312488079071,
      "reward_std": 0.5529607236385345,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.48906248807907104,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.796875,
      "epoch": 0.7434842249657064,
      "grad_norm": 4.078711855133753,
      "kl": 0.126220703125,
      "learning_rate": 7.536363636363636e-07,
      "loss": 0.0051,
      "reward": 1.4265625476837158,
      "reward_std": 0.6099326908588409,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7234375178813934,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 98.234375,
      "epoch": 0.7462277091906722,
      "grad_norm": 5.828898668341959,
      "kl": 0.0631103515625,
      "learning_rate": 7.527272727272727e-07,
      "loss": 0.0025,
      "reward": 1.5179687142372131,
      "reward_std": 0.5480734258890152,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7523437142372131,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.484375,
      "epoch": 0.7489711934156379,
      "grad_norm": 6.955776732075884,
      "kl": 0.087890625,
      "learning_rate": 7.518181818181818e-07,
      "loss": 0.0035,
      "reward": 1.2203125357627869,
      "reward_std": 0.40472525358200073,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6265624761581421,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.140625,
      "epoch": 0.7517146776406035,
      "grad_norm": 3.884958639901895,
      "kl": 0.08935546875,
      "learning_rate": 7.509090909090908e-07,
      "loss": 0.0036,
      "reward": 1.6171875596046448,
      "reward_std": 0.43695899844169617,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8046874701976776,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 96.703125,
      "epoch": 0.7544581618655692,
      "grad_norm": 3.7953559845276397,
      "kl": 0.089111328125,
      "learning_rate": 7.5e-07,
      "loss": 0.0036,
      "reward": 1.545312523841858,
      "reward_std": 0.5323244035243988,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7796874940395355,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.625,
      "epoch": 0.757201646090535,
      "grad_norm": 4.135276215930814,
      "kl": 0.100830078125,
      "learning_rate": 7.490909090909091e-07,
      "loss": 0.004,
      "reward": 1.137499988079071,
      "reward_std": 0.5229451805353165,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.574999988079071,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.28125,
      "epoch": 0.7599451303155007,
      "grad_norm": 3.7538164014579056,
      "kl": 0.091552734375,
      "learning_rate": 7.481818181818182e-07,
      "loss": 0.0037,
      "reward": 1.39453125,
      "reward_std": 0.6234084516763687,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6914062798023224,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.359375,
      "epoch": 0.7626886145404664,
      "grad_norm": 5.0738501302107855,
      "kl": 0.0986328125,
      "learning_rate": 7.472727272727272e-07,
      "loss": 0.0039,
      "reward": 1.0726562142372131,
      "reward_std": 0.8105998933315277,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5414062440395355,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.765625,
      "epoch": 0.7654320987654321,
      "grad_norm": 3.458442593359515,
      "kl": 0.0830078125,
      "learning_rate": 7.463636363636363e-07,
      "loss": 0.0033,
      "reward": 1.3437500596046448,
      "reward_std": 0.642494797706604,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6874999701976776,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.265625,
      "epoch": 0.7681755829903978,
      "grad_norm": 4.725759896770685,
      "kl": 0.083251953125,
      "learning_rate": 7.454545454545455e-07,
      "loss": 0.0033,
      "reward": 1.1656250357627869,
      "reward_std": 0.3226143643260002,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.6031249761581421,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.25,
      "epoch": 0.7709190672153635,
      "grad_norm": 3.678317888450733,
      "kl": 0.1083984375,
      "learning_rate": 7.445454545454544e-07,
      "loss": 0.0043,
      "reward": 1.1148437559604645,
      "reward_std": 0.4589616358280182,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5679687410593033,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.921875,
      "epoch": 0.7736625514403292,
      "grad_norm": 14.099429377447535,
      "kl": 0.08740234375,
      "learning_rate": 7.436363636363636e-07,
      "loss": 0.0035,
      "reward": 1.4703125357627869,
      "reward_std": 0.4954650402069092,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7359375059604645,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.359375,
      "epoch": 0.7764060356652949,
      "grad_norm": 3.358461975282723,
      "kl": 0.085693359375,
      "learning_rate": 7.427272727272727e-07,
      "loss": 0.0034,
      "reward": 1.1437499523162842,
      "reward_std": 0.45597282610833645,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.581250011920929,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.359375,
      "epoch": 0.7791495198902606,
      "grad_norm": 5.233594477059757,
      "kl": 0.095947265625,
      "learning_rate": 7.418181818181818e-07,
      "loss": 0.0038,
      "reward": 1.33984375,
      "reward_std": 0.6549213230609894,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6835937201976776,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.46875,
      "epoch": 0.7818930041152263,
      "grad_norm": 2.478879945186959,
      "kl": 0.101318359375,
      "learning_rate": 7.409090909090909e-07,
      "loss": 0.0041,
      "reward": 1.7281250357627869,
      "reward_std": 0.37530940771102905,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8531250059604645,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.0625,
      "epoch": 0.7846364883401921,
      "grad_norm": 2.9609063225313297,
      "kl": 0.099609375,
      "learning_rate": 7.4e-07,
      "loss": 0.004,
      "reward": 1.2867187857627869,
      "reward_std": 0.5633288323879242,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6617186963558197,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.59375,
      "epoch": 0.7873799725651578,
      "grad_norm": 3.84297620567891,
      "kl": 0.09228515625,
      "learning_rate": 7.390909090909091e-07,
      "loss": 0.0037,
      "reward": 1.453125,
      "reward_std": 0.5088267177343369,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.734375,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.484375,
      "epoch": 0.7901234567901234,
      "grad_norm": 1.732041600478566,
      "kl": 0.114013671875,
      "learning_rate": 7.381818181818182e-07,
      "loss": 0.0046,
      "reward": 1.253125011920929,
      "reward_std": 0.20044078677892685,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6437499821186066,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.890625,
      "epoch": 0.7928669410150891,
      "grad_norm": 3.494611350142717,
      "kl": 0.098388671875,
      "learning_rate": 7.372727272727272e-07,
      "loss": 0.0039,
      "reward": 1.0382812321186066,
      "reward_std": 0.7175245881080627,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5382812023162842,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.03125,
      "epoch": 0.7956104252400549,
      "grad_norm": 4.409782914550581,
      "kl": 0.09814453125,
      "learning_rate": 7.363636363636363e-07,
      "loss": 0.0039,
      "reward": 1.0171875357627869,
      "reward_std": 0.4266436994075775,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.5328124761581421,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.3125,
      "epoch": 0.7983539094650206,
      "grad_norm": 2.8626066477859005,
      "kl": 0.09521484375,
      "learning_rate": 7.354545454545455e-07,
      "loss": 0.0038,
      "reward": 1.479687511920929,
      "reward_std": 0.4619990587234497,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.745312511920929,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.609375,
      "epoch": 0.8010973936899863,
      "grad_norm": 4.176612663324125,
      "kl": 0.095703125,
      "learning_rate": 7.345454545454544e-07,
      "loss": 0.0038,
      "reward": 1.2265625,
      "reward_std": 0.554766446352005,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6328125,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.90625,
      "epoch": 0.803840877914952,
      "grad_norm": 2.4249557893758578,
      "kl": 0.109130859375,
      "learning_rate": 7.336363636363636e-07,
      "loss": 0.0044,
      "reward": 1.7718749642372131,
      "reward_std": 0.21178293507546186,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8812499940395355,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.703125,
      "epoch": 0.8065843621399177,
      "grad_norm": 2.871803780416522,
      "kl": 0.09130859375,
      "learning_rate": 7.327272727272728e-07,
      "loss": 0.0037,
      "reward": 1.3429687321186066,
      "reward_std": 0.31595955044031143,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6710937321186066,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.3125,
      "epoch": 0.8093278463648834,
      "grad_norm": 10.806987704832263,
      "kl": 0.1357421875,
      "learning_rate": 7.318181818181818e-07,
      "loss": 0.0054,
      "reward": 1.2625000476837158,
      "reward_std": 0.6731216311454773,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6375000178813934,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.375,
      "epoch": 0.8120713305898491,
      "grad_norm": 3.46173976943176,
      "kl": 0.12841796875,
      "learning_rate": 7.309090909090909e-07,
      "loss": 0.0051,
      "reward": 1.702343761920929,
      "reward_std": 0.3345009386539459,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8585937321186066,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.453125,
      "epoch": 0.8148148148148148,
      "grad_norm": 3.9733750299455375,
      "kl": 0.081298828125,
      "learning_rate": 7.3e-07,
      "loss": 0.0033,
      "reward": 1.6203124523162842,
      "reward_std": 0.5902264714241028,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7921874821186066,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.25,
      "epoch": 0.8175582990397805,
      "grad_norm": 12.925700937011783,
      "kl": 0.088623046875,
      "learning_rate": 7.290909090909091e-07,
      "loss": 0.0035,
      "reward": 1.739843726158142,
      "reward_std": 0.4693824499845505,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8648437261581421,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.453125,
      "epoch": 0.8203017832647462,
      "grad_norm": 2.1252410795267442,
      "kl": 0.08349609375,
      "learning_rate": 7.281818181818181e-07,
      "loss": 0.0034,
      "reward": 1.6500000357627869,
      "reward_std": 0.365024596452713,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8218750059604645,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.375,
      "epoch": 0.823045267489712,
      "grad_norm": 2.9274319672566915,
      "kl": 0.091552734375,
      "learning_rate": 7.272727272727272e-07,
      "loss": 0.0037,
      "reward": 1.592187523841858,
      "reward_std": 0.3340279906988144,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7953124940395355,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.046875,
      "epoch": 0.8257887517146777,
      "grad_norm": 3.5682926071799645,
      "kl": 0.094482421875,
      "learning_rate": 7.263636363636363e-07,
      "loss": 0.0038,
      "reward": 1.622656226158142,
      "reward_std": 0.5170932710170746,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8101562559604645,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.1875,
      "epoch": 0.8285322359396433,
      "grad_norm": 4.223087291480602,
      "kl": 0.084228515625,
      "learning_rate": 7.254545454545455e-07,
      "loss": 0.0034,
      "reward": 1.1343750357627869,
      "reward_std": 0.49241510033607483,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5874999463558197,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.359375,
      "epoch": 0.831275720164609,
      "grad_norm": 2.86803311983425,
      "kl": 0.103271484375,
      "learning_rate": 7.245454545454544e-07,
      "loss": 0.0041,
      "reward": 1.5828125476837158,
      "reward_std": 0.43118855357170105,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.801562488079071,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.609375,
      "epoch": 0.8340192043895748,
      "grad_norm": 2.5900651993580857,
      "kl": 0.093505859375,
      "learning_rate": 7.236363636363636e-07,
      "loss": 0.0037,
      "reward": 1.3578124642372131,
      "reward_std": 0.3414458632469177,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.109375,
      "epoch": 0.8367626886145405,
      "grad_norm": 19.974214898027473,
      "kl": 0.086181640625,
      "learning_rate": 7.227272727272728e-07,
      "loss": 0.0034,
      "reward": 1.682812511920929,
      "reward_std": 0.46156272292137146,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.839062511920929,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.59375,
      "epoch": 0.8395061728395061,
      "grad_norm": 4.462795625676432,
      "kl": 0.104736328125,
      "learning_rate": 7.218181818181817e-07,
      "loss": 0.0042,
      "reward": 1.474218726158142,
      "reward_std": 0.4678943157196045,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7398437261581421,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.515625,
      "epoch": 0.8422496570644719,
      "grad_norm": 5.604391149460443,
      "kl": 0.084228515625,
      "learning_rate": 7.209090909090909e-07,
      "loss": 0.0034,
      "reward": 1.217968761920929,
      "reward_std": 0.5433058440685272,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6085937023162842,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.109375,
      "epoch": 0.8449931412894376,
      "grad_norm": 6.693685254800663,
      "kl": 0.09228515625,
      "learning_rate": 7.2e-07,
      "loss": 0.0037,
      "reward": 1.3624999523162842,
      "reward_std": 0.7833757400512695,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.690625011920929,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.640625,
      "epoch": 0.8477366255144033,
      "grad_norm": 8.54674736781978,
      "kl": 0.079833984375,
      "learning_rate": 7.190909090909091e-07,
      "loss": 0.0032,
      "reward": 1.043749988079071,
      "reward_std": 0.457779660820961,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5437500178813934,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.84375,
      "epoch": 0.850480109739369,
      "grad_norm": 2.4299194682326593,
      "kl": 0.080078125,
      "learning_rate": 7.181818181818181e-07,
      "loss": 0.0032,
      "reward": 1.550000011920929,
      "reward_std": 0.36771687865257263,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.784375011920929,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.421875,
      "epoch": 0.8532235939643347,
      "grad_norm": 9.62096693731744,
      "kl": 0.12158203125,
      "learning_rate": 7.172727272727272e-07,
      "loss": 0.0049,
      "reward": 1.089062511920929,
      "reward_std": 0.784207671880722,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5578124821186066,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.375,
      "epoch": 0.8559670781893004,
      "grad_norm": 3.791843879642304,
      "kl": 0.09521484375,
      "learning_rate": 7.163636363636363e-07,
      "loss": 0.0038,
      "reward": 1.3015625476837158,
      "reward_std": 0.3239322304725647,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.660937488079071,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.46875,
      "epoch": 0.8587105624142661,
      "grad_norm": 4.566693887033749,
      "kl": 0.08251953125,
      "learning_rate": 7.154545454545454e-07,
      "loss": 0.0033,
      "reward": 1.403124988079071,
      "reward_std": 0.6231254190206528,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.715624988079071,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.65625,
      "epoch": 0.8614540466392319,
      "grad_norm": 18.23495938082825,
      "kl": 0.117431640625,
      "learning_rate": 7.145454545454545e-07,
      "loss": 0.0047,
      "reward": 1.0859375,
      "reward_std": 0.6859345436096191,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5546874403953552,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.234375,
      "epoch": 0.8641975308641975,
      "grad_norm": 4.832892304043935,
      "kl": 0.100341796875,
      "learning_rate": 7.136363636363636e-07,
      "loss": 0.004,
      "reward": 1.6624999642372131,
      "reward_std": 0.5350576937198639,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8343749940395355,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.40625,
      "epoch": 0.8669410150891632,
      "grad_norm": 3.296079249308525,
      "kl": 0.103759765625,
      "learning_rate": 7.127272727272728e-07,
      "loss": 0.0041,
      "reward": 1.396875023841858,
      "reward_std": 0.6269551515579224,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7093749642372131,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.859375,
      "epoch": 0.869684499314129,
      "grad_norm": 3.8568552276327703,
      "kl": 0.0849609375,
      "learning_rate": 7.118181818181817e-07,
      "loss": 0.0034,
      "reward": 1.890625,
      "reward_std": 0.20640429854393005,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9375,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.546875,
      "epoch": 0.8724279835390947,
      "grad_norm": 4.024591910607265,
      "kl": 0.09814453125,
      "learning_rate": 7.109090909090909e-07,
      "loss": 0.0039,
      "reward": 1.6203124523162842,
      "reward_std": 0.611024409532547,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7921874821186066,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.0,
      "epoch": 0.8751714677640604,
      "grad_norm": 4.977602479562242,
      "kl": 0.10791015625,
      "learning_rate": 7.1e-07,
      "loss": 0.0043,
      "reward": 1.431249976158142,
      "reward_std": 0.7265175879001617,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.6968749761581421,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.671875,
      "epoch": 0.877914951989026,
      "grad_norm": 12.201487507644098,
      "kl": 0.09423828125,
      "learning_rate": 7.09090909090909e-07,
      "loss": 0.0038,
      "reward": 1.667187511920929,
      "reward_std": 0.4393137991428375,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.823437511920929,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.609375,
      "epoch": 0.8806584362139918,
      "grad_norm": 5.6109455126827426,
      "kl": 0.095947265625,
      "learning_rate": 7.081818181818181e-07,
      "loss": 0.0038,
      "reward": 1.578125,
      "reward_std": 0.4120820192620158,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7968749701976776,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.28125,
      "epoch": 0.8834019204389575,
      "grad_norm": 3.029237010880203,
      "kl": 0.079833984375,
      "learning_rate": 7.072727272727272e-07,
      "loss": 0.0032,
      "reward": 1.500781238079071,
      "reward_std": 0.599192276597023,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7351562678813934,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.90625,
      "epoch": 0.8861454046639232,
      "grad_norm": 3.0855938992202625,
      "kl": 0.094970703125,
      "learning_rate": 7.063636363636364e-07,
      "loss": 0.0038,
      "reward": 1.278906226158142,
      "reward_std": 0.3176858276128769,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6382812559604645,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.328125,
      "epoch": 0.8888888888888888,
      "grad_norm": 6.174475362725212,
      "kl": 0.104736328125,
      "learning_rate": 7.054545454545454e-07,
      "loss": 0.0042,
      "reward": 1.3726562857627869,
      "reward_std": 0.6849194169044495,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.7007812559604645,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.578125,
      "epoch": 0.8916323731138546,
      "grad_norm": 4.340367067952867,
      "kl": 0.09912109375,
      "learning_rate": 7.045454545454545e-07,
      "loss": 0.004,
      "reward": 1.42578125,
      "reward_std": 0.645444929599762,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7226562201976776,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.9375,
      "epoch": 0.8943758573388203,
      "grad_norm": 2.8142124440241765,
      "kl": 0.099853515625,
      "learning_rate": 7.036363636363636e-07,
      "loss": 0.004,
      "reward": 1.2023437321186066,
      "reward_std": 0.4010866731405258,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6085937321186066,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.984375,
      "epoch": 0.897119341563786,
      "grad_norm": 9.240569190949303,
      "kl": 0.097412109375,
      "learning_rate": 7.027272727272728e-07,
      "loss": 0.0039,
      "reward": 1.3351562023162842,
      "reward_std": 0.7516996264457703,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6789062321186066,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.0625,
      "epoch": 0.8998628257887518,
      "grad_norm": 4.041779286763977,
      "kl": 0.119873046875,
      "learning_rate": 7.018181818181817e-07,
      "loss": 0.0048,
      "reward": 1.420312523841858,
      "reward_std": 0.3459370732307434,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7171874940395355,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.796875,
      "epoch": 0.9026063100137174,
      "grad_norm": 4.242021387631853,
      "kl": 0.093017578125,
      "learning_rate": 7.009090909090909e-07,
      "loss": 0.0037,
      "reward": 1.607812523841858,
      "reward_std": 0.4460008144378662,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7953125238418579,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.609375,
      "epoch": 0.9053497942386831,
      "grad_norm": 4.9465256422264945,
      "kl": 0.1025390625,
      "learning_rate": 7e-07,
      "loss": 0.0041,
      "reward": 1.5195311903953552,
      "reward_std": 0.3289015144109726,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7695312201976776,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.78125,
      "epoch": 0.9080932784636488,
      "grad_norm": 3.4771150742508716,
      "kl": 0.083984375,
      "learning_rate": 6.99090909090909e-07,
      "loss": 0.0034,
      "reward": 1.7789062857627869,
      "reward_std": 0.4107031673192978,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8882812857627869,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.3125,
      "epoch": 0.9108367626886146,
      "grad_norm": 3.4677948106320637,
      "kl": 0.085693359375,
      "learning_rate": 6.981818181818182e-07,
      "loss": 0.0034,
      "reward": 1.2460936903953552,
      "reward_std": 0.6076911389827728,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.63671875,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.0,
      "epoch": 0.9135802469135802,
      "grad_norm": 23.080171367895154,
      "kl": 0.130615234375,
      "learning_rate": 6.972727272727273e-07,
      "loss": 0.0052,
      "reward": 1.426562488079071,
      "reward_std": 0.5667466223239899,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7234375178813934,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.53125,
      "epoch": 0.9163237311385459,
      "grad_norm": 3.5573105855329077,
      "kl": 0.101318359375,
      "learning_rate": 6.963636363636364e-07,
      "loss": 0.0041,
      "reward": 1.153124988079071,
      "reward_std": 0.442140594124794,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5906249582767487,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.3125,
      "epoch": 0.9190672153635117,
      "grad_norm": 16.20583521957908,
      "kl": 0.09375,
      "learning_rate": 6.954545454545454e-07,
      "loss": 0.0037,
      "reward": 1.2859374284744263,
      "reward_std": 0.5812228620052338,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6453124582767487,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.859375,
      "epoch": 0.9218106995884774,
      "grad_norm": 8.148470377319255,
      "kl": 0.0859375,
      "learning_rate": 6.945454545454545e-07,
      "loss": 0.0034,
      "reward": 1.262499988079071,
      "reward_std": 0.5951336622238159,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6218749582767487,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.234375,
      "epoch": 0.9245541838134431,
      "grad_norm": 2.4416096060166383,
      "kl": 0.09228515625,
      "learning_rate": 6.936363636363636e-07,
      "loss": 0.0037,
      "reward": 1.668749988079071,
      "reward_std": 0.3197062835097313,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.824999988079071,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.796875,
      "epoch": 0.9272976680384087,
      "grad_norm": 3.2162665539035724,
      "kl": 0.0927734375,
      "learning_rate": 6.927272727272727e-07,
      "loss": 0.0037,
      "reward": 1.6093750596046448,
      "reward_std": 0.4507560729980469,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8125,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.046875,
      "epoch": 0.9300411522633745,
      "grad_norm": 17.23422076000473,
      "kl": 0.080810546875,
      "learning_rate": 6.918181818181817e-07,
      "loss": 0.0032,
      "reward": 1.639843761920929,
      "reward_std": 0.292666919529438,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.827343761920929,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.125,
      "epoch": 0.9327846364883402,
      "grad_norm": 6.520218077277935,
      "kl": 0.1044921875,
      "learning_rate": 6.909090909090909e-07,
      "loss": 0.0042,
      "reward": 1.5898436903953552,
      "reward_std": 0.5847151577472687,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7929687201976776,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.984375,
      "epoch": 0.9355281207133059,
      "grad_norm": 3.632307809145378,
      "kl": 0.103759765625,
      "learning_rate": 6.9e-07,
      "loss": 0.0041,
      "reward": 0.9554687440395355,
      "reward_std": 0.5614143311977386,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.5023437440395355,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.28125,
      "epoch": 0.9382716049382716,
      "grad_norm": 6.791169700208845,
      "kl": 0.107421875,
      "learning_rate": 6.89090909090909e-07,
      "loss": 0.0043,
      "reward": 1.4710937142372131,
      "reward_std": 0.30878862738609314,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7367187440395355,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.15625,
      "epoch": 0.9410150891632373,
      "grad_norm": 9.180934723679384,
      "kl": 0.102294921875,
      "learning_rate": 6.881818181818182e-07,
      "loss": 0.0041,
      "reward": 1.016406238079071,
      "reward_std": 0.5839601010084152,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.532031238079071,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.375,
      "epoch": 0.943758573388203,
      "grad_norm": 3.256295819561447,
      "kl": 0.108154296875,
      "learning_rate": 6.872727272727273e-07,
      "loss": 0.0043,
      "reward": 1.2203125357627869,
      "reward_std": 0.5046594589948654,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6265624761581421,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.453125,
      "epoch": 0.9465020576131687,
      "grad_norm": 3.6115730033643945,
      "kl": 0.1015625,
      "learning_rate": 6.863636363636363e-07,
      "loss": 0.0041,
      "reward": 1.5234375,
      "reward_std": 0.6625894606113434,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7734375,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.078125,
      "epoch": 0.9492455418381345,
      "grad_norm": 4.100928270883839,
      "kl": 0.11083984375,
      "learning_rate": 6.854545454545454e-07,
      "loss": 0.0044,
      "reward": 1.309374988079071,
      "reward_std": 0.3420407697558403,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6531250178813934,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.796875,
      "epoch": 0.9519890260631001,
      "grad_norm": 6.918298580051922,
      "kl": 0.106689453125,
      "learning_rate": 6.845454545454545e-07,
      "loss": 0.0043,
      "reward": 1.548437476158142,
      "reward_std": 0.20821654796600342,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7671874761581421,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.90625,
      "epoch": 0.9547325102880658,
      "grad_norm": 6.920880779530441,
      "kl": 0.10791015625,
      "learning_rate": 6.836363636363636e-07,
      "loss": 0.0043,
      "reward": 1.535937488079071,
      "reward_std": 0.7484138607978821,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.770312488079071,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.390625,
      "epoch": 0.9574759945130316,
      "grad_norm": 3.228855877922397,
      "kl": 0.125,
      "learning_rate": 6.827272727272727e-07,
      "loss": 0.005,
      "reward": 1.6398437023162842,
      "reward_std": 0.4850577563047409,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8273437321186066,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.265625,
      "epoch": 0.9602194787379973,
      "grad_norm": 1.4808004667582153,
      "kl": 0.11572265625,
      "learning_rate": 6.818181818181817e-07,
      "loss": 0.0046,
      "reward": 1.821874976158142,
      "reward_std": 0.10994113236665726,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9156250059604645,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.140625,
      "epoch": 0.9629629629629629,
      "grad_norm": 3.4206053989322953,
      "kl": 0.115234375,
      "learning_rate": 6.809090909090909e-07,
      "loss": 0.0046,
      "reward": 1.239062488079071,
      "reward_std": 0.6298282146453857,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6296875178813934,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.375,
      "epoch": 0.9657064471879286,
      "grad_norm": 3.82268090438329,
      "kl": 0.1162109375,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0047,
      "reward": 1.609375,
      "reward_std": 0.6609872579574585,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8125,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.734375,
      "epoch": 0.9684499314128944,
      "grad_norm": 4.857664777910766,
      "kl": 0.14697265625,
      "learning_rate": 6.79090909090909e-07,
      "loss": 0.0059,
      "reward": 1.412500023841858,
      "reward_std": 0.46737395226955414,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7093750238418579,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.421875,
      "epoch": 0.9711934156378601,
      "grad_norm": 3.6194631196809253,
      "kl": 0.105712890625,
      "learning_rate": 6.781818181818182e-07,
      "loss": 0.0042,
      "reward": 1.5984375476837158,
      "reward_std": 0.5569853782653809,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.801562488079071,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.078125,
      "epoch": 0.9739368998628258,
      "grad_norm": 4.709451670730105,
      "kl": 0.10400390625,
      "learning_rate": 6.772727272727273e-07,
      "loss": 0.0042,
      "reward": 1.487500011920929,
      "reward_std": 0.5324640423059464,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.721875011920929,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.375,
      "epoch": 0.9766803840877915,
      "grad_norm": 6.2815158563240825,
      "kl": 0.12548828125,
      "learning_rate": 6.763636363636363e-07,
      "loss": 0.005,
      "reward": 1.3414062559604645,
      "reward_std": 0.4287596568465233,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6851562559604645,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.046875,
      "epoch": 0.9794238683127572,
      "grad_norm": 4.754282552821982,
      "kl": 0.13671875,
      "learning_rate": 6.754545454545454e-07,
      "loss": 0.0055,
      "reward": 1.1171875,
      "reward_std": 0.480400413274765,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5703125,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.03125,
      "epoch": 0.9821673525377229,
      "grad_norm": 3.161618547824301,
      "kl": 0.1015625,
      "learning_rate": 6.745454545454545e-07,
      "loss": 0.0041,
      "reward": 1.801562488079071,
      "reward_std": 0.3133033514022827,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8953125178813934,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.3125,
      "epoch": 0.9849108367626886,
      "grad_norm": 3.137174508984925,
      "kl": 0.096435546875,
      "learning_rate": 6.736363636363635e-07,
      "loss": 0.0039,
      "reward": 1.4281249940395355,
      "reward_std": 0.37352126836776733,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7249999940395355,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.859375,
      "epoch": 0.9876543209876543,
      "grad_norm": 5.442071087429494,
      "kl": 0.10009765625,
      "learning_rate": 6.727272727272727e-07,
      "loss": 0.004,
      "reward": 0.9671874940395355,
      "reward_std": 0.6916260123252869,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.49843746423721313,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.21875,
      "epoch": 0.99039780521262,
      "grad_norm": 6.404313944657274,
      "kl": 0.0986328125,
      "learning_rate": 6.718181818181818e-07,
      "loss": 0.0039,
      "reward": 1.4789062142372131,
      "reward_std": 0.6432763636112213,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7445312440395355,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.765625,
      "epoch": 0.9931412894375857,
      "grad_norm": 11.533715639852097,
      "kl": 0.114990234375,
      "learning_rate": 6.709090909090909e-07,
      "loss": 0.0046,
      "reward": 1.285937488079071,
      "reward_std": 0.7446453869342804,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6609374582767487,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.171875,
      "epoch": 0.9958847736625515,
      "grad_norm": 7.3920091754927375,
      "kl": 0.095947265625,
      "learning_rate": 6.7e-07,
      "loss": 0.0038,
      "reward": 1.6062500476837158,
      "reward_std": 0.5926486253738403,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8093749582767487,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 104.0,
      "epoch": 0.9986282578875172,
      "grad_norm": 3.98245690956048,
      "kl": 0.085693359375,
      "learning_rate": 6.69090909090909e-07,
      "loss": 0.0038,
      "reward": 1.050000000745058,
      "reward_std": 0.999468058347702,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5500000007450581,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.71875,
      "epoch": 1.0027434842249656,
      "grad_norm": 2.911604760532795,
      "kl": 0.09619140625,
      "learning_rate": 6.681818181818182e-07,
      "loss": 0.0038,
      "reward": 1.442968726158142,
      "reward_std": 0.35979723185300827,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7242186963558197,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.203125,
      "epoch": 1.0054869684499315,
      "grad_norm": 7.010322344814963,
      "kl": 0.08837890625,
      "learning_rate": 6.672727272727273e-07,
      "loss": 0.0035,
      "reward": 1.3468749523162842,
      "reward_std": 0.5796718001365662,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6906249821186066,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.625,
      "epoch": 1.008230452674897,
      "grad_norm": 4.912622392676742,
      "kl": 0.08544921875,
      "learning_rate": 6.663636363636363e-07,
      "loss": 0.0034,
      "reward": 1.3984375,
      "reward_std": 0.5179575830698013,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6953125,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.65625,
      "epoch": 1.010973936899863,
      "grad_norm": 2.0977130478706805,
      "kl": 0.12109375,
      "learning_rate": 6.654545454545454e-07,
      "loss": 0.0048,
      "reward": 1.6640625,
      "reward_std": 0.2591197192668915,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8359375,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.5625,
      "epoch": 1.0137174211248285,
      "grad_norm": 3.2382017857278753,
      "kl": 0.1162109375,
      "learning_rate": 6.645454545454545e-07,
      "loss": 0.0047,
      "reward": 1.5703125,
      "reward_std": 0.5361149609088898,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7890624701976776,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.046875,
      "epoch": 1.0164609053497942,
      "grad_norm": 7.678867298043466,
      "kl": 0.100341796875,
      "learning_rate": 6.636363636363636e-07,
      "loss": 0.004,
      "reward": 1.40234375,
      "reward_std": 0.6340524554252625,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7148437201976776,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.734375,
      "epoch": 1.01920438957476,
      "grad_norm": 4.289702688201466,
      "kl": 0.08544921875,
      "learning_rate": 6.627272727272727e-07,
      "loss": 0.0034,
      "reward": 1.598437488079071,
      "reward_std": 0.34804536402225494,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.801562488079071,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.53125,
      "epoch": 1.0219478737997256,
      "grad_norm": 8.3854367781373,
      "kl": 0.080810546875,
      "learning_rate": 6.618181818181818e-07,
      "loss": 0.0032,
      "reward": 1.3406250476837158,
      "reward_std": 0.6063534319400787,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.684374988079071,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.46875,
      "epoch": 1.0246913580246915,
      "grad_norm": 2.8046480040356845,
      "kl": 0.08154296875,
      "learning_rate": 6.609090909090909e-07,
      "loss": 0.0033,
      "reward": 1.193750023841858,
      "reward_std": 0.30354222655296326,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.5843749940395355,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.0,
      "epoch": 1.027434842249657,
      "grad_norm": 2.762046165937544,
      "kl": 0.095703125,
      "learning_rate": 6.6e-07,
      "loss": 0.0038,
      "reward": 1.550000011920929,
      "reward_std": 0.09536151215434074,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.784375011920929,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.171875,
      "epoch": 1.0301783264746227,
      "grad_norm": 12.076907213539458,
      "kl": 0.0986328125,
      "learning_rate": 6.59090909090909e-07,
      "loss": 0.0039,
      "reward": 1.504687488079071,
      "reward_std": 0.5868997871875763,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.754687488079071,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.265625,
      "epoch": 1.0329218106995885,
      "grad_norm": 3.580412178061941,
      "kl": 0.091064453125,
      "learning_rate": 6.581818181818182e-07,
      "loss": 0.0036,
      "reward": 1.498437523841858,
      "reward_std": 0.6327075660228729,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7328124642372131,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.390625,
      "epoch": 1.0356652949245542,
      "grad_norm": 13.372269942081257,
      "kl": 0.113525390625,
      "learning_rate": 6.572727272727272e-07,
      "loss": 0.0045,
      "reward": 1.045312523841858,
      "reward_std": 0.3369172066450119,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5453125089406967,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.328125,
      "epoch": 1.0384087791495198,
      "grad_norm": 5.734247996169968,
      "kl": 0.112548828125,
      "learning_rate": 6.563636363636363e-07,
      "loss": 0.0045,
      "reward": 1.5640625357627869,
      "reward_std": 0.2916586846113205,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7828125059604645,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.65625,
      "epoch": 1.0411522633744856,
      "grad_norm": 3.5003013068620623,
      "kl": 0.098388671875,
      "learning_rate": 6.554545454545454e-07,
      "loss": 0.0039,
      "reward": 1.4835937023162842,
      "reward_std": 0.49376851320266724,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7492187023162842,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.578125,
      "epoch": 1.0438957475994513,
      "grad_norm": 3.90417718511622,
      "kl": 0.105712890625,
      "learning_rate": 6.545454545454546e-07,
      "loss": 0.0042,
      "reward": 1.2640624642372131,
      "reward_std": 0.7536377906799316,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6390624642372131,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.640625,
      "epoch": 1.046639231824417,
      "grad_norm": 2.516289316470427,
      "kl": 0.07958984375,
      "learning_rate": 6.536363636363636e-07,
      "loss": 0.0032,
      "reward": 1.553125023841858,
      "reward_std": 0.42002078890800476,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7875000238418579,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.046875,
      "epoch": 1.0493827160493827,
      "grad_norm": 8.187450982168903,
      "kl": 0.114013671875,
      "learning_rate": 6.527272727272727e-07,
      "loss": 0.0046,
      "reward": 1.834375023841858,
      "reward_std": 0.36994873732328415,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9125000238418579,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.90625,
      "epoch": 1.0521262002743483,
      "grad_norm": 4.6540049962198795,
      "kl": 0.10986328125,
      "learning_rate": 6.518181818181818e-07,
      "loss": 0.0044,
      "reward": 1.489062488079071,
      "reward_std": 0.3555447794497013,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.754687488079071,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.921875,
      "epoch": 1.0548696844993142,
      "grad_norm": 2.9326006901858266,
      "kl": 0.148193359375,
      "learning_rate": 6.509090909090908e-07,
      "loss": 0.0059,
      "reward": 0.985156238079071,
      "reward_std": 0.43424905836582184,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.5164062082767487,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.421875,
      "epoch": 1.0576131687242798,
      "grad_norm": 13.627648816241393,
      "kl": 0.14697265625,
      "learning_rate": 6.5e-07,
      "loss": 0.0059,
      "reward": 1.3671875,
      "reward_std": 0.4714278131723404,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6953125,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.59375,
      "epoch": 1.0603566529492456,
      "grad_norm": 2.028231122702671,
      "kl": 0.09228515625,
      "learning_rate": 6.49090909090909e-07,
      "loss": 0.0037,
      "reward": 1.903124988079071,
      "reward_std": 0.27400387078523636,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.949999988079071,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.859375,
      "epoch": 1.0631001371742113,
      "grad_norm": 2.3011736950684885,
      "kl": 0.121337890625,
      "learning_rate": 6.481818181818182e-07,
      "loss": 0.0048,
      "reward": 1.550000011920929,
      "reward_std": 0.3014083132147789,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7687499821186066,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.21875,
      "epoch": 1.0658436213991769,
      "grad_norm": 4.479044650293866,
      "kl": 0.152099609375,
      "learning_rate": 6.472727272727272e-07,
      "loss": 0.0061,
      "reward": 1.276562511920929,
      "reward_std": 0.5062314867973328,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6515624523162842,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.15625,
      "epoch": 1.0685871056241427,
      "grad_norm": 6.413333872117701,
      "kl": 0.095947265625,
      "learning_rate": 6.463636363636363e-07,
      "loss": 0.0038,
      "reward": 1.745312511920929,
      "reward_std": 0.4144056588411331,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.870312511920929,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.4375,
      "epoch": 1.0713305898491083,
      "grad_norm": 3.9246839250901453,
      "kl": 0.091064453125,
      "learning_rate": 6.454545454545455e-07,
      "loss": 0.0036,
      "reward": 1.639062523841858,
      "reward_std": 0.5730085074901581,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.7953124940395355,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.84375,
      "epoch": 1.074074074074074,
      "grad_norm": 5.849058308896624,
      "kl": 0.122314453125,
      "learning_rate": 6.445454545454546e-07,
      "loss": 0.0049,
      "reward": 1.5796875357627869,
      "reward_std": 0.5560208112001419,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7984374761581421,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.171875,
      "epoch": 1.0768175582990398,
      "grad_norm": 4.424568502709933,
      "kl": 0.10693359375,
      "learning_rate": 6.436363636363636e-07,
      "loss": 0.0043,
      "reward": 1.756250023841858,
      "reward_std": 0.3756431043148041,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8499999940395355,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.09375,
      "epoch": 1.0795610425240054,
      "grad_norm": 3.8477460328846846,
      "kl": 0.1474609375,
      "learning_rate": 6.427272727272727e-07,
      "loss": 0.0059,
      "reward": 1.471875011920929,
      "reward_std": 0.4663201868534088,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.721875011920929,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.75,
      "epoch": 1.0823045267489713,
      "grad_norm": 1.7667673078114647,
      "kl": 0.14453125,
      "learning_rate": 6.418181818181818e-07,
      "loss": 0.0058,
      "reward": 1.610937476158142,
      "reward_std": 0.11885568499565125,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8140625059604645,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.015625,
      "epoch": 1.0850480109739369,
      "grad_norm": 4.041008655001557,
      "kl": 0.120849609375,
      "learning_rate": 6.409090909090908e-07,
      "loss": 0.0048,
      "reward": 1.4820312857627869,
      "reward_std": 0.6050707995891571,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7476562261581421,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.015625,
      "epoch": 1.0877914951989025,
      "grad_norm": 3.9773345438051457,
      "kl": 0.14453125,
      "learning_rate": 6.4e-07,
      "loss": 0.0058,
      "reward": 1.545312523841858,
      "reward_std": 0.5625161677598953,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7640624940395355,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.375,
      "epoch": 1.0905349794238683,
      "grad_norm": 3.6785156309909492,
      "kl": 0.134765625,
      "learning_rate": 6.39090909090909e-07,
      "loss": 0.0054,
      "reward": 1.5703124403953552,
      "reward_std": 0.5366352498531342,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7734375,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.234375,
      "epoch": 1.093278463648834,
      "grad_norm": 9.913354874557868,
      "kl": 0.1259765625,
      "learning_rate": 6.381818181818182e-07,
      "loss": 0.005,
      "reward": 1.0429687201976776,
      "reward_std": 0.3137985020875931,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5429687201976776,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.890625,
      "epoch": 1.0960219478737998,
      "grad_norm": 5.590408639861752,
      "kl": 0.121826171875,
      "learning_rate": 6.372727272727273e-07,
      "loss": 0.0049,
      "reward": 1.198437511920929,
      "reward_std": 0.7430685758590698,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.604687511920929,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.625,
      "epoch": 1.0987654320987654,
      "grad_norm": 5.793826424725683,
      "kl": 0.124755859375,
      "learning_rate": 6.363636363636363e-07,
      "loss": 0.005,
      "reward": 1.696093738079071,
      "reward_std": 0.3521786089986563,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.852343738079071,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.4375,
      "epoch": 1.101508916323731,
      "grad_norm": 6.137194041317214,
      "kl": 0.125,
      "learning_rate": 6.354545454545455e-07,
      "loss": 0.005,
      "reward": 1.3140625357627869,
      "reward_std": 0.519576907157898,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6734374761581421,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.3125,
      "epoch": 1.1042524005486969,
      "grad_norm": 4.912481398404394,
      "kl": 0.130859375,
      "learning_rate": 6.345454545454545e-07,
      "loss": 0.0052,
      "reward": 1.2992187142372131,
      "reward_std": 0.8708893656730652,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6585937440395355,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.953125,
      "epoch": 1.1069958847736625,
      "grad_norm": 4.70721200434814,
      "kl": 0.122314453125,
      "learning_rate": 6.336363636363636e-07,
      "loss": 0.0049,
      "reward": 1.2437500357627869,
      "reward_std": 0.4800318330526352,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6343749761581421,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.796875,
      "epoch": 1.1097393689986284,
      "grad_norm": 3.8086599476638066,
      "kl": 0.128173828125,
      "learning_rate": 6.327272727272727e-07,
      "loss": 0.0051,
      "reward": 1.467968761920929,
      "reward_std": 0.4594145566225052,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.717968761920929,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.75,
      "epoch": 1.112482853223594,
      "grad_norm": 7.097538837072882,
      "kl": 0.154296875,
      "learning_rate": 6.318181818181818e-07,
      "loss": 0.0062,
      "reward": 1.57421875,
      "reward_std": 0.5617561936378479,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.79296875,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.890625,
      "epoch": 1.1152263374485596,
      "grad_norm": 4.337147165160251,
      "kl": 0.130859375,
      "learning_rate": 6.309090909090908e-07,
      "loss": 0.0052,
      "reward": 1.4664062857627869,
      "reward_std": 0.38797758519649506,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7320312857627869,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.25,
      "epoch": 1.1179698216735254,
      "grad_norm": 11.477377991572434,
      "kl": 0.4287109375,
      "learning_rate": 6.3e-07,
      "loss": 0.0171,
      "reward": 1.4914062023162842,
      "reward_std": 0.7070780694484711,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7414062321186066,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.15625,
      "epoch": 1.120713305898491,
      "grad_norm": 3.1650276913328623,
      "kl": 0.1318359375,
      "learning_rate": 6.290909090909091e-07,
      "loss": 0.0053,
      "reward": 1.6742187142372131,
      "reward_std": 0.44198162853717804,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8304687440395355,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.40625,
      "epoch": 1.123456790123457,
      "grad_norm": 3.244376837236869,
      "kl": 0.097412109375,
      "learning_rate": 6.281818181818181e-07,
      "loss": 0.0039,
      "reward": 1.163281261920929,
      "reward_std": 0.21971211954951286,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.600781261920929,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.21875,
      "epoch": 1.1262002743484225,
      "grad_norm": 119.55222008861053,
      "kl": 0.14453125,
      "learning_rate": 6.272727272727273e-07,
      "loss": 0.0058,
      "reward": 1.415624976158142,
      "reward_std": 0.516777977347374,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7124999761581421,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.390625,
      "epoch": 1.1289437585733881,
      "grad_norm": 3.217253515690669,
      "kl": 0.12890625,
      "learning_rate": 6.263636363636363e-07,
      "loss": 0.0051,
      "reward": 1.728906273841858,
      "reward_std": 0.40882396697998047,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8695312738418579,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 60.390625,
      "epoch": 1.131687242798354,
      "grad_norm": 4.822658004657836,
      "kl": 0.15625,
      "learning_rate": 6.254545454545455e-07,
      "loss": 0.0062,
      "reward": 1.4898437559604645,
      "reward_std": 0.09943688940256834,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7554687261581421,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 64.484375,
      "epoch": 1.1344307270233196,
      "grad_norm": 20.237362408713906,
      "kl": 0.17138671875,
      "learning_rate": 6.245454545454545e-07,
      "loss": 0.0068,
      "reward": 1.46875,
      "reward_std": 0.6092181205749512,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.734375,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.515625,
      "epoch": 1.1371742112482854,
      "grad_norm": 3.455645129224603,
      "kl": 0.115478515625,
      "learning_rate": 6.236363636363636e-07,
      "loss": 0.0046,
      "reward": 1.634374976158142,
      "reward_std": 0.4435206800699234,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8062499761581421,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 65.453125,
      "epoch": 1.139917695473251,
      "grad_norm": 6.928489906125386,
      "kl": 0.14599609375,
      "learning_rate": 6.227272727272727e-07,
      "loss": 0.0059,
      "reward": 1.607812523841858,
      "reward_std": 0.3787750005722046,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8109374940395355,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.53125,
      "epoch": 1.1426611796982167,
      "grad_norm": 4.335808496303622,
      "kl": 0.16162109375,
      "learning_rate": 6.218181818181817e-07,
      "loss": 0.0065,
      "reward": 1.370312511920929,
      "reward_std": 0.30234459042549133,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6984374821186066,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.359375,
      "epoch": 1.1454046639231825,
      "grad_norm": 21.102134572561926,
      "kl": 0.1298828125,
      "learning_rate": 6.209090909090908e-07,
      "loss": 0.0052,
      "reward": 1.3117187321186066,
      "reward_std": 0.25837691873311996,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6710937470197678,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.25,
      "epoch": 1.1481481481481481,
      "grad_norm": 4.015658771046256,
      "kl": 0.14990234375,
      "learning_rate": 6.2e-07,
      "loss": 0.006,
      "reward": 1.609375,
      "reward_std": 0.46780863404273987,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8125,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.640625,
      "epoch": 1.1508916323731138,
      "grad_norm": 5.415690577625519,
      "kl": 0.1494140625,
      "learning_rate": 6.190909090909091e-07,
      "loss": 0.006,
      "reward": 1.5187499523162842,
      "reward_std": 0.45385438203811646,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7687499821186066,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.328125,
      "epoch": 1.1536351165980796,
      "grad_norm": 3.3292903149711597,
      "kl": 0.1240234375,
      "learning_rate": 6.181818181818181e-07,
      "loss": 0.005,
      "reward": 1.4609375,
      "reward_std": 0.6869581341743469,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7421874701976776,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.109375,
      "epoch": 1.1563786008230452,
      "grad_norm": 5.649453075128176,
      "kl": 0.119873046875,
      "learning_rate": 6.172727272727273e-07,
      "loss": 0.0048,
      "reward": 1.642187476158142,
      "reward_std": 0.28229840844869614,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8296875059604645,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.3125,
      "epoch": 1.159122085048011,
      "grad_norm": 4.931504755535921,
      "kl": 0.12451171875,
      "learning_rate": 6.163636363636363e-07,
      "loss": 0.005,
      "reward": 1.3453125953674316,
      "reward_std": 0.5185706317424774,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6734375059604645,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.21875,
      "epoch": 1.1618655692729767,
      "grad_norm": 4.230390624232728,
      "kl": 0.14111328125,
      "learning_rate": 6.154545454545455e-07,
      "loss": 0.0056,
      "reward": 1.5476562976837158,
      "reward_std": 0.6381469368934631,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7820312082767487,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.546875,
      "epoch": 1.1646090534979423,
      "grad_norm": 1.9034830148607589,
      "kl": 0.1240234375,
      "learning_rate": 6.145454545454545e-07,
      "loss": 0.005,
      "reward": 1.682812511920929,
      "reward_std": 0.15528614819049835,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.823437511920929,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.859375,
      "epoch": 1.1673525377229081,
      "grad_norm": 8.461247119068744,
      "kl": 0.36181640625,
      "learning_rate": 6.136363636363636e-07,
      "loss": 0.0144,
      "reward": 1.6953125,
      "reward_std": 0.5045858174562454,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8515625,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.65625,
      "epoch": 1.1700960219478738,
      "grad_norm": 3.09962122399241,
      "kl": 0.1318359375,
      "learning_rate": 6.127272727272727e-07,
      "loss": 0.0053,
      "reward": 1.043749988079071,
      "reward_std": 0.39528100192546844,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.543749988079071,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.171875,
      "epoch": 1.1728395061728394,
      "grad_norm": 4.196491201372685,
      "kl": 0.24853515625,
      "learning_rate": 6.118181818181818e-07,
      "loss": 0.01,
      "reward": 1.4593749642372131,
      "reward_std": 0.43471772968769073,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7093749642372131,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.6875,
      "epoch": 1.1755829903978052,
      "grad_norm": 1.439836627773731,
      "kl": 0.080810546875,
      "learning_rate": 6.109090909090909e-07,
      "loss": 0.0032,
      "reward": 1.565625011920929,
      "reward_std": 0.18724938482046127,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.784375011920929,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.296875,
      "epoch": 1.1783264746227708,
      "grad_norm": 3.6172862056669377,
      "kl": 0.11083984375,
      "learning_rate": 6.1e-07,
      "loss": 0.0044,
      "reward": 1.3890624642372131,
      "reward_std": 0.6481033265590668,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7015625238418579,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.390625,
      "epoch": 1.1810699588477367,
      "grad_norm": 5.37902502650951,
      "kl": 0.096923828125,
      "learning_rate": 6.090909090909091e-07,
      "loss": 0.0039,
      "reward": 1.775781273841858,
      "reward_std": 0.3665124475955963,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8851562440395355,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.703125,
      "epoch": 1.1838134430727023,
      "grad_norm": 4.021069933817606,
      "kl": 0.122802734375,
      "learning_rate": 6.081818181818181e-07,
      "loss": 0.0049,
      "reward": 1.399218738079071,
      "reward_std": 0.43452630192041397,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.711718738079071,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.625,
      "epoch": 1.186556927297668,
      "grad_norm": 2.371316302774302,
      "kl": 0.0908203125,
      "learning_rate": 6.072727272727273e-07,
      "loss": 0.0036,
      "reward": 1.251562476158142,
      "reward_std": 0.22032135352492332,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6265625059604645,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.6875,
      "epoch": 1.1893004115226338,
      "grad_norm": 3.562586641125276,
      "kl": 0.088134765625,
      "learning_rate": 6.063636363636363e-07,
      "loss": 0.0035,
      "reward": 1.3554686903953552,
      "reward_std": 0.6390049308538437,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.68359375,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.40625,
      "epoch": 1.1920438957475994,
      "grad_norm": 3.2369462814293923,
      "kl": 0.10302734375,
      "learning_rate": 6.054545454545454e-07,
      "loss": 0.0041,
      "reward": 1.5859375,
      "reward_std": 0.4189769923686981,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7578124403953552,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.71875,
      "epoch": 1.1947873799725652,
      "grad_norm": 4.340266580844012,
      "kl": 0.087646484375,
      "learning_rate": 6.045454545454545e-07,
      "loss": 0.0035,
      "reward": 1.7515625357627869,
      "reward_std": 0.2581353858113289,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8609375059604645,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.8125,
      "epoch": 1.1975308641975309,
      "grad_norm": 4.637561852398335,
      "kl": 0.080078125,
      "learning_rate": 6.036363636363636e-07,
      "loss": 0.0032,
      "reward": 1.895312488079071,
      "reward_std": 0.2395406812429428,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.926562488079071,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.84375,
      "epoch": 1.2002743484224965,
      "grad_norm": 5.829391306788839,
      "kl": 0.084228515625,
      "learning_rate": 6.027272727272728e-07,
      "loss": 0.0034,
      "reward": 1.4124999642372131,
      "reward_std": 0.6100139319896698,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6937500089406967,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.34375,
      "epoch": 1.2030178326474623,
      "grad_norm": 2.929133446760945,
      "kl": 0.07958984375,
      "learning_rate": 6.018181818181818e-07,
      "loss": 0.0032,
      "reward": 1.404687523841858,
      "reward_std": 0.5406377464532852,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6859374940395355,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.703125,
      "epoch": 1.205761316872428,
      "grad_norm": 6.924132587020331,
      "kl": 0.081787109375,
      "learning_rate": 6.009090909090909e-07,
      "loss": 0.0033,
      "reward": 1.2460936903953552,
      "reward_std": 0.47603175044059753,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.62109375,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.546875,
      "epoch": 1.2085048010973938,
      "grad_norm": 52.78660154107059,
      "kl": 0.075927734375,
      "learning_rate": 6e-07,
      "loss": 0.003,
      "reward": 1.6765625476837158,
      "reward_std": 0.3512795567512512,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.817187488079071,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.578125,
      "epoch": 1.2112482853223594,
      "grad_norm": 11.455020404299585,
      "kl": 0.0633544921875,
      "learning_rate": 5.99090909090909e-07,
      "loss": 0.0025,
      "reward": 1.649999976158142,
      "reward_std": 0.4676620364189148,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.7906250357627869,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.046875,
      "epoch": 1.213991769547325,
      "grad_norm": 6.88451765941755,
      "kl": 0.0638427734375,
      "learning_rate": 5.981818181818181e-07,
      "loss": 0.0026,
      "reward": 1.5304688215255737,
      "reward_std": 0.43148937821388245,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7492187321186066,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.3125,
      "epoch": 1.2167352537722909,
      "grad_norm": 4.809497740022853,
      "kl": 0.1015625,
      "learning_rate": 5.972727272727273e-07,
      "loss": 0.0041,
      "reward": 1.459375023841858,
      "reward_std": 0.3682703413069248,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7249999940395355,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.75,
      "epoch": 1.2194787379972565,
      "grad_norm": 5.3235611348346,
      "kl": 0.070068359375,
      "learning_rate": 5.963636363636363e-07,
      "loss": 0.0028,
      "reward": 1.458593726158142,
      "reward_std": 0.3838815838098526,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7085937559604645,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.09375,
      "epoch": 1.2222222222222223,
      "grad_norm": 5.054732585213759,
      "kl": 0.071044921875,
      "learning_rate": 5.954545454545454e-07,
      "loss": 0.0028,
      "reward": 1.4960938096046448,
      "reward_std": 0.38986389338970184,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7460937201976776,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.375,
      "epoch": 1.224965706447188,
      "grad_norm": 3.7264747718150946,
      "kl": 0.08642578125,
      "learning_rate": 5.945454545454546e-07,
      "loss": 0.0035,
      "reward": 1.1820312142372131,
      "reward_std": 0.5168150663375854,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.5882812142372131,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.28125,
      "epoch": 1.2277091906721536,
      "grad_norm": 6.107198492961743,
      "kl": 0.173095703125,
      "learning_rate": 5.936363636363636e-07,
      "loss": 0.0069,
      "reward": 1.6039061546325684,
      "reward_std": 0.5601920038461685,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7914062440395355,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.125,
      "epoch": 1.2304526748971194,
      "grad_norm": 4.458635095195637,
      "kl": 0.1337890625,
      "learning_rate": 5.927272727272728e-07,
      "loss": 0.0053,
      "reward": 1.262499988079071,
      "reward_std": 0.4394213408231735,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6375000178813934,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.78125,
      "epoch": 1.233196159122085,
      "grad_norm": 4.552607093157277,
      "kl": 0.08056640625,
      "learning_rate": 5.918181818181818e-07,
      "loss": 0.0032,
      "reward": 1.5703125,
      "reward_std": 0.4914304316043854,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7734375,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.296875,
      "epoch": 1.2359396433470509,
      "grad_norm": 5.07754883381758,
      "kl": 0.1396484375,
      "learning_rate": 5.909090909090909e-07,
      "loss": 0.0056,
      "reward": 1.2890625,
      "reward_std": 0.43803659081459045,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6484375,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.578125,
      "epoch": 1.2386831275720165,
      "grad_norm": 2.578924195516793,
      "kl": 0.087890625,
      "learning_rate": 5.9e-07,
      "loss": 0.0035,
      "reward": 1.557812511920929,
      "reward_std": 0.4418707937002182,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7765624523162842,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.359375,
      "epoch": 1.241426611796982,
      "grad_norm": 3.4245702428563156,
      "kl": 0.10595703125,
      "learning_rate": 5.89090909090909e-07,
      "loss": 0.0042,
      "reward": 1.6875,
      "reward_std": 0.4009390026330948,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8437500298023224,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.703125,
      "epoch": 1.244170096021948,
      "grad_norm": 5.119261486750348,
      "kl": 0.088623046875,
      "learning_rate": 5.881818181818181e-07,
      "loss": 0.0035,
      "reward": 1.3804687857627869,
      "reward_std": 0.5968214869499207,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6617187261581421,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.078125,
      "epoch": 1.2469135802469136,
      "grad_norm": 6.440970252082927,
      "kl": 0.11181640625,
      "learning_rate": 5.872727272727273e-07,
      "loss": 0.0045,
      "reward": 1.8210937976837158,
      "reward_std": 0.4471282362937927,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.899218738079071,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.84375,
      "epoch": 1.2496570644718794,
      "grad_norm": 3.8773183626162235,
      "kl": 0.140625,
      "learning_rate": 5.863636363636362e-07,
      "loss": 0.0056,
      "reward": 1.223437488079071,
      "reward_std": 0.48718254268169403,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.614062488079071,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.859375,
      "epoch": 1.252400548696845,
      "grad_norm": 2.9298753340037322,
      "kl": 0.097412109375,
      "learning_rate": 5.854545454545454e-07,
      "loss": 0.0039,
      "reward": 1.760937511920929,
      "reward_std": 0.5081861466169357,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8703124821186066,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.734375,
      "epoch": 1.2551440329218106,
      "grad_norm": 3.8792742707571604,
      "kl": 0.09326171875,
      "learning_rate": 5.845454545454546e-07,
      "loss": 0.0037,
      "reward": 1.75,
      "reward_std": 0.42434723675251007,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8750000298023224,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.46875,
      "epoch": 1.2578875171467763,
      "grad_norm": 3.23678170692346,
      "kl": 0.09716796875,
      "learning_rate": 5.836363636363636e-07,
      "loss": 0.0039,
      "reward": 1.4023438096046448,
      "reward_std": 0.38366977870464325,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7148437201976776,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 95.421875,
      "epoch": 1.260631001371742,
      "grad_norm": 6.6456792173455375,
      "kl": 0.110595703125,
      "learning_rate": 5.827272727272727e-07,
      "loss": 0.0044,
      "reward": 0.8304687440395355,
      "reward_std": 0.649567037820816,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.43984371423721313,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.0,
      "epoch": 1.263374485596708,
      "grad_norm": 2.9585309457636804,
      "kl": 0.107666015625,
      "learning_rate": 5.818181818181818e-07,
      "loss": 0.0043,
      "reward": 1.603906273841858,
      "reward_std": 0.5155107527971268,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7914062142372131,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.328125,
      "epoch": 1.2661179698216736,
      "grad_norm": 6.193455070609818,
      "kl": 0.1220703125,
      "learning_rate": 5.809090909090909e-07,
      "loss": 0.0049,
      "reward": 1.3578124642372131,
      "reward_std": 0.6153180003166199,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.046875,
      "epoch": 1.2688614540466392,
      "grad_norm": 6.112679464169271,
      "kl": 0.14990234375,
      "learning_rate": 5.8e-07,
      "loss": 0.006,
      "reward": 1.241406261920929,
      "reward_std": 0.4134686142206192,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6320312321186066,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.53125,
      "epoch": 1.2716049382716048,
      "grad_norm": 2.9616624839056933,
      "kl": 0.1162109375,
      "learning_rate": 5.79090909090909e-07,
      "loss": 0.0047,
      "reward": 1.4453125,
      "reward_std": 0.15817427728325129,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7265624701976776,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.078125,
      "epoch": 1.2743484224965707,
      "grad_norm": 7.09015056585456,
      "kl": 0.115234375,
      "learning_rate": 5.781818181818181e-07,
      "loss": 0.0046,
      "reward": 1.2546875476837158,
      "reward_std": 0.2974911257624626,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.645312488079071,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.78125,
      "epoch": 1.2770919067215363,
      "grad_norm": 2.6284994699392006,
      "kl": 0.13232421875,
      "learning_rate": 5.772727272727273e-07,
      "loss": 0.0053,
      "reward": 1.811718761920929,
      "reward_std": 0.3243738263845444,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9054687917232513,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.484375,
      "epoch": 1.2798353909465021,
      "grad_norm": 4.86271779024084,
      "kl": 0.12646484375,
      "learning_rate": 5.763636363636363e-07,
      "loss": 0.0051,
      "reward": 1.1578125059604645,
      "reward_std": 0.38512764871120453,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5953124612569809,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.0625,
      "epoch": 1.2825788751714677,
      "grad_norm": 12.418372543140359,
      "kl": 0.115478515625,
      "learning_rate": 5.754545454545454e-07,
      "loss": 0.0046,
      "reward": 0.895312488079071,
      "reward_std": 0.3500541225075722,
      "rewards/choice_reward": 0.421875,
      "rewards/roam_reward": 0.47343748807907104,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.078125,
      "epoch": 1.2853223593964334,
      "grad_norm": 15.054577189255165,
      "kl": 0.10546875,
      "learning_rate": 5.745454545454546e-07,
      "loss": 0.0042,
      "reward": 1.7906250357627869,
      "reward_std": 0.4052163362503052,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8843750059604645,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.109375,
      "epoch": 1.2880658436213992,
      "grad_norm": 0.23607626191653258,
      "kl": 0.09375,
      "learning_rate": 5.736363636363636e-07,
      "loss": 0.0037,
      "reward": 2.0,
      "reward_std": 0.0,
      "rewards/choice_reward": 1.0,
      "rewards/roam_reward": 1.0,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.640625,
      "epoch": 1.2908093278463648,
      "grad_norm": 5.63969040120091,
      "kl": 0.10107421875,
      "learning_rate": 5.727272727272727e-07,
      "loss": 0.004,
      "reward": 1.323437511920929,
      "reward_std": 0.5278631448745728,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.651562511920929,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.796875,
      "epoch": 1.2935528120713307,
      "grad_norm": 4.383849861820226,
      "kl": 0.13671875,
      "learning_rate": 5.718181818181818e-07,
      "loss": 0.0055,
      "reward": 1.465624988079071,
      "reward_std": 0.42359398305416107,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.731249988079071,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.65625,
      "epoch": 1.2962962962962963,
      "grad_norm": 3.288610200800226,
      "kl": 0.111083984375,
      "learning_rate": 5.709090909090909e-07,
      "loss": 0.0044,
      "reward": 1.7421875,
      "reward_std": 0.44370877742767334,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8671875298023224,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.703125,
      "epoch": 1.299039780521262,
      "grad_norm": 9.827746624272217,
      "kl": 0.101318359375,
      "learning_rate": 5.699999999999999e-07,
      "loss": 0.0041,
      "reward": 1.074999988079071,
      "reward_std": 0.5073258280754089,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.559374988079071,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.015625,
      "epoch": 1.3017832647462277,
      "grad_norm": 3.348360712200408,
      "kl": 0.13037109375,
      "learning_rate": 5.69090909090909e-07,
      "loss": 0.0052,
      "reward": 1.087499976158142,
      "reward_std": 0.6843429207801819,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5562499761581421,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.609375,
      "epoch": 1.3045267489711934,
      "grad_norm": 14.093744111040772,
      "kl": 0.12451171875,
      "learning_rate": 5.681818181818182e-07,
      "loss": 0.005,
      "reward": 1.4609375,
      "reward_std": 0.5192954540252686,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7421875,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.65625,
      "epoch": 1.3072702331961592,
      "grad_norm": 9.855896100482207,
      "kl": 0.13916015625,
      "learning_rate": 5.672727272727273e-07,
      "loss": 0.0056,
      "reward": 1.60546875,
      "reward_std": 0.3087161513976753,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.80859375,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.953125,
      "epoch": 1.3100137174211248,
      "grad_norm": 3.842627802662204,
      "kl": 0.106689453125,
      "learning_rate": 5.663636363636363e-07,
      "loss": 0.0043,
      "reward": 1.668749988079071,
      "reward_std": 0.5410042703151703,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.840624988079071,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.625,
      "epoch": 1.3127572016460904,
      "grad_norm": 3.1006416439220086,
      "kl": 0.111328125,
      "learning_rate": 5.654545454545454e-07,
      "loss": 0.0045,
      "reward": 1.7156250476837158,
      "reward_std": 0.13258251920342445,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.856249988079071,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.25,
      "epoch": 1.3155006858710563,
      "grad_norm": 1.897058956293695,
      "kl": 0.110107421875,
      "learning_rate": 5.645454545454546e-07,
      "loss": 0.0044,
      "reward": 1.996874988079071,
      "reward_std": 0.008838835172355175,
      "rewards/choice_reward": 1.0,
      "rewards/roam_reward": 0.996874988079071,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.75,
      "epoch": 1.318244170096022,
      "grad_norm": 4.35448032695745,
      "kl": 0.11328125,
      "learning_rate": 5.636363636363635e-07,
      "loss": 0.0045,
      "reward": 1.485937476158142,
      "reward_std": 0.5335795134305954,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7515624463558197,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.484375,
      "epoch": 1.3209876543209877,
      "grad_norm": 7.093619038036287,
      "kl": 0.124755859375,
      "learning_rate": 5.627272727272727e-07,
      "loss": 0.005,
      "reward": 1.188281238079071,
      "reward_std": 0.4797893762588501,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.610156238079071,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.0,
      "epoch": 1.3237311385459534,
      "grad_norm": 3.5193018775527736,
      "kl": 0.110595703125,
      "learning_rate": 5.618181818181818e-07,
      "loss": 0.0044,
      "reward": 1.8015625476837158,
      "reward_std": 0.4114743173122406,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.879687488079071,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.140625,
      "epoch": 1.326474622770919,
      "grad_norm": 4.831405820614919,
      "kl": 0.11328125,
      "learning_rate": 5.609090909090909e-07,
      "loss": 0.0045,
      "reward": 1.818750023841858,
      "reward_std": 0.36465759575366974,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.8968750238418579,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.890625,
      "epoch": 1.3292181069958848,
      "grad_norm": 7.379749718313667,
      "kl": 0.12548828125,
      "learning_rate": 5.6e-07,
      "loss": 0.005,
      "reward": 1.2062499523162842,
      "reward_std": 0.5794805586338043,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6124999523162842,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.359375,
      "epoch": 1.3319615912208504,
      "grad_norm": 2.487766952563411,
      "kl": 0.116455078125,
      "learning_rate": 5.590909090909091e-07,
      "loss": 0.0047,
      "reward": 1.7109375,
      "reward_std": 0.2527294456958771,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8515624701976776,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.15625,
      "epoch": 1.3347050754458163,
      "grad_norm": 9.399922702963035,
      "kl": 0.103515625,
      "learning_rate": 5.581818181818182e-07,
      "loss": 0.0041,
      "reward": 1.6734375357627869,
      "reward_std": 0.3038512021303177,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8453125357627869,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.046875,
      "epoch": 1.337448559670782,
      "grad_norm": 8.353208322666267,
      "kl": 0.130126953125,
      "learning_rate": 5.572727272727273e-07,
      "loss": 0.0052,
      "reward": 1.251562476158142,
      "reward_std": 0.5889841318130493,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6421875059604645,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.0,
      "epoch": 1.3401920438957475,
      "grad_norm": 4.434212844778169,
      "kl": 0.120849609375,
      "learning_rate": 5.563636363636363e-07,
      "loss": 0.0048,
      "reward": 1.403124988079071,
      "reward_std": 0.5678224265575409,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.715624988079071,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.84375,
      "epoch": 1.3429355281207134,
      "grad_norm": 5.101295400485607,
      "kl": 0.11669921875,
      "learning_rate": 5.554545454545454e-07,
      "loss": 0.0047,
      "reward": 1.584375023841858,
      "reward_std": 0.23689022660255432,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.8031249940395355,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.078125,
      "epoch": 1.345679012345679,
      "grad_norm": 10.778408611138774,
      "kl": 0.16162109375,
      "learning_rate": 5.545454545454546e-07,
      "loss": 0.0065,
      "reward": 1.393750011920929,
      "reward_std": 0.4552782475948334,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7062499821186066,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.140625,
      "epoch": 1.3484224965706448,
      "grad_norm": 2.9983226699679957,
      "kl": 0.093017578125,
      "learning_rate": 5.536363636363635e-07,
      "loss": 0.0037,
      "reward": 1.3453125357627869,
      "reward_std": 0.44656118750572205,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6890625059604645,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.453125,
      "epoch": 1.3511659807956105,
      "grad_norm": 3.350677952118022,
      "kl": 0.104736328125,
      "learning_rate": 5.527272727272727e-07,
      "loss": 0.0042,
      "reward": 1.581250011920929,
      "reward_std": 0.4546556994318962,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7999999523162842,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.453125,
      "epoch": 1.353909465020576,
      "grad_norm": 4.15019426566186,
      "kl": 0.103515625,
      "learning_rate": 5.518181818181818e-07,
      "loss": 0.0042,
      "reward": 1.495312511920929,
      "reward_std": 0.45879365503787994,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7609374821186066,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.328125,
      "epoch": 1.356652949245542,
      "grad_norm": 9.8280328163177,
      "kl": 0.117431640625,
      "learning_rate": 5.509090909090909e-07,
      "loss": 0.0047,
      "reward": 1.5812499523162842,
      "reward_std": 0.5516014248132706,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7999999821186066,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.71875,
      "epoch": 1.3593964334705075,
      "grad_norm": 8.289229537545548,
      "kl": 0.1904296875,
      "learning_rate": 5.5e-07,
      "loss": 0.0076,
      "reward": 1.3085936903953552,
      "reward_std": 0.4198668450117111,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.66796875,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.34375,
      "epoch": 1.3621399176954734,
      "grad_norm": 5.25299152931547,
      "kl": 0.123779296875,
      "learning_rate": 5.490909090909091e-07,
      "loss": 0.005,
      "reward": 1.590624988079071,
      "reward_std": 0.4451471269130707,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.793749988079071,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.75,
      "epoch": 1.364883401920439,
      "grad_norm": 5.404401168995441,
      "kl": 0.18359375,
      "learning_rate": 5.481818181818182e-07,
      "loss": 0.0073,
      "reward": 1.2242187857627869,
      "reward_std": 0.17593394219875336,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6304687559604645,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.796875,
      "epoch": 1.3676268861454046,
      "grad_norm": 8.571801986242038,
      "kl": 0.101806640625,
      "learning_rate": 5.472727272727272e-07,
      "loss": 0.0041,
      "reward": 1.612500011920929,
      "reward_std": 0.3662673383951187,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.815625011920929,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.9375,
      "epoch": 1.3703703703703702,
      "grad_norm": 5.118638981200302,
      "kl": 0.125732421875,
      "learning_rate": 5.463636363636363e-07,
      "loss": 0.005,
      "reward": 1.323437511920929,
      "reward_std": 0.5765286982059479,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6671874821186066,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.75,
      "epoch": 1.373113854595336,
      "grad_norm": 2.610331549333793,
      "kl": 0.100341796875,
      "learning_rate": 5.454545454545454e-07,
      "loss": 0.004,
      "reward": 1.524999976158142,
      "reward_std": 0.2198822721838951,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7750000059604645,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.875,
      "epoch": 1.3758573388203017,
      "grad_norm": 3.028713970445138,
      "kl": 0.09423828125,
      "learning_rate": 5.445454545454546e-07,
      "loss": 0.0038,
      "reward": 1.2359374463558197,
      "reward_std": 0.47837182879447937,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6265625059604645,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.234375,
      "epoch": 1.3786008230452675,
      "grad_norm": 5.823742179238196,
      "kl": 0.119140625,
      "learning_rate": 5.436363636363635e-07,
      "loss": 0.0048,
      "reward": 1.948437511920929,
      "reward_std": 0.13620062451809645,
      "rewards/choice_reward": 0.984375,
      "rewards/roam_reward": 0.964062511920929,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.65625,
      "epoch": 1.3813443072702332,
      "grad_norm": 3.6877422509468816,
      "kl": 0.0947265625,
      "learning_rate": 5.427272727272727e-07,
      "loss": 0.0038,
      "reward": 1.8132812976837158,
      "reward_std": 0.40527141094207764,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.8914062678813934,
      "step": 503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.5625,
      "epoch": 1.3840877914951988,
      "grad_norm": 3.7952095790573237,
      "kl": 0.097900390625,
      "learning_rate": 5.418181818181819e-07,
      "loss": 0.0039,
      "reward": 1.39453125,
      "reward_std": 0.6221335232257843,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7070312201976776,
      "step": 504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.796875,
      "epoch": 1.3868312757201646,
      "grad_norm": 2.987830267870743,
      "kl": 0.100341796875,
      "learning_rate": 5.409090909090908e-07,
      "loss": 0.004,
      "reward": 1.517187476158142,
      "reward_std": 0.38894159346818924,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7671874761581421,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.390625,
      "epoch": 1.3895747599451302,
      "grad_norm": 2.7113804376330464,
      "kl": 0.116455078125,
      "learning_rate": 5.4e-07,
      "loss": 0.0047,
      "reward": 1.649999976158142,
      "reward_std": 0.2602013275027275,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8218749761581421,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.765625,
      "epoch": 1.392318244170096,
      "grad_norm": 3.3082946283729946,
      "kl": 0.123046875,
      "learning_rate": 5.390909090909091e-07,
      "loss": 0.0049,
      "reward": 1.7109375,
      "reward_std": 0.4046720564365387,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8515625,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.859375,
      "epoch": 1.3950617283950617,
      "grad_norm": 2.647995577478945,
      "kl": 0.09521484375,
      "learning_rate": 5.381818181818182e-07,
      "loss": 0.0038,
      "reward": 1.832812488079071,
      "reward_std": 0.4148809313774109,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9109375178813934,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.375,
      "epoch": 1.3978052126200273,
      "grad_norm": 4.815418563626276,
      "kl": 0.16552734375,
      "learning_rate": 5.372727272727272e-07,
      "loss": 0.0066,
      "reward": 1.53125,
      "reward_std": 0.1755305677652359,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.765625,
      "step": 509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.640625,
      "epoch": 1.4005486968449932,
      "grad_norm": 3.8831795024860827,
      "kl": 0.0830078125,
      "learning_rate": 5.363636363636363e-07,
      "loss": 0.0033,
      "reward": 1.4921874403953552,
      "reward_std": 0.37759263068437576,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7578124701976776,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.59375,
      "epoch": 1.4032921810699588,
      "grad_norm": 2.928606079305618,
      "kl": 0.09619140625,
      "learning_rate": 5.354545454545454e-07,
      "loss": 0.0039,
      "reward": 1.669531226158142,
      "reward_std": 0.2422011522576213,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8414062261581421,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.625,
      "epoch": 1.4060356652949246,
      "grad_norm": 13.053683686361019,
      "kl": 0.11669921875,
      "learning_rate": 5.345454545454545e-07,
      "loss": 0.0047,
      "reward": 1.6304687857627869,
      "reward_std": 0.5575891137123108,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8179687559604645,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.671875,
      "epoch": 1.4087791495198903,
      "grad_norm": 5.3727465109473975,
      "kl": 0.10791015625,
      "learning_rate": 5.336363636363636e-07,
      "loss": 0.0043,
      "reward": 1.6382812857627869,
      "reward_std": 0.43155744671821594,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8101562559604645,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.859375,
      "epoch": 1.4115226337448559,
      "grad_norm": 3.641029486434777,
      "kl": 0.10205078125,
      "learning_rate": 5.327272727272727e-07,
      "loss": 0.0041,
      "reward": 1.76953125,
      "reward_std": 0.36117296665906906,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.87890625,
      "step": 514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.21875,
      "epoch": 1.4142661179698217,
      "grad_norm": 6.256028372574312,
      "kl": 0.09130859375,
      "learning_rate": 5.318181818181819e-07,
      "loss": 0.0037,
      "reward": 1.603124976158142,
      "reward_std": 0.4715536832809448,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7906250059604645,
      "step": 515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.34375,
      "epoch": 1.4170096021947873,
      "grad_norm": 3.5855068495143723,
      "kl": 0.111328125,
      "learning_rate": 5.309090909090908e-07,
      "loss": 0.0045,
      "reward": 1.475000023841858,
      "reward_std": 0.37462590634822845,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7406249940395355,
      "step": 516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.328125,
      "epoch": 1.4197530864197532,
      "grad_norm": 13.345238848509045,
      "kl": 0.09375,
      "learning_rate": 5.3e-07,
      "loss": 0.0038,
      "reward": 1.3984374403953552,
      "reward_std": 0.5894862413406372,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7109375,
      "step": 517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.609375,
      "epoch": 1.4224965706447188,
      "grad_norm": 3.487109567518966,
      "kl": 0.110107421875,
      "learning_rate": 5.290909090909091e-07,
      "loss": 0.0044,
      "reward": 1.216406226158142,
      "reward_std": 0.5684505552053452,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.5914062559604645,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.78125,
      "epoch": 1.4252400548696844,
      "grad_norm": 3.4179403170394216,
      "kl": 0.110595703125,
      "learning_rate": 5.281818181818181e-07,
      "loss": 0.0044,
      "reward": 1.489062488079071,
      "reward_std": 0.4661572128534317,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.739062488079071,
      "step": 519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.328125,
      "epoch": 1.4279835390946503,
      "grad_norm": 4.304257394593773,
      "kl": 0.09521484375,
      "learning_rate": 5.272727272727272e-07,
      "loss": 0.0038,
      "reward": 1.431249976158142,
      "reward_std": 0.5100004374980927,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7281250059604645,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.328125,
      "epoch": 1.4307270233196159,
      "grad_norm": 7.649430043436629,
      "kl": 0.098388671875,
      "learning_rate": 5.263636363636363e-07,
      "loss": 0.0039,
      "reward": 1.7992187142372131,
      "reward_std": 0.2976684421300888,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8929687440395355,
      "step": 521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.5625,
      "epoch": 1.4334705075445817,
      "grad_norm": 5.650031888030248,
      "kl": 0.1015625,
      "learning_rate": 5.254545454545455e-07,
      "loss": 0.0041,
      "reward": 1.44140625,
      "reward_std": 0.4561328738927841,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7226562201976776,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.671875,
      "epoch": 1.4362139917695473,
      "grad_norm": 4.660635426861998,
      "kl": 0.0869140625,
      "learning_rate": 5.245454545454545e-07,
      "loss": 0.0035,
      "reward": 1.7624999284744263,
      "reward_std": 0.3338359519839287,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.887499988079071,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.046875,
      "epoch": 1.438957475994513,
      "grad_norm": 2.6979251782646902,
      "kl": 0.104248046875,
      "learning_rate": 5.236363636363636e-07,
      "loss": 0.0042,
      "reward": 1.9203124642372131,
      "reward_std": 0.22539027780294418,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.9515624940395355,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.5,
      "epoch": 1.4417009602194788,
      "grad_norm": 18.90388830504221,
      "kl": 0.096923828125,
      "learning_rate": 5.227272727272727e-07,
      "loss": 0.0039,
      "reward": 1.3968749642372131,
      "reward_std": 0.4366264343261719,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7093750238418579,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.328125,
      "epoch": 1.4444444444444444,
      "grad_norm": 3.489414046490887,
      "kl": 0.088134765625,
      "learning_rate": 5.218181818181819e-07,
      "loss": 0.0035,
      "reward": 1.8656249642372131,
      "reward_std": 0.25108102709054947,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9281250238418579,
      "step": 526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.28125,
      "epoch": 1.4471879286694103,
      "grad_norm": 5.09523654572199,
      "kl": 0.15087890625,
      "learning_rate": 5.209090909090908e-07,
      "loss": 0.006,
      "reward": 1.4765625,
      "reward_std": 0.4231499135494232,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7421875298023224,
      "step": 527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.578125,
      "epoch": 1.4499314128943759,
      "grad_norm": 2.7909099615882034,
      "kl": 0.1279296875,
      "learning_rate": 5.2e-07,
      "loss": 0.0051,
      "reward": 1.658593773841858,
      "reward_std": 0.272448793053627,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8304687440395355,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.875,
      "epoch": 1.4526748971193415,
      "grad_norm": 4.183851213619129,
      "kl": 0.096923828125,
      "learning_rate": 5.190909090909091e-07,
      "loss": 0.0039,
      "reward": 0.7804687321186066,
      "reward_std": 0.5706401765346527,
      "rewards/choice_reward": 0.359375,
      "rewards/roam_reward": 0.42109374701976776,
      "step": 529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.203125,
      "epoch": 1.4554183813443073,
      "grad_norm": 5.202662608016372,
      "kl": 0.11962890625,
      "learning_rate": 5.181818181818181e-07,
      "loss": 0.0048,
      "reward": 1.42578125,
      "reward_std": 0.5004200339317322,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.72265625,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.875,
      "epoch": 1.458161865569273,
      "grad_norm": 2.784816392977025,
      "kl": 0.11474609375,
      "learning_rate": 5.172727272727272e-07,
      "loss": 0.0046,
      "reward": 1.532812476158142,
      "reward_std": 0.24656263273209333,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7671875059604645,
      "step": 531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.796875,
      "epoch": 1.4609053497942388,
      "grad_norm": 2.9087030973929977,
      "kl": 0.088134765625,
      "learning_rate": 5.163636363636364e-07,
      "loss": 0.0035,
      "reward": 1.133593738079071,
      "reward_std": 0.463393434882164,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5867187082767487,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.4375,
      "epoch": 1.4636488340192044,
      "grad_norm": 3.1964784565555124,
      "kl": 0.107177734375,
      "learning_rate": 5.154545454545455e-07,
      "loss": 0.0043,
      "reward": 1.903906226158142,
      "reward_std": 0.13558390457183123,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9507812559604645,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.171875,
      "epoch": 1.46639231824417,
      "grad_norm": 3.9945355817322614,
      "kl": 0.113037109375,
      "learning_rate": 5.145454545454545e-07,
      "loss": 0.0045,
      "reward": 1.04296875,
      "reward_std": 0.40418025851249695,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5429687201976776,
      "step": 534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.25,
      "epoch": 1.4691358024691357,
      "grad_norm": 8.348785816730594,
      "kl": 0.09619140625,
      "learning_rate": 5.136363636363636e-07,
      "loss": 0.0039,
      "reward": 1.46875,
      "reward_std": 0.5412850975990295,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7343749701976776,
      "step": 535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.609375,
      "epoch": 1.4718792866941015,
      "grad_norm": 7.640760240916528,
      "kl": 0.115478515625,
      "learning_rate": 5.127272727272727e-07,
      "loss": 0.0046,
      "reward": 1.2687499523162842,
      "reward_std": 0.7544959485530853,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6437499970197678,
      "step": 536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.84375,
      "epoch": 1.4746227709190673,
      "grad_norm": 4.266240020097782,
      "kl": 0.1005859375,
      "learning_rate": 5.118181818181818e-07,
      "loss": 0.004,
      "reward": 1.295312523841858,
      "reward_std": 0.5077711343765259,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6546874940395355,
      "step": 537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.359375,
      "epoch": 1.477366255144033,
      "grad_norm": 3.923833439442307,
      "kl": 0.11181640625,
      "learning_rate": 5.109090909090908e-07,
      "loss": 0.0045,
      "reward": 1.323437511920929,
      "reward_std": 0.4369395971298218,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6359374821186066,
      "step": 538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.0625,
      "epoch": 1.4801097393689986,
      "grad_norm": 2.857980806131285,
      "kl": 0.1064453125,
      "learning_rate": 5.1e-07,
      "loss": 0.0042,
      "reward": 1.561718761920929,
      "reward_std": 0.4069434851408005,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.780468761920929,
      "step": 539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.140625,
      "epoch": 1.4828532235939642,
      "grad_norm": 12.797512293581908,
      "kl": 0.1123046875,
      "learning_rate": 5.090909090909091e-07,
      "loss": 0.0045,
      "reward": 1.63671875,
      "reward_std": 0.3925085961818695,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.82421875,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.734375,
      "epoch": 1.48559670781893,
      "grad_norm": 2.833671268147599,
      "kl": 0.10400390625,
      "learning_rate": 5.081818181818181e-07,
      "loss": 0.0042,
      "reward": 1.376562476158142,
      "reward_std": 0.40079691261053085,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.7046875059604645,
      "step": 541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.03125,
      "epoch": 1.4883401920438957,
      "grad_norm": 3.82926451926805,
      "kl": 0.089599609375,
      "learning_rate": 5.072727272727273e-07,
      "loss": 0.0036,
      "reward": 1.091406226158142,
      "reward_std": 0.6246378421783447,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5601562261581421,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.453125,
      "epoch": 1.4910836762688615,
      "grad_norm": 3.7490085338944876,
      "kl": 0.143798828125,
      "learning_rate": 5.063636363636364e-07,
      "loss": 0.0058,
      "reward": 1.0445312559604645,
      "reward_std": 0.3862235099077225,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5445312261581421,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.0625,
      "epoch": 1.4938271604938271,
      "grad_norm": 9.781717240616556,
      "kl": 0.10400390625,
      "learning_rate": 5.054545454545454e-07,
      "loss": 0.0042,
      "reward": 1.1070312559604645,
      "reward_std": 0.45780712366104126,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5757812559604645,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.578125,
      "epoch": 1.4965706447187928,
      "grad_norm": 2.6181307764218715,
      "kl": 0.123291015625,
      "learning_rate": 5.045454545454545e-07,
      "loss": 0.0049,
      "reward": 1.890625,
      "reward_std": 0.30935921519994736,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9375000298023224,
      "step": 545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.953125,
      "epoch": 1.4993141289437586,
      "grad_norm": 6.164237942689379,
      "kl": 0.101318359375,
      "learning_rate": 5.036363636363636e-07,
      "loss": 0.0041,
      "reward": 0.960156261920929,
      "reward_std": 0.2131277248263359,
      "rewards/choice_reward": 0.453125,
      "rewards/roam_reward": 0.5070312321186066,
      "step": 546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.765625,
      "epoch": 1.5020576131687244,
      "grad_norm": 8.03304964733052,
      "kl": 0.12548828125,
      "learning_rate": 5.027272727272727e-07,
      "loss": 0.005,
      "reward": 1.434374988079071,
      "reward_std": 0.6121031939983368,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.731249988079071,
      "step": 547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.03125,
      "epoch": 1.50480109739369,
      "grad_norm": 17.883401277948902,
      "kl": 0.115234375,
      "learning_rate": 5.018181818181818e-07,
      "loss": 0.0046,
      "reward": 1.8890624642372131,
      "reward_std": 0.18478976469486952,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9359374940395355,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.625,
      "epoch": 1.5075445816186557,
      "grad_norm": 4.871786541222827,
      "kl": 0.11083984375,
      "learning_rate": 5.009090909090908e-07,
      "loss": 0.0044,
      "reward": 1.75390625,
      "reward_std": 0.3689107820391655,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8789062201976776,
      "step": 549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.421875,
      "epoch": 1.5102880658436213,
      "grad_norm": 11.945953118265406,
      "kl": 0.09521484375,
      "learning_rate": 5e-07,
      "loss": 0.0038,
      "reward": 1.5843749642372131,
      "reward_std": 0.41377370804548264,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.8031250238418579,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.578125,
      "epoch": 1.5130315500685871,
      "grad_norm": 3.2427799757731357,
      "kl": 0.0966796875,
      "learning_rate": 4.99090909090909e-07,
      "loss": 0.0039,
      "reward": 1.5499999523162842,
      "reward_std": 0.09355210792273283,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7843749523162842,
      "step": 551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.765625,
      "epoch": 1.5157750342935528,
      "grad_norm": 27.58114087642646,
      "kl": 0.114501953125,
      "learning_rate": 4.981818181818181e-07,
      "loss": 0.0046,
      "reward": 1.536718726158142,
      "reward_std": 0.6080199331045151,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7554687261581421,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.921875,
      "epoch": 1.5185185185185186,
      "grad_norm": 3.117876526704857,
      "kl": 0.115966796875,
      "learning_rate": 4.972727272727272e-07,
      "loss": 0.0046,
      "reward": 1.732812523841858,
      "reward_std": 0.25190678983926773,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8734375238418579,
      "step": 553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.5,
      "epoch": 1.5212620027434842,
      "grad_norm": 4.933936379602721,
      "kl": 0.1103515625,
      "learning_rate": 4.963636363636364e-07,
      "loss": 0.0044,
      "reward": 1.703125,
      "reward_std": 0.3168279826641083,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.859375,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.6875,
      "epoch": 1.5240054869684498,
      "grad_norm": 4.628806200489313,
      "kl": 0.16357421875,
      "learning_rate": 4.954545454545454e-07,
      "loss": 0.0065,
      "reward": 1.431249976158142,
      "reward_std": 0.47491100430488586,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.6968749910593033,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.984375,
      "epoch": 1.5267489711934157,
      "grad_norm": 10.569874783958957,
      "kl": 0.11279296875,
      "learning_rate": 4.945454545454545e-07,
      "loss": 0.0045,
      "reward": 1.254687488079071,
      "reward_std": 0.4676324874162674,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6453124582767487,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.96875,
      "epoch": 1.5294924554183813,
      "grad_norm": 23.02839701452903,
      "kl": 0.15673828125,
      "learning_rate": 4.936363636363636e-07,
      "loss": 0.0063,
      "reward": 1.5046875476837158,
      "reward_std": 0.650595635175705,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7546875178813934,
      "step": 557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.796875,
      "epoch": 1.5322359396433471,
      "grad_norm": 3.2318566693408313,
      "kl": 0.14208984375,
      "learning_rate": 4.927272727272728e-07,
      "loss": 0.0057,
      "reward": 1.3976562023162842,
      "reward_std": 0.2317863437347114,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7101562321186066,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.484375,
      "epoch": 1.5349794238683128,
      "grad_norm": 6.730100046084902,
      "kl": 0.13916015625,
      "learning_rate": 4.918181818181818e-07,
      "loss": 0.0056,
      "reward": 1.1320313215255737,
      "reward_std": 0.539662778377533,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5851562321186066,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.71875,
      "epoch": 1.5377229080932784,
      "grad_norm": 4.391655190501676,
      "kl": 0.10498046875,
      "learning_rate": 4.909090909090909e-07,
      "loss": 0.0042,
      "reward": 1.609375,
      "reward_std": 0.4833652973175049,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7968749701976776,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.40625,
      "epoch": 1.540466392318244,
      "grad_norm": 3.4647848378452,
      "kl": 0.1181640625,
      "learning_rate": 4.9e-07,
      "loss": 0.0047,
      "reward": 1.43359375,
      "reward_std": 0.44188737869262695,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7304687201976776,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.078125,
      "epoch": 1.5432098765432098,
      "grad_norm": 4.4882395443760155,
      "kl": 0.138427734375,
      "learning_rate": 4.89090909090909e-07,
      "loss": 0.0055,
      "reward": 1.56640625,
      "reward_std": 0.3243209421634674,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.76953125,
      "step": 562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.109375,
      "epoch": 1.5459533607681757,
      "grad_norm": 3.5210911842576804,
      "kl": 0.135498046875,
      "learning_rate": 4.881818181818181e-07,
      "loss": 0.0054,
      "reward": 1.532812476158142,
      "reward_std": 0.4216124564409256,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7671875059604645,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.546875,
      "epoch": 1.5486968449931413,
      "grad_norm": 1.6962515713747988,
      "kl": 0.106689453125,
      "learning_rate": 4.872727272727272e-07,
      "loss": 0.0043,
      "reward": 1.732812523841858,
      "reward_std": 0.2328590601682663,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8734375238418579,
      "step": 564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.328125,
      "epoch": 1.551440329218107,
      "grad_norm": 3.930797977742594,
      "kl": 0.11767578125,
      "learning_rate": 4.863636363636364e-07,
      "loss": 0.0047,
      "reward": 1.615625023841858,
      "reward_std": 0.22946061939001083,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7875000238418579,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.265625,
      "epoch": 1.5541838134430725,
      "grad_norm": 7.4504102904463965,
      "kl": 0.114013671875,
      "learning_rate": 4.854545454545454e-07,
      "loss": 0.0046,
      "reward": 1.524218738079071,
      "reward_std": 0.5162283182144165,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.774218738079071,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.0,
      "epoch": 1.5569272976680384,
      "grad_norm": 2.417056436812491,
      "kl": 0.08837890625,
      "learning_rate": 4.845454545454545e-07,
      "loss": 0.0035,
      "reward": 1.80078125,
      "reward_std": 0.40284576267004013,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.87890625,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.09375,
      "epoch": 1.5596707818930042,
      "grad_norm": 3.7816972340293797,
      "kl": 0.121337890625,
      "learning_rate": 4.836363636363636e-07,
      "loss": 0.0049,
      "reward": 1.399999976158142,
      "reward_std": 0.32486552000045776,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7124999761581421,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.703125,
      "epoch": 1.5624142661179699,
      "grad_norm": 4.999981285474218,
      "kl": 0.1396484375,
      "learning_rate": 4.827272727272727e-07,
      "loss": 0.0056,
      "reward": 1.757031261920929,
      "reward_std": 0.42812520265579224,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8820312321186066,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.953125,
      "epoch": 1.5651577503429355,
      "grad_norm": 5.426287768038498,
      "kl": 0.1103515625,
      "learning_rate": 4.818181818181818e-07,
      "loss": 0.0044,
      "reward": 1.6609374284744263,
      "reward_std": 0.4797618091106415,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.832812488079071,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.046875,
      "epoch": 1.567901234567901,
      "grad_norm": 5.988679095790291,
      "kl": 0.12060546875,
      "learning_rate": 4.809090909090909e-07,
      "loss": 0.0048,
      "reward": 1.38671875,
      "reward_std": 0.5065616965293884,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6992187201976776,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.296875,
      "epoch": 1.570644718792867,
      "grad_norm": 4.7176649262724615,
      "kl": 0.108642578125,
      "learning_rate": 4.8e-07,
      "loss": 0.0043,
      "reward": 1.5374999642372131,
      "reward_std": 0.49469953775405884,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7718749940395355,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.4375,
      "epoch": 1.5733882030178328,
      "grad_norm": 4.621444143827856,
      "kl": 0.140380859375,
      "learning_rate": 4.79090909090909e-07,
      "loss": 0.0056,
      "reward": 1.6992186903953552,
      "reward_std": 0.4136374071240425,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8554687201976776,
      "step": 573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.515625,
      "epoch": 1.5761316872427984,
      "grad_norm": 2.88230948081132,
      "kl": 0.12646484375,
      "learning_rate": 4.781818181818182e-07,
      "loss": 0.005,
      "reward": 1.69921875,
      "reward_std": 0.25533968955278397,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8554687798023224,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.984375,
      "epoch": 1.578875171467764,
      "grad_norm": 4.184751540073032,
      "kl": 0.12255859375,
      "learning_rate": 4.772727272727273e-07,
      "loss": 0.0049,
      "reward": 1.626562476158142,
      "reward_std": 0.36238789558410645,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8140625059604645,
      "step": 575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.859375,
      "epoch": 1.5816186556927296,
      "grad_norm": 1.4444185806466716,
      "kl": 0.113037109375,
      "learning_rate": 4.7636363636363635e-07,
      "loss": 0.0045,
      "reward": 1.435937523841858,
      "reward_std": 0.19391006976366043,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7328125238418579,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.234375,
      "epoch": 1.5843621399176955,
      "grad_norm": 5.755466833556867,
      "kl": 0.14013671875,
      "learning_rate": 4.754545454545454e-07,
      "loss": 0.0056,
      "reward": 1.638281226158142,
      "reward_std": 0.5257053971290588,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8257812857627869,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.921875,
      "epoch": 1.5871056241426613,
      "grad_norm": 13.267074126831844,
      "kl": 0.14453125,
      "learning_rate": 4.7454545454545454e-07,
      "loss": 0.0058,
      "reward": 1.232812523841858,
      "reward_std": 0.4578182101249695,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6234374642372131,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 64.546875,
      "epoch": 1.589849108367627,
      "grad_norm": 3.8694886430696425,
      "kl": 0.09716796875,
      "learning_rate": 4.736363636363636e-07,
      "loss": 0.0039,
      "reward": 1.4031250476837158,
      "reward_std": 0.3256668224930763,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7156249582767487,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.1875,
      "epoch": 1.5925925925925926,
      "grad_norm": 15.953835080678518,
      "kl": 0.09765625,
      "learning_rate": 4.727272727272727e-07,
      "loss": 0.0039,
      "reward": 1.5390625596046448,
      "reward_std": 0.2961009591817856,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7734375,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.796875,
      "epoch": 1.5953360768175582,
      "grad_norm": 3.4278361707564002,
      "kl": 0.132568359375,
      "learning_rate": 4.718181818181818e-07,
      "loss": 0.0053,
      "reward": 1.28125,
      "reward_std": 0.33917755633592606,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.65625,
      "step": 581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.3125,
      "epoch": 1.598079561042524,
      "grad_norm": 3.9189807225679303,
      "kl": 0.14111328125,
      "learning_rate": 4.709090909090909e-07,
      "loss": 0.0057,
      "reward": 1.384374976158142,
      "reward_std": 0.4875795245170593,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6968750059604645,
      "step": 582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.875,
      "epoch": 1.6008230452674899,
      "grad_norm": 9.724241563545263,
      "kl": 0.12158203125,
      "learning_rate": 4.6999999999999995e-07,
      "loss": 0.0049,
      "reward": 1.649999976158142,
      "reward_std": 0.5250689834356308,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8218749761581421,
      "step": 583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.65625,
      "epoch": 1.6035665294924555,
      "grad_norm": 6.05474610978915,
      "kl": 0.142333984375,
      "learning_rate": 4.6909090909090907e-07,
      "loss": 0.0057,
      "reward": 1.389062523841858,
      "reward_std": 0.5110916793346405,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.15625,
      "epoch": 1.606310013717421,
      "grad_norm": 6.233889163012752,
      "kl": 0.1552734375,
      "learning_rate": 4.681818181818182e-07,
      "loss": 0.0062,
      "reward": 1.703125,
      "reward_std": 0.42360812425613403,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.84375,
      "step": 585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.125,
      "epoch": 1.6090534979423867,
      "grad_norm": 3.2489686821377086,
      "kl": 0.116455078125,
      "learning_rate": 4.6727272727272727e-07,
      "loss": 0.0047,
      "reward": 1.13671875,
      "reward_std": 0.5294448733329773,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5898436903953552,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.46875,
      "epoch": 1.6117969821673526,
      "grad_norm": 2.3855608825870394,
      "kl": 0.134765625,
      "learning_rate": 4.6636363636363634e-07,
      "loss": 0.0054,
      "reward": 1.890625,
      "reward_std": 0.24214322119951248,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9375,
      "step": 587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.515625,
      "epoch": 1.6145404663923184,
      "grad_norm": 4.168536384395145,
      "kl": 0.12841796875,
      "learning_rate": 4.6545454545454546e-07,
      "loss": 0.0052,
      "reward": 1.3265624642372131,
      "reward_std": 0.510853722691536,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6703124642372131,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.78125,
      "epoch": 1.617283950617284,
      "grad_norm": 4.689041205415565,
      "kl": 0.1416015625,
      "learning_rate": 4.6454545454545453e-07,
      "loss": 0.0057,
      "reward": 1.1421875357627869,
      "reward_std": 0.5705660581588745,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5796874761581421,
      "step": 589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.234375,
      "epoch": 1.6200274348422496,
      "grad_norm": 3.0173678990110187,
      "kl": 0.15771484375,
      "learning_rate": 4.636363636363636e-07,
      "loss": 0.0063,
      "reward": 1.4898437857627869,
      "reward_std": 0.33587054908275604,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7398437261581421,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.65625,
      "epoch": 1.6227709190672153,
      "grad_norm": 3.6987374890571068,
      "kl": 0.094482421875,
      "learning_rate": 4.6272727272727267e-07,
      "loss": 0.0038,
      "reward": 1.805468738079071,
      "reward_std": 0.25329074263572693,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.899218738079071,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.53125,
      "epoch": 1.625514403292181,
      "grad_norm": 4.2905130258744615,
      "kl": 0.127685546875,
      "learning_rate": 4.618181818181818e-07,
      "loss": 0.0051,
      "reward": 1.4921875,
      "reward_std": 0.09280776232481003,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7578125,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.09375,
      "epoch": 1.6282578875171467,
      "grad_norm": 5.484523251838174,
      "kl": 0.15087890625,
      "learning_rate": 4.609090909090909e-07,
      "loss": 0.0061,
      "reward": 1.178125023841858,
      "reward_std": 0.6947203576564789,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.5999999940395355,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.6875,
      "epoch": 1.6310013717421126,
      "grad_norm": 10.441892796687878,
      "kl": 0.1103515625,
      "learning_rate": 4.6e-07,
      "loss": 0.0044,
      "reward": 1.2367187142372131,
      "reward_std": 0.5257429778575897,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6273437142372131,
      "step": 594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.34375,
      "epoch": 1.6337448559670782,
      "grad_norm": 4.828817797430915,
      "kl": 0.13134765625,
      "learning_rate": 4.5909090909090906e-07,
      "loss": 0.0052,
      "reward": 1.578906238079071,
      "reward_std": 0.2987357899546623,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.797656238079071,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.203125,
      "epoch": 1.6364883401920438,
      "grad_norm": 9.194137482032842,
      "kl": 0.12890625,
      "learning_rate": 4.581818181818182e-07,
      "loss": 0.0052,
      "reward": 1.161718726158142,
      "reward_std": 0.3223042516037822,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5992186963558197,
      "step": 596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.765625,
      "epoch": 1.6392318244170097,
      "grad_norm": 4.86268093816095,
      "kl": 0.1103515625,
      "learning_rate": 4.5727272727272725e-07,
      "loss": 0.0044,
      "reward": 1.665624976158142,
      "reward_std": 0.4220944941043854,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8375000059604645,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.625,
      "epoch": 1.6419753086419753,
      "grad_norm": 2.8619324737582934,
      "kl": 0.116943359375,
      "learning_rate": 4.563636363636363e-07,
      "loss": 0.0047,
      "reward": 1.7578125,
      "reward_std": 0.28776606172323227,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8828125,
      "step": 598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.46875,
      "epoch": 1.6447187928669411,
      "grad_norm": 3.2934620958997036,
      "kl": 0.1123046875,
      "learning_rate": 4.5545454545454545e-07,
      "loss": 0.0045,
      "reward": 1.8453125357627869,
      "reward_std": 0.3014649823307991,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9234375059604645,
      "step": 599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 61.8125,
      "epoch": 1.6474622770919067,
      "grad_norm": 3.7353753031206534,
      "kl": 0.14208984375,
      "learning_rate": 4.545454545454545e-07,
      "loss": 0.0057,
      "reward": 1.544531226158142,
      "reward_std": 0.30668093264102936,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7789062261581421,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.359375,
      "epoch": 1.6502057613168724,
      "grad_norm": 2.4930601686369034,
      "kl": 0.106201171875,
      "learning_rate": 4.536363636363636e-07,
      "loss": 0.0042,
      "reward": 1.576562523841858,
      "reward_std": 0.2080155168659985,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7953125238418579,
      "step": 601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.765625,
      "epoch": 1.652949245541838,
      "grad_norm": 44.7657764016526,
      "kl": 0.107177734375,
      "learning_rate": 4.5272727272727266e-07,
      "loss": 0.0043,
      "reward": 1.547656238079071,
      "reward_std": 0.6531318426132202,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.782031238079071,
      "step": 602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.765625,
      "epoch": 1.6556927297668038,
      "grad_norm": 3.435565573409563,
      "kl": 0.134765625,
      "learning_rate": 4.5181818181818184e-07,
      "loss": 0.0054,
      "reward": 1.4265624284744263,
      "reward_std": 0.4840279370546341,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.723437488079071,
      "step": 603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.15625,
      "epoch": 1.6584362139917697,
      "grad_norm": 4.287230409751256,
      "kl": 0.11669921875,
      "learning_rate": 4.509090909090909e-07,
      "loss": 0.0047,
      "reward": 1.306249976158142,
      "reward_std": 0.32932937145233154,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6656249761581421,
      "step": 604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.953125,
      "epoch": 1.6611796982167353,
      "grad_norm": 6.264101524320277,
      "kl": 0.130859375,
      "learning_rate": 4.5e-07,
      "loss": 0.0052,
      "reward": 1.182812511920929,
      "reward_std": 0.6257371604442596,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.604687511920929,
      "step": 605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.390625,
      "epoch": 1.663923182441701,
      "grad_norm": 7.173487886744967,
      "kl": 0.28466796875,
      "learning_rate": 4.4909090909090905e-07,
      "loss": 0.0114,
      "reward": 1.397656261920929,
      "reward_std": 0.6923157572746277,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.710156261920929,
      "step": 606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.671875,
      "epoch": 1.6666666666666665,
      "grad_norm": 3.2794289247863477,
      "kl": 0.089111328125,
      "learning_rate": 4.481818181818182e-07,
      "loss": 0.0036,
      "reward": 1.670312523841858,
      "reward_std": 0.29969462007284164,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8421875238418579,
      "step": 607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.34375,
      "epoch": 1.6694101508916324,
      "grad_norm": 5.6576754017177295,
      "kl": 0.0986328125,
      "learning_rate": 4.4727272727272724e-07,
      "loss": 0.0039,
      "reward": 1.7078125476837158,
      "reward_std": 0.4768216609954834,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.832812488079071,
      "step": 608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.46875,
      "epoch": 1.6721536351165982,
      "grad_norm": 10.137304702290834,
      "kl": 0.142578125,
      "learning_rate": 4.463636363636363e-07,
      "loss": 0.0057,
      "reward": 1.7421875,
      "reward_std": 0.5135543495416641,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8671875,
      "step": 609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.359375,
      "epoch": 1.6748971193415638,
      "grad_norm": 3.438968403143286,
      "kl": 0.124755859375,
      "learning_rate": 4.4545454545454544e-07,
      "loss": 0.005,
      "reward": 1.3140625357627869,
      "reward_std": 0.4177750498056412,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6578125059604645,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.671875,
      "epoch": 1.6776406035665294,
      "grad_norm": 4.865075486656035,
      "kl": 0.088623046875,
      "learning_rate": 4.445454545454545e-07,
      "loss": 0.0035,
      "reward": 1.217187523841858,
      "reward_std": 0.6815090179443359,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6078124940395355,
      "step": 611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.890625,
      "epoch": 1.680384087791495,
      "grad_norm": 5.572468848996713,
      "kl": 0.10595703125,
      "learning_rate": 4.4363636363636363e-07,
      "loss": 0.0042,
      "reward": 1.2195312976837158,
      "reward_std": 0.40889422595500946,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.625781238079071,
      "step": 612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.890625,
      "epoch": 1.683127572016461,
      "grad_norm": 2.5481794991899283,
      "kl": 0.105224609375,
      "learning_rate": 4.427272727272727e-07,
      "loss": 0.0042,
      "reward": 1.5843749642372131,
      "reward_std": 0.3832883834838867,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7875000238418579,
      "step": 613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.140625,
      "epoch": 1.6858710562414267,
      "grad_norm": 4.31918666722551,
      "kl": 0.10107421875,
      "learning_rate": 4.4181818181818183e-07,
      "loss": 0.0041,
      "reward": 1.40625,
      "reward_std": 0.4046207368373871,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7031249701976776,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.46875,
      "epoch": 1.6886145404663924,
      "grad_norm": 3.681758432983122,
      "kl": 0.130859375,
      "learning_rate": 4.409090909090909e-07,
      "loss": 0.0052,
      "reward": 0.811718761920929,
      "reward_std": 0.5958449244499207,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.4210937023162842,
      "step": 615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.203125,
      "epoch": 1.691358024691358,
      "grad_norm": 5.594612819372044,
      "kl": 0.091796875,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0037,
      "reward": 1.2671875357627869,
      "reward_std": 0.4626980125904083,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6421874761581421,
      "step": 616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.6875,
      "epoch": 1.6941015089163236,
      "grad_norm": 3.7730089340092716,
      "kl": 0.1142578125,
      "learning_rate": 4.390909090909091e-07,
      "loss": 0.0046,
      "reward": 1.449999988079071,
      "reward_std": 0.4162885248661041,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.731249988079071,
      "step": 617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.078125,
      "epoch": 1.6968449931412894,
      "grad_norm": 4.643329708435787,
      "kl": 0.0986328125,
      "learning_rate": 4.3818181818181816e-07,
      "loss": 0.0039,
      "reward": 1.4625000357627869,
      "reward_std": 0.5247026681900024,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7281249761581421,
      "step": 618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.0625,
      "epoch": 1.6995884773662553,
      "grad_norm": 3.6048763111718376,
      "kl": 0.08935546875,
      "learning_rate": 4.3727272727272723e-07,
      "loss": 0.0036,
      "reward": 1.125781238079071,
      "reward_std": 0.44691573083400726,
      "rewards/choice_reward": 0.5625,
      "rewards/roam_reward": 0.5632812231779099,
      "step": 619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.453125,
      "epoch": 1.702331961591221,
      "grad_norm": 4.4723464168826075,
      "kl": 0.147705078125,
      "learning_rate": 4.363636363636363e-07,
      "loss": 0.0059,
      "reward": 1.6921875476837158,
      "reward_std": 0.4920467585325241,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8328125476837158,
      "step": 620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.21875,
      "epoch": 1.7050754458161865,
      "grad_norm": 3.3386140274358804,
      "kl": 0.114990234375,
      "learning_rate": 4.3545454545454543e-07,
      "loss": 0.0046,
      "reward": 1.299218773841858,
      "reward_std": 0.5712471902370453,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6585937440395355,
      "step": 621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.71875,
      "epoch": 1.7078189300411522,
      "grad_norm": 2.621699357498961,
      "kl": 0.10546875,
      "learning_rate": 4.3454545454545455e-07,
      "loss": 0.0042,
      "reward": 1.7265625,
      "reward_std": 0.3452789708971977,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8671874701976776,
      "step": 622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 67.75,
      "epoch": 1.710562414266118,
      "grad_norm": 5.4962516246306,
      "kl": 0.12109375,
      "learning_rate": 4.336363636363636e-07,
      "loss": 0.0049,
      "reward": 1.54296875,
      "reward_std": 0.5655232667922974,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7773437201976776,
      "step": 623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.03125,
      "epoch": 1.7133058984910838,
      "grad_norm": 3.6643652460769727,
      "kl": 0.081298828125,
      "learning_rate": 4.327272727272727e-07,
      "loss": 0.0033,
      "reward": 1.635937511920929,
      "reward_std": 0.305075004696846,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.807812511920929,
      "step": 624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.828125,
      "epoch": 1.7160493827160495,
      "grad_norm": 4.032061968412479,
      "kl": 0.11181640625,
      "learning_rate": 4.318181818181818e-07,
      "loss": 0.0045,
      "reward": 1.167187511920929,
      "reward_std": 0.7204632759094238,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.5890624821186066,
      "step": 625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.953125,
      "epoch": 1.718792866941015,
      "grad_norm": 5.402039980385757,
      "kl": 0.11572265625,
      "learning_rate": 4.309090909090909e-07,
      "loss": 0.0046,
      "reward": 1.650781273841858,
      "reward_std": 0.6595319211483002,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8226562440395355,
      "step": 626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.984375,
      "epoch": 1.7215363511659807,
      "grad_norm": 5.258114174397278,
      "kl": 0.099365234375,
      "learning_rate": 4.2999999999999996e-07,
      "loss": 0.004,
      "reward": 1.697656273841858,
      "reward_std": 0.4678082764148712,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8539062440395355,
      "step": 627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.5,
      "epoch": 1.7242798353909465,
      "grad_norm": 4.930678158123674,
      "kl": 0.11865234375,
      "learning_rate": 4.290909090909091e-07,
      "loss": 0.0047,
      "reward": 1.543749988079071,
      "reward_std": 0.5408697128295898,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.778124988079071,
      "step": 628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.484375,
      "epoch": 1.7270233196159122,
      "grad_norm": 4.254385209226658,
      "kl": 0.11083984375,
      "learning_rate": 4.2818181818181815e-07,
      "loss": 0.0044,
      "reward": 1.7835937142372131,
      "reward_std": 0.30397772043943405,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8773437738418579,
      "step": 629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.328125,
      "epoch": 1.729766803840878,
      "grad_norm": 3.393291956631288,
      "kl": 0.133544921875,
      "learning_rate": 4.272727272727272e-07,
      "loss": 0.0054,
      "reward": 1.2789062857627869,
      "reward_std": 0.184274067170918,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6539062559604645,
      "step": 630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.90625,
      "epoch": 1.7325102880658436,
      "grad_norm": 3.301216041669399,
      "kl": 0.10302734375,
      "learning_rate": 4.2636363636363635e-07,
      "loss": 0.0041,
      "reward": 1.4796875715255737,
      "reward_std": 0.4603932499885559,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7453124821186066,
      "step": 631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.78125,
      "epoch": 1.7352537722908092,
      "grad_norm": 5.67118213131503,
      "kl": 0.1005859375,
      "learning_rate": 4.2545454545454547e-07,
      "loss": 0.004,
      "reward": 1.578125,
      "reward_std": 0.33960459381341934,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7968749701976776,
      "step": 632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.5625,
      "epoch": 1.737997256515775,
      "grad_norm": 16.737984619382317,
      "kl": 0.125244140625,
      "learning_rate": 4.2454545454545454e-07,
      "loss": 0.005,
      "reward": 1.3218750357627869,
      "reward_std": 0.6554046273231506,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6656249761581421,
      "step": 633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.484375,
      "epoch": 1.7407407407407407,
      "grad_norm": 23.548761490488786,
      "kl": 0.105224609375,
      "learning_rate": 4.236363636363636e-07,
      "loss": 0.0042,
      "reward": 1.5374999642372131,
      "reward_std": 0.4010937958955765,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7718749940395355,
      "step": 634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.390625,
      "epoch": 1.7434842249657065,
      "grad_norm": 3.9195780766238997,
      "kl": 0.111083984375,
      "learning_rate": 4.227272727272727e-07,
      "loss": 0.0044,
      "reward": 1.430468738079071,
      "reward_std": 0.13726941123604774,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.727343738079071,
      "step": 635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.84375,
      "epoch": 1.7462277091906722,
      "grad_norm": 4.673777791186161,
      "kl": 0.128662109375,
      "learning_rate": 4.218181818181818e-07,
      "loss": 0.0052,
      "reward": 1.7734375,
      "reward_std": 0.32488279044628143,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8828124701976776,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.90625,
      "epoch": 1.7489711934156378,
      "grad_norm": 4.151451727323081,
      "kl": 0.11572265625,
      "learning_rate": 4.209090909090909e-07,
      "loss": 0.0046,
      "reward": 1.5406250357627869,
      "reward_std": 0.6340201795101166,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7749999761581421,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.65625,
      "epoch": 1.7517146776406034,
      "grad_norm": 4.804736060113988,
      "kl": 0.120849609375,
      "learning_rate": 4.1999999999999995e-07,
      "loss": 0.0048,
      "reward": 1.5140624642372131,
      "reward_std": 0.675272136926651,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7328124940395355,
      "step": 638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.59375,
      "epoch": 1.7544581618655692,
      "grad_norm": 4.015887733422738,
      "kl": 0.10302734375,
      "learning_rate": 4.1909090909090907e-07,
      "loss": 0.0041,
      "reward": 1.204687476158142,
      "reward_std": 0.45923222601413727,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6109374761581421,
      "step": 639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.453125,
      "epoch": 1.757201646090535,
      "grad_norm": 1.7349024210804238,
      "kl": 0.115234375,
      "learning_rate": 4.1818181818181814e-07,
      "loss": 0.0046,
      "reward": 1.6375000476837158,
      "reward_std": 0.13271774351596832,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8249999582767487,
      "step": 640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.15625,
      "epoch": 1.7599451303155007,
      "grad_norm": 2.9720485278284987,
      "kl": 0.130859375,
      "learning_rate": 4.1727272727272727e-07,
      "loss": 0.0052,
      "reward": 1.864062488079071,
      "reward_std": 0.22904610633850098,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9109375178813934,
      "step": 641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.0625,
      "epoch": 1.7626886145404663,
      "grad_norm": 2.9198017047150593,
      "kl": 0.15087890625,
      "learning_rate": 4.1636363636363634e-07,
      "loss": 0.006,
      "reward": 1.598437488079071,
      "reward_std": 0.48394007980823517,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8015624582767487,
      "step": 642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.84375,
      "epoch": 1.765432098765432,
      "grad_norm": 4.5819328944239714,
      "kl": 0.13330078125,
      "learning_rate": 4.1545454545454546e-07,
      "loss": 0.0053,
      "reward": 1.57421875,
      "reward_std": 0.5152434408664703,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7773437201976776,
      "step": 643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.8125,
      "epoch": 1.7681755829903978,
      "grad_norm": 2.9261270753295068,
      "kl": 0.10205078125,
      "learning_rate": 4.1454545454545453e-07,
      "loss": 0.0041,
      "reward": 1.646875023841858,
      "reward_std": 0.49189814925193787,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8187499940395355,
      "step": 644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.46875,
      "epoch": 1.7709190672153636,
      "grad_norm": 14.343148169313086,
      "kl": 0.105712890625,
      "learning_rate": 4.136363636363636e-07,
      "loss": 0.0042,
      "reward": 1.561718761920929,
      "reward_std": 0.2597835436463356,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.780468761920929,
      "step": 645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.5,
      "epoch": 1.7736625514403292,
      "grad_norm": 7.952157247384164,
      "kl": 0.18896484375,
      "learning_rate": 4.1272727272727273e-07,
      "loss": 0.0076,
      "reward": 1.48046875,
      "reward_std": 0.5070348381996155,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.74609375,
      "step": 646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.015625,
      "epoch": 1.7764060356652949,
      "grad_norm": 4.110872634394268,
      "kl": 0.102783203125,
      "learning_rate": 4.118181818181818e-07,
      "loss": 0.0041,
      "reward": 1.28515625,
      "reward_std": 0.5382444858551025,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.66015625,
      "step": 647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.25,
      "epoch": 1.7791495198902605,
      "grad_norm": 4.348289505877314,
      "kl": 0.111083984375,
      "learning_rate": 4.1090909090909087e-07,
      "loss": 0.0044,
      "reward": 0.8304687738418579,
      "reward_std": 0.8299466967582703,
      "rewards/choice_reward": 0.390625,
      "rewards/roam_reward": 0.4398437440395355,
      "step": 648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.09375,
      "epoch": 1.7818930041152263,
      "grad_norm": 3.642364932683399,
      "kl": 0.1640625,
      "learning_rate": 4.0999999999999994e-07,
      "loss": 0.0066,
      "reward": 1.5789062976837158,
      "reward_std": 0.2790534794330597,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7820312678813934,
      "step": 649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.78125,
      "epoch": 1.7846364883401922,
      "grad_norm": 12.979111094428676,
      "kl": 0.104736328125,
      "learning_rate": 4.090909090909091e-07,
      "loss": 0.0042,
      "reward": 1.66796875,
      "reward_std": 0.45473532378673553,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.83984375,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.765625,
      "epoch": 1.7873799725651578,
      "grad_norm": 2.1645407688475804,
      "kl": 0.115478515625,
      "learning_rate": 4.081818181818182e-07,
      "loss": 0.0046,
      "reward": 1.671875,
      "reward_std": 0.21265346556901932,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8437500298023224,
      "step": 651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.53125,
      "epoch": 1.7901234567901234,
      "grad_norm": 12.934790238319271,
      "kl": 0.115478515625,
      "learning_rate": 4.0727272727272726e-07,
      "loss": 0.0046,
      "reward": 1.686718761920929,
      "reward_std": 0.15634502097964287,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.842968761920929,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.578125,
      "epoch": 1.792866941015089,
      "grad_norm": 2.165748636888658,
      "kl": 0.122314453125,
      "learning_rate": 4.0636363636363633e-07,
      "loss": 0.0049,
      "reward": 1.965624988079071,
      "reward_std": 0.09722718130797148,
      "rewards/choice_reward": 0.984375,
      "rewards/roam_reward": 0.981249988079071,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.953125,
      "epoch": 1.7956104252400549,
      "grad_norm": 6.944281402277241,
      "kl": 0.15185546875,
      "learning_rate": 4.0545454545454545e-07,
      "loss": 0.0061,
      "reward": 1.46484375,
      "reward_std": 0.5038956105709076,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7304687201976776,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.796875,
      "epoch": 1.7983539094650207,
      "grad_norm": 4.714143244008144,
      "kl": 0.133544921875,
      "learning_rate": 4.045454545454545e-07,
      "loss": 0.0053,
      "reward": 1.044531226158142,
      "reward_std": 0.2878062203526497,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5445312559604645,
      "step": 655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.453125,
      "epoch": 1.8010973936899863,
      "grad_norm": 3.510647708440207,
      "kl": 0.16796875,
      "learning_rate": 4.036363636363636e-07,
      "loss": 0.0067,
      "reward": 1.1023436784744263,
      "reward_std": 0.5571488589048386,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5710937082767487,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.59375,
      "epoch": 1.803840877914952,
      "grad_norm": 2.2097975631180664,
      "kl": 0.111572265625,
      "learning_rate": 4.027272727272727e-07,
      "loss": 0.0045,
      "reward": 1.845312476158142,
      "reward_std": 0.14059560280293226,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9234374761581421,
      "step": 657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.625,
      "epoch": 1.8065843621399176,
      "grad_norm": 16.711973588366792,
      "kl": 0.15478515625,
      "learning_rate": 4.018181818181818e-07,
      "loss": 0.0062,
      "reward": 1.635156273841858,
      "reward_std": 0.2930929362773895,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8226562440395355,
      "step": 658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.8125,
      "epoch": 1.8093278463648834,
      "grad_norm": 5.499431166564593,
      "kl": 0.12841796875,
      "learning_rate": 4.0090909090909086e-07,
      "loss": 0.0051,
      "reward": 1.571093738079071,
      "reward_std": 0.4677266329526901,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7898437976837158,
      "step": 659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.03125,
      "epoch": 1.8120713305898493,
      "grad_norm": 3.3422892166875124,
      "kl": 0.125244140625,
      "learning_rate": 4e-07,
      "loss": 0.005,
      "reward": 1.5195311903953552,
      "reward_std": 0.5444206297397614,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7539062201976776,
      "step": 660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.8125,
      "epoch": 1.8148148148148149,
      "grad_norm": 3.4788880821145636,
      "kl": 0.0966796875,
      "learning_rate": 3.990909090909091e-07,
      "loss": 0.0039,
      "reward": 1.357812523841858,
      "reward_std": 0.5057743340730667,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.53125,
      "epoch": 1.8175582990397805,
      "grad_norm": 5.254697267894281,
      "kl": 0.11572265625,
      "learning_rate": 3.981818181818182e-07,
      "loss": 0.0046,
      "reward": 1.267187476158142,
      "reward_std": 0.5789059698581696,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6421874761581421,
      "step": 662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.109375,
      "epoch": 1.8203017832647461,
      "grad_norm": 8.348894651119013,
      "kl": 0.105712890625,
      "learning_rate": 3.9727272727272725e-07,
      "loss": 0.0042,
      "reward": 1.6343750357627869,
      "reward_std": 0.46311406791210175,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8062499761581421,
      "step": 663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.59375,
      "epoch": 1.823045267489712,
      "grad_norm": 22.507933396073454,
      "kl": 0.104736328125,
      "learning_rate": 3.9636363636363637e-07,
      "loss": 0.0042,
      "reward": 1.87109375,
      "reward_std": 0.22839166969060898,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.93359375,
      "step": 664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.96875,
      "epoch": 1.8257887517146778,
      "grad_norm": 5.566050816433911,
      "kl": 0.15380859375,
      "learning_rate": 3.9545454545454544e-07,
      "loss": 0.0062,
      "reward": 1.389062523841858,
      "reward_std": 0.5681444704532623,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6859374940395355,
      "step": 665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.515625,
      "epoch": 1.8285322359396434,
      "grad_norm": 4.828734408740684,
      "kl": 0.125732421875,
      "learning_rate": 3.945454545454545e-07,
      "loss": 0.005,
      "reward": 1.395312488079071,
      "reward_std": 0.5245100259780884,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7078125178813934,
      "step": 666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.34375,
      "epoch": 1.831275720164609,
      "grad_norm": 7.921463334118804,
      "kl": 0.10888671875,
      "learning_rate": 3.936363636363636e-07,
      "loss": 0.0043,
      "reward": 1.787500023841858,
      "reward_std": 0.3248981386423111,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8812500238418579,
      "step": 667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 65.8125,
      "epoch": 1.8340192043895747,
      "grad_norm": 10.30012458597402,
      "kl": 0.146484375,
      "learning_rate": 3.927272727272727e-07,
      "loss": 0.0059,
      "reward": 1.3664062321186066,
      "reward_std": 0.3465329110622406,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.694531261920929,
      "step": 668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.671875,
      "epoch": 1.8367626886145405,
      "grad_norm": 3.219222098247594,
      "kl": 0.095703125,
      "learning_rate": 3.9181818181818183e-07,
      "loss": 0.0038,
      "reward": 1.5828125476837158,
      "reward_std": 0.20820768177509308,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.8015625476837158,
      "step": 669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.703125,
      "epoch": 1.8395061728395061,
      "grad_norm": 4.128373118044727,
      "kl": 0.13623046875,
      "learning_rate": 3.909090909090909e-07,
      "loss": 0.0055,
      "reward": 1.517187476158142,
      "reward_std": 0.45177534222602844,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7671874761581421,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.484375,
      "epoch": 1.842249657064472,
      "grad_norm": 6.845986594822972,
      "kl": 0.1064453125,
      "learning_rate": 3.8999999999999997e-07,
      "loss": 0.0043,
      "reward": 1.460156261920929,
      "reward_std": 0.6346226334571838,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7414062321186066,
      "step": 671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.125,
      "epoch": 1.8449931412894376,
      "grad_norm": 3.1518606793665565,
      "kl": 0.1015625,
      "learning_rate": 3.890909090909091e-07,
      "loss": 0.0041,
      "reward": 1.546875,
      "reward_std": 0.429079607129097,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.78125,
      "step": 672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.90625,
      "epoch": 1.8477366255144032,
      "grad_norm": 2.8245002884513646,
      "kl": 0.103271484375,
      "learning_rate": 3.8818181818181817e-07,
      "loss": 0.0041,
      "reward": 1.564843773841858,
      "reward_std": 0.2965794578194618,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7835937440395355,
      "step": 673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.015625,
      "epoch": 1.850480109739369,
      "grad_norm": 6.586851850905839,
      "kl": 0.12744140625,
      "learning_rate": 3.8727272727272724e-07,
      "loss": 0.0051,
      "reward": 1.552343726158142,
      "reward_std": 0.6085359454154968,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7710937559604645,
      "step": 674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.78125,
      "epoch": 1.8532235939643347,
      "grad_norm": 5.987304118798936,
      "kl": 0.116943359375,
      "learning_rate": 3.8636363636363636e-07,
      "loss": 0.0047,
      "reward": 1.310156226158142,
      "reward_std": 0.49685972929000854,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6695312261581421,
      "step": 675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.8125,
      "epoch": 1.8559670781893005,
      "grad_norm": 3.561802904173307,
      "kl": 0.109375,
      "learning_rate": 3.8545454545454543e-07,
      "loss": 0.0044,
      "reward": 1.3343749046325684,
      "reward_std": 0.5377295911312103,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6781249940395355,
      "step": 676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.15625,
      "epoch": 1.8587105624142661,
      "grad_norm": 4.178869011124296,
      "kl": 0.107177734375,
      "learning_rate": 3.845454545454545e-07,
      "loss": 0.0043,
      "reward": 1.5750000476837158,
      "reward_std": 0.5704424381256104,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7937499582767487,
      "step": 677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.890625,
      "epoch": 1.8614540466392318,
      "grad_norm": 6.402272643220298,
      "kl": 0.19482421875,
      "learning_rate": 3.8363636363636357e-07,
      "loss": 0.0078,
      "reward": 1.8406250476837158,
      "reward_std": 0.2866463288664818,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.903124988079071,
      "step": 678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.140625,
      "epoch": 1.8641975308641974,
      "grad_norm": 2.8322674489238135,
      "kl": 0.1171875,
      "learning_rate": 3.8272727272727275e-07,
      "loss": 0.0047,
      "reward": 1.7257812023162842,
      "reward_std": 0.33524563908576965,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8664062321186066,
      "step": 679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.125,
      "epoch": 1.8669410150891632,
      "grad_norm": 2.868827204779528,
      "kl": 0.137451171875,
      "learning_rate": 3.818181818181818e-07,
      "loss": 0.0055,
      "reward": 1.86328125,
      "reward_std": 0.24384143203496933,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9101562798023224,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.6875,
      "epoch": 1.869684499314129,
      "grad_norm": 3.46762257318447,
      "kl": 0.1103515625,
      "learning_rate": 3.809090909090909e-07,
      "loss": 0.0044,
      "reward": 1.6320312023162842,
      "reward_std": 0.29449596256017685,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8039062023162842,
      "step": 681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.59375,
      "epoch": 1.8724279835390947,
      "grad_norm": 8.428313963135231,
      "kl": 0.111572265625,
      "learning_rate": 3.7999999999999996e-07,
      "loss": 0.0045,
      "reward": 0.9843749403953552,
      "reward_std": 0.7652322351932526,
      "rewards/choice_reward": 0.46875,
      "rewards/roam_reward": 0.5156249701976776,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.234375,
      "epoch": 1.8751714677640603,
      "grad_norm": 3.6179704590270587,
      "kl": 0.1181640625,
      "learning_rate": 3.790909090909091e-07,
      "loss": 0.0047,
      "reward": 1.6671875715255737,
      "reward_std": 0.46436551213264465,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8234374821186066,
      "step": 683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.484375,
      "epoch": 1.877914951989026,
      "grad_norm": 3.0428398037737105,
      "kl": 0.117919921875,
      "learning_rate": 3.7818181818181816e-07,
      "loss": 0.0047,
      "reward": 0.98046875,
      "reward_std": 0.5315862596035004,
      "rewards/choice_reward": 0.484375,
      "rewards/roam_reward": 0.49609375,
      "step": 684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.75,
      "epoch": 1.8806584362139918,
      "grad_norm": 3.110597703409818,
      "kl": 0.09521484375,
      "learning_rate": 3.7727272727272723e-07,
      "loss": 0.0038,
      "reward": 1.0960937142372131,
      "reward_std": 0.4587474763393402,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5648437440395355,
      "step": 685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.21875,
      "epoch": 1.8834019204389576,
      "grad_norm": 6.979388930776721,
      "kl": 0.1064453125,
      "learning_rate": 3.7636363636363635e-07,
      "loss": 0.0043,
      "reward": 1.598437488079071,
      "reward_std": 0.481655552983284,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8015625178813934,
      "step": 686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 64.40625,
      "epoch": 1.8861454046639232,
      "grad_norm": 2.6032786356111064,
      "kl": 0.104248046875,
      "learning_rate": 3.754545454545454e-07,
      "loss": 0.0042,
      "reward": 1.583593726158142,
      "reward_std": 0.1682818979024887,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.8023437261581421,
      "step": 687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.65625,
      "epoch": 1.8888888888888888,
      "grad_norm": 6.423702414090454,
      "kl": 0.11865234375,
      "learning_rate": 3.7454545454545454e-07,
      "loss": 0.0048,
      "reward": 1.9015624523162842,
      "reward_std": 0.23329924046993256,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.9328124523162842,
      "step": 688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.390625,
      "epoch": 1.8916323731138545,
      "grad_norm": 45.432742969077395,
      "kl": 0.109375,
      "learning_rate": 3.736363636363636e-07,
      "loss": 0.0044,
      "reward": 1.560156226158142,
      "reward_std": 0.2677382007241249,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7789062559604645,
      "step": 689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.234375,
      "epoch": 1.8943758573388203,
      "grad_norm": 15.396758274487498,
      "kl": 0.1298828125,
      "learning_rate": 3.7272727272727274e-07,
      "loss": 0.0052,
      "reward": 1.870312511920929,
      "reward_std": 0.23891766369342804,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.932812511920929,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.765625,
      "epoch": 1.8971193415637861,
      "grad_norm": 3.9482502637428425,
      "kl": 0.113037109375,
      "learning_rate": 3.718181818181818e-07,
      "loss": 0.0045,
      "reward": 1.549218773841858,
      "reward_std": 0.3217321038246155,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7835937440395355,
      "step": 691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.140625,
      "epoch": 1.8998628257887518,
      "grad_norm": 4.5071017302960295,
      "kl": 0.10009765625,
      "learning_rate": 3.709090909090909e-07,
      "loss": 0.004,
      "reward": 1.2515625357627869,
      "reward_std": 0.4278821647167206,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6421874463558197,
      "step": 692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.75,
      "epoch": 1.9026063100137174,
      "grad_norm": 2.244419627785846,
      "kl": 0.110595703125,
      "learning_rate": 3.7e-07,
      "loss": 0.0044,
      "reward": 1.643750011920929,
      "reward_std": 0.2198822796344757,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.831250011920929,
      "step": 693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.796875,
      "epoch": 1.905349794238683,
      "grad_norm": 4.384703844771046,
      "kl": 0.092041015625,
      "learning_rate": 3.690909090909091e-07,
      "loss": 0.0037,
      "reward": 1.189062476158142,
      "reward_std": 0.25614288449287415,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.6109374761581421,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.484375,
      "epoch": 1.9080932784636488,
      "grad_norm": 5.563534909388885,
      "kl": 0.099365234375,
      "learning_rate": 3.6818181818181815e-07,
      "loss": 0.004,
      "reward": 1.778124988079071,
      "reward_std": 0.23368481919169426,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.887499988079071,
      "step": 695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.03125,
      "epoch": 1.9108367626886147,
      "grad_norm": 4.027865113179509,
      "kl": 0.12548828125,
      "learning_rate": 3.672727272727272e-07,
      "loss": 0.005,
      "reward": 1.228906273841858,
      "reward_std": 0.6479907333850861,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6195312142372131,
      "step": 696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.671875,
      "epoch": 1.9135802469135803,
      "grad_norm": 4.882822249350005,
      "kl": 0.105224609375,
      "learning_rate": 3.663636363636364e-07,
      "loss": 0.0042,
      "reward": 1.271093726158142,
      "reward_std": 0.40971219539642334,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6148436963558197,
      "step": 697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.984375,
      "epoch": 1.916323731138546,
      "grad_norm": 3.900258569974571,
      "kl": 0.102783203125,
      "learning_rate": 3.6545454545454546e-07,
      "loss": 0.0041,
      "reward": 1.635156273841858,
      "reward_std": 0.46128255128860474,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8226562738418579,
      "step": 698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.546875,
      "epoch": 1.9190672153635115,
      "grad_norm": 2.981850065788348,
      "kl": 0.098876953125,
      "learning_rate": 3.6454545454545453e-07,
      "loss": 0.004,
      "reward": 0.946093738079071,
      "reward_std": 0.39172008633613586,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.44609374552965164,
      "step": 699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.71875,
      "epoch": 1.9218106995884774,
      "grad_norm": 3.098333194742997,
      "kl": 0.119384765625,
      "learning_rate": 3.636363636363636e-07,
      "loss": 0.0048,
      "reward": 1.6484374403953552,
      "reward_std": 0.39533631503582,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8203124701976776,
      "step": 700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.046875,
      "epoch": 1.9245541838134432,
      "grad_norm": 5.62601109975019,
      "kl": 0.108154296875,
      "learning_rate": 3.6272727272727273e-07,
      "loss": 0.0043,
      "reward": 1.2968750596046448,
      "reward_std": 0.5830048173666,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.65625,
      "step": 701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.234375,
      "epoch": 1.9272976680384089,
      "grad_norm": 6.641903451864133,
      "kl": 0.093505859375,
      "learning_rate": 3.618181818181818e-07,
      "loss": 0.0037,
      "reward": 1.2617188096046448,
      "reward_std": 0.637585312128067,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.63671875,
      "step": 702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.390625,
      "epoch": 1.9300411522633745,
      "grad_norm": 3.2872816549573898,
      "kl": 0.0869140625,
      "learning_rate": 3.6090909090909087e-07,
      "loss": 0.0035,
      "reward": 1.5468749403953552,
      "reward_std": 0.49051383882761,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.78125,
      "step": 703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.46875,
      "epoch": 1.93278463648834,
      "grad_norm": 3.6181831610348354,
      "kl": 0.11181640625,
      "learning_rate": 3.6e-07,
      "loss": 0.0045,
      "reward": 1.3617187142372131,
      "reward_std": 0.24989521596580744,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6898437440395355,
      "step": 704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.40625,
      "epoch": 1.935528120713306,
      "grad_norm": 3.109006593704769,
      "kl": 0.10595703125,
      "learning_rate": 3.5909090909090907e-07,
      "loss": 0.0042,
      "reward": 1.7625000476837158,
      "reward_std": 0.48489242792129517,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.871874988079071,
      "step": 705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.734375,
      "epoch": 1.9382716049382716,
      "grad_norm": 5.473098192483056,
      "kl": 0.123779296875,
      "learning_rate": 3.5818181818181814e-07,
      "loss": 0.005,
      "reward": 1.4718750715255737,
      "reward_std": 0.25257018208503723,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7374999523162842,
      "step": 706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.734375,
      "epoch": 1.9410150891632374,
      "grad_norm": 4.445413875832428,
      "kl": 0.11083984375,
      "learning_rate": 3.5727272727272726e-07,
      "loss": 0.0044,
      "reward": 1.393750011920929,
      "reward_std": 0.5073824375867844,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7062499821186066,
      "step": 707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.578125,
      "epoch": 1.943758573388203,
      "grad_norm": 4.560508526779024,
      "kl": 0.197998046875,
      "learning_rate": 3.563636363636364e-07,
      "loss": 0.0079,
      "reward": 1.428125023841858,
      "reward_std": 0.39360281080007553,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7249999642372131,
      "step": 708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.40625,
      "epoch": 1.9465020576131686,
      "grad_norm": 2.17085628132125,
      "kl": 0.09814453125,
      "learning_rate": 3.5545454545454545e-07,
      "loss": 0.0039,
      "reward": 1.800000011920929,
      "reward_std": 0.3996886909008026,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.893750011920929,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.28125,
      "epoch": 1.9492455418381345,
      "grad_norm": 3.57937268248927,
      "kl": 0.126220703125,
      "learning_rate": 3.545454545454545e-07,
      "loss": 0.005,
      "reward": 1.3960937857627869,
      "reward_std": 0.6148386299610138,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.6929687559604645,
      "step": 710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.375,
      "epoch": 1.9519890260631,
      "grad_norm": 3.908202040629821,
      "kl": 0.08984375,
      "learning_rate": 3.536363636363636e-07,
      "loss": 0.0036,
      "reward": 1.6328125596046448,
      "reward_std": 0.4680548459291458,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8203125,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.125,
      "epoch": 1.954732510288066,
      "grad_norm": 2.184472228624725,
      "kl": 0.1005859375,
      "learning_rate": 3.527272727272727e-07,
      "loss": 0.004,
      "reward": 1.965624988079071,
      "reward_std": 0.09722718130797148,
      "rewards/choice_reward": 1.0,
      "rewards/roam_reward": 0.965624988079071,
      "step": 712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.203125,
      "epoch": 1.9574759945130316,
      "grad_norm": 7.86769835988023,
      "kl": 0.100830078125,
      "learning_rate": 3.518181818181818e-07,
      "loss": 0.004,
      "reward": 1.56640625,
      "reward_std": 0.366900697350502,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.78515625,
      "step": 713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.578125,
      "epoch": 1.9602194787379972,
      "grad_norm": 4.349028025159725,
      "kl": 0.10107421875,
      "learning_rate": 3.5090909090909086e-07,
      "loss": 0.004,
      "reward": 1.495312511920929,
      "reward_std": 0.4397459104657173,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7609374523162842,
      "step": 714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.078125,
      "epoch": 1.9629629629629628,
      "grad_norm": 3.6619919266466,
      "kl": 0.106689453125,
      "learning_rate": 3.5e-07,
      "loss": 0.0043,
      "reward": 1.5539062023162842,
      "reward_std": 0.3623511716723442,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7882812321186066,
      "step": 715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.453125,
      "epoch": 1.9657064471879286,
      "grad_norm": 8.744295572219698,
      "kl": 0.09619140625,
      "learning_rate": 3.490909090909091e-07,
      "loss": 0.0039,
      "reward": 1.357812523841858,
      "reward_std": 0.450033038854599,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859374940395355,
      "step": 716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.5625,
      "epoch": 1.9684499314128945,
      "grad_norm": 5.8340616106872885,
      "kl": 0.103515625,
      "learning_rate": 3.481818181818182e-07,
      "loss": 0.0041,
      "reward": 1.456250011920929,
      "reward_std": 0.42738912999629974,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7374999523162842,
      "step": 717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.734375,
      "epoch": 1.97119341563786,
      "grad_norm": 6.850305195091276,
      "kl": 0.173828125,
      "learning_rate": 3.4727272727272725e-07,
      "loss": 0.007,
      "reward": 1.5437500476837158,
      "reward_std": 0.4188317656517029,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.778124988079071,
      "step": 718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.15625,
      "epoch": 1.9739368998628257,
      "grad_norm": 3.5350921655226535,
      "kl": 0.098388671875,
      "learning_rate": 3.4636363636363637e-07,
      "loss": 0.0039,
      "reward": 1.642968773841858,
      "reward_std": 0.42786234617233276,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8304687738418579,
      "step": 719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.4375,
      "epoch": 1.9766803840877913,
      "grad_norm": 4.031256948107078,
      "kl": 0.139892578125,
      "learning_rate": 3.4545454545454544e-07,
      "loss": 0.0056,
      "reward": 1.5546875596046448,
      "reward_std": 0.47294948995113373,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7265625,
      "step": 720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.71875,
      "epoch": 1.9794238683127572,
      "grad_norm": 5.45796597753534,
      "kl": 0.108642578125,
      "learning_rate": 3.445454545454545e-07,
      "loss": 0.0043,
      "reward": 1.260156273841858,
      "reward_std": 0.2761820927262306,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6195312440395355,
      "step": 721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.546875,
      "epoch": 1.982167352537723,
      "grad_norm": 4.478599438907719,
      "kl": 0.112060546875,
      "learning_rate": 3.4363636363636364e-07,
      "loss": 0.0045,
      "reward": 1.6734375357627869,
      "reward_std": 0.3208591490983963,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8453124761581421,
      "step": 722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.0,
      "epoch": 1.9849108367626886,
      "grad_norm": 6.9127924619208025,
      "kl": 0.1376953125,
      "learning_rate": 3.427272727272727e-07,
      "loss": 0.0055,
      "reward": 1.4648436903953552,
      "reward_std": 0.4159834384918213,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.74609375,
      "step": 723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.515625,
      "epoch": 1.9876543209876543,
      "grad_norm": 12.186255810002473,
      "kl": 0.10693359375,
      "learning_rate": 3.418181818181818e-07,
      "loss": 0.0043,
      "reward": 1.634374976158142,
      "reward_std": 0.24806804955005646,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8062500059604645,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.6875,
      "epoch": 1.9903978052126199,
      "grad_norm": 4.467905979983616,
      "kl": 0.1025390625,
      "learning_rate": 3.4090909090909085e-07,
      "loss": 0.0041,
      "reward": 1.5320312976837158,
      "reward_std": 0.6255079209804535,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.766406238079071,
      "step": 725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.09375,
      "epoch": 1.9931412894375857,
      "grad_norm": 2.9699493149022604,
      "kl": 0.103759765625,
      "learning_rate": 3.4000000000000003e-07,
      "loss": 0.0042,
      "reward": 1.504687488079071,
      "reward_std": 0.46675867587327957,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.754687488079071,
      "step": 726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.375,
      "epoch": 1.9958847736625516,
      "grad_norm": 7.305345924991261,
      "kl": 0.12255859375,
      "learning_rate": 3.390909090909091e-07,
      "loss": 0.0049,
      "reward": 1.297656238079071,
      "reward_std": 0.4874522387981415,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.641406238079071,
      "step": 727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.0,
      "epoch": 1.9986282578875172,
      "grad_norm": 2.604818082870772,
      "kl": 0.092041015625,
      "learning_rate": 3.3818181818181817e-07,
      "loss": 0.0045,
      "reward": 1.050000000745058,
      "reward_std": 0.3358757197856903,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5500000007450581,
      "step": 728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.828125,
      "epoch": 2.0027434842249656,
      "grad_norm": 2.707911215722067,
      "kl": 0.135986328125,
      "learning_rate": 3.3727272727272724e-07,
      "loss": 0.0054,
      "reward": 1.59375,
      "reward_std": 0.40535733103752136,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.796875,
      "step": 729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.203125,
      "epoch": 2.0054869684499312,
      "grad_norm": 8.722483521923781,
      "kl": 0.123291015625,
      "learning_rate": 3.3636363636363636e-07,
      "loss": 0.0049,
      "reward": 1.4906249642372131,
      "reward_std": 0.4499505013227463,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7562499940395355,
      "step": 730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.765625,
      "epoch": 2.0082304526748973,
      "grad_norm": 2.664329462311894,
      "kl": 0.12646484375,
      "learning_rate": 3.3545454545454543e-07,
      "loss": 0.0051,
      "reward": 1.475000023841858,
      "reward_std": 0.4457009881734848,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7406249940395355,
      "step": 731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.6875,
      "epoch": 2.010973936899863,
      "grad_norm": 3.5090566136071533,
      "kl": 0.095458984375,
      "learning_rate": 3.345454545454545e-07,
      "loss": 0.0038,
      "reward": 1.551562488079071,
      "reward_std": 0.286793552339077,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.785937488079071,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.484375,
      "epoch": 2.0137174211248285,
      "grad_norm": 2.7358123022416807,
      "kl": 0.12890625,
      "learning_rate": 3.3363636363636363e-07,
      "loss": 0.0051,
      "reward": 1.731249988079071,
      "reward_std": 0.1946544125676155,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8718750178813934,
      "step": 733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.5625,
      "epoch": 2.016460905349794,
      "grad_norm": 1.964439423485233,
      "kl": 0.104248046875,
      "learning_rate": 3.327272727272727e-07,
      "loss": 0.0042,
      "reward": 1.9375,
      "reward_std": 0.1767766922712326,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.96875,
      "step": 734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.609375,
      "epoch": 2.01920438957476,
      "grad_norm": 4.757411261475242,
      "kl": 0.1103515625,
      "learning_rate": 3.318181818181818e-07,
      "loss": 0.0044,
      "reward": 1.8453125357627869,
      "reward_std": 0.3698854446411133,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9234375059604645,
      "step": 735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.828125,
      "epoch": 2.021947873799726,
      "grad_norm": 4.546067844658221,
      "kl": 0.1025390625,
      "learning_rate": 3.309090909090909e-07,
      "loss": 0.0041,
      "reward": 1.6734375357627869,
      "reward_std": 0.25190678983926773,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8453124761581421,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.8125,
      "epoch": 2.0246913580246915,
      "grad_norm": 3.9872697883190553,
      "kl": 0.093994140625,
      "learning_rate": 3.3e-07,
      "loss": 0.0038,
      "reward": 1.607812523841858,
      "reward_std": 0.29563237726688385,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8109374940395355,
      "step": 737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.953125,
      "epoch": 2.027434842249657,
      "grad_norm": 2.421994079021264,
      "kl": 0.103759765625,
      "learning_rate": 3.290909090909091e-07,
      "loss": 0.0042,
      "reward": 1.834375023841858,
      "reward_std": 0.2595315985381603,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9125000238418579,
      "step": 738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.59375,
      "epoch": 2.0301783264746227,
      "grad_norm": 2.248903579189386,
      "kl": 0.10498046875,
      "learning_rate": 3.2818181818181816e-07,
      "loss": 0.0042,
      "reward": 1.9375,
      "reward_std": 0.1767766922712326,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.9687500298023224,
      "step": 739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.0,
      "epoch": 2.0329218106995883,
      "grad_norm": 6.04867003732456,
      "kl": 0.126953125,
      "learning_rate": 3.272727272727273e-07,
      "loss": 0.0051,
      "reward": 1.576562523841858,
      "reward_std": 0.3976454883813858,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7796874642372131,
      "step": 740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.390625,
      "epoch": 2.0356652949245544,
      "grad_norm": 4.843446654352449,
      "kl": 0.131103515625,
      "learning_rate": 3.2636363636363635e-07,
      "loss": 0.0052,
      "reward": 1.447656273841858,
      "reward_std": 0.3401695191860199,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7289062440395355,
      "step": 741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.25,
      "epoch": 2.03840877914952,
      "grad_norm": 2.3481660236622033,
      "kl": 0.102783203125,
      "learning_rate": 3.254545454545454e-07,
      "loss": 0.0041,
      "reward": 1.5093750357627869,
      "reward_std": 0.212132029235363,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7593750357627869,
      "step": 742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.53125,
      "epoch": 2.0411522633744856,
      "grad_norm": 2.9260190897672067,
      "kl": 0.117431640625,
      "learning_rate": 3.245454545454545e-07,
      "loss": 0.0047,
      "reward": 1.643750011920929,
      "reward_std": 0.31682799756526947,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8312499821186066,
      "step": 743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.609375,
      "epoch": 2.0438957475994513,
      "grad_norm": 5.932083100738138,
      "kl": 0.12353515625,
      "learning_rate": 3.236363636363636e-07,
      "loss": 0.0049,
      "reward": 1.5929688215255737,
      "reward_std": 0.5680141746997833,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.764843761920929,
      "step": 744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.109375,
      "epoch": 2.046639231824417,
      "grad_norm": 9.699257534549908,
      "kl": 0.1025390625,
      "learning_rate": 3.2272727272727274e-07,
      "loss": 0.0041,
      "reward": 1.474218726158142,
      "reward_std": 0.518870085477829,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7242187261581421,
      "step": 745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.53125,
      "epoch": 2.049382716049383,
      "grad_norm": 1.9887217834761526,
      "kl": 0.08251953125,
      "learning_rate": 3.218181818181818e-07,
      "loss": 0.0033,
      "reward": 1.5234374403953552,
      "reward_std": 0.34977400302886963,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7734375,
      "step": 746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.75,
      "epoch": 2.0521262002743486,
      "grad_norm": 5.05821835905173,
      "kl": 0.10693359375,
      "learning_rate": 3.209090909090909e-07,
      "loss": 0.0043,
      "reward": 1.550000011920929,
      "reward_std": 0.46106648445129395,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.784375011920929,
      "step": 747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.359375,
      "epoch": 2.054869684499314,
      "grad_norm": 4.383821715548243,
      "kl": 0.126953125,
      "learning_rate": 3.2e-07,
      "loss": 0.0051,
      "reward": 1.8593750596046448,
      "reward_std": 0.33364592492580414,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9218750298023224,
      "step": 748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.203125,
      "epoch": 2.05761316872428,
      "grad_norm": 3.593747564892723,
      "kl": 0.094482421875,
      "learning_rate": 3.190909090909091e-07,
      "loss": 0.0038,
      "reward": 1.0468749403953552,
      "reward_std": 0.4195588529109955,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.53125,
      "step": 749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.90625,
      "epoch": 2.0603566529492454,
      "grad_norm": 2.070362597813592,
      "kl": 0.106689453125,
      "learning_rate": 3.1818181818181815e-07,
      "loss": 0.0043,
      "reward": 1.732812523841858,
      "reward_std": 0.19391006976366043,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8734375238418579,
      "step": 750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.875,
      "epoch": 2.063100137174211,
      "grad_norm": 1.4578266059771372,
      "kl": 0.107666015625,
      "learning_rate": 3.1727272727272727e-07,
      "loss": 0.0043,
      "reward": 1.621874988079071,
      "reward_std": 0.21226724609732628,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.793749988079071,
      "step": 751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.78125,
      "epoch": 2.065843621399177,
      "grad_norm": 3.0790482141875115,
      "kl": 0.08740234375,
      "learning_rate": 3.1636363636363634e-07,
      "loss": 0.0035,
      "reward": 1.6578125357627869,
      "reward_std": 0.40604209899902344,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8296874761581421,
      "step": 752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.71875,
      "epoch": 2.0685871056241427,
      "grad_norm": 3.557197238977422,
      "kl": 0.129150390625,
      "learning_rate": 3.154545454545454e-07,
      "loss": 0.0052,
      "reward": 1.578125,
      "reward_std": 0.45973244309425354,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.78125,
      "step": 753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.53125,
      "epoch": 2.0713305898491083,
      "grad_norm": 15.43956507169567,
      "kl": 0.171142578125,
      "learning_rate": 3.1454545454545454e-07,
      "loss": 0.0069,
      "reward": 1.709375023841858,
      "reward_std": 0.3877876400947571,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8500000238418579,
      "step": 754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.6875,
      "epoch": 2.074074074074074,
      "grad_norm": 6.911957444749393,
      "kl": 0.10498046875,
      "learning_rate": 3.1363636363636366e-07,
      "loss": 0.0042,
      "reward": 1.774999976158142,
      "reward_std": 0.29579664394259453,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8843749761581421,
      "step": 755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.3125,
      "epoch": 2.0768175582990396,
      "grad_norm": 3.0039043899115296,
      "kl": 0.112060546875,
      "learning_rate": 3.1272727272727273e-07,
      "loss": 0.0045,
      "reward": 1.714062511920929,
      "reward_std": 0.2952996604144573,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.854687511920929,
      "step": 756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.65625,
      "epoch": 2.0795610425240056,
      "grad_norm": 5.021720651179575,
      "kl": 0.1083984375,
      "learning_rate": 3.118181818181818e-07,
      "loss": 0.0043,
      "reward": 1.7281250357627869,
      "reward_std": 0.511000894010067,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8687499761581421,
      "step": 757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.515625,
      "epoch": 2.0823045267489713,
      "grad_norm": 9.233165776712532,
      "kl": 0.097412109375,
      "learning_rate": 3.1090909090909087e-07,
      "loss": 0.0039,
      "reward": 1.6695312857627869,
      "reward_std": 0.2206084057688713,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8414062559604645,
      "step": 758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.796875,
      "epoch": 2.085048010973937,
      "grad_norm": 3.577220183692013,
      "kl": 0.09716796875,
      "learning_rate": 3.1e-07,
      "loss": 0.0039,
      "reward": 1.903124988079071,
      "reward_std": 0.26434044539928436,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9500000178813934,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.8125,
      "epoch": 2.0877914951989025,
      "grad_norm": 3.5183811498721904,
      "kl": 0.11962890625,
      "learning_rate": 3.0909090909090907e-07,
      "loss": 0.0048,
      "reward": 1.282031238079071,
      "reward_std": 0.40158161520957947,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6570312082767487,
      "step": 760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.78125,
      "epoch": 2.090534979423868,
      "grad_norm": 3.3151064924082942,
      "kl": 0.13818359375,
      "learning_rate": 3.0818181818181814e-07,
      "loss": 0.0055,
      "reward": 1.715624988079071,
      "reward_std": 0.298599936068058,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8406250178813934,
      "step": 761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.265625,
      "epoch": 2.093278463648834,
      "grad_norm": 9.575535716339024,
      "kl": 0.1171875,
      "learning_rate": 3.0727272727272726e-07,
      "loss": 0.0047,
      "reward": 1.682812511920929,
      "reward_std": 0.3616396635770798,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8390624821186066,
      "step": 762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.90625,
      "epoch": 2.0960219478738,
      "grad_norm": 5.799133083162924,
      "kl": 0.1103515625,
      "learning_rate": 3.0636363636363633e-07,
      "loss": 0.0044,
      "reward": 1.443750023841858,
      "reward_std": 0.555210679769516,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7249999940395355,
      "step": 763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.4375,
      "epoch": 2.0987654320987654,
      "grad_norm": 4.9470429795418,
      "kl": 0.095703125,
      "learning_rate": 3.0545454545454546e-07,
      "loss": 0.0038,
      "reward": 1.6390624642372131,
      "reward_std": 0.24187414348125458,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8109375238418579,
      "step": 764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.015625,
      "epoch": 2.101508916323731,
      "grad_norm": 2.9719822175731756,
      "kl": 0.13818359375,
      "learning_rate": 3.0454545454545453e-07,
      "loss": 0.0055,
      "reward": 1.8593749403953552,
      "reward_std": 0.18882093112915754,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.921875,
      "step": 765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 62.390625,
      "epoch": 2.1042524005486967,
      "grad_norm": 2.8071825422509127,
      "kl": 0.112548828125,
      "learning_rate": 3.0363636363636365e-07,
      "loss": 0.0045,
      "reward": 1.434374988079071,
      "reward_std": 0.23727848380804062,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7312499582767487,
      "step": 766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.140625,
      "epoch": 2.1069958847736627,
      "grad_norm": 2.783798353824772,
      "kl": 0.10107421875,
      "learning_rate": 3.027272727272727e-07,
      "loss": 0.004,
      "reward": 1.1953125,
      "reward_std": 0.17222431302070618,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6015624850988388,
      "step": 767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.125,
      "epoch": 2.1097393689986284,
      "grad_norm": 3.7969867351903215,
      "kl": 0.1171875,
      "learning_rate": 3.018181818181818e-07,
      "loss": 0.0047,
      "reward": 1.505468726158142,
      "reward_std": 0.5320801958441734,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7554687261581421,
      "step": 768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.671875,
      "epoch": 2.112482853223594,
      "grad_norm": 2.4280428057760908,
      "kl": 0.106689453125,
      "learning_rate": 3.009090909090909e-07,
      "loss": 0.0043,
      "reward": 1.03125,
      "reward_std": 0.22097086161375046,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5312499850988388,
      "step": 769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.109375,
      "epoch": 2.1152263374485596,
      "grad_norm": 4.325617079844216,
      "kl": 0.104248046875,
      "learning_rate": 3e-07,
      "loss": 0.0042,
      "reward": 1.3859374523162842,
      "reward_std": 0.36535846441984177,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6984374523162842,
      "step": 770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.921875,
      "epoch": 2.117969821673525,
      "grad_norm": 2.602909903575354,
      "kl": 0.0966796875,
      "learning_rate": 2.9909090909090906e-07,
      "loss": 0.0039,
      "reward": 1.7406249046325684,
      "reward_std": 0.34644468128681183,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8656249940395355,
      "step": 771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.765625,
      "epoch": 2.1207133058984913,
      "grad_norm": 3.290830115950048,
      "kl": 0.142578125,
      "learning_rate": 2.9818181818181813e-07,
      "loss": 0.0057,
      "reward": 1.475000023841858,
      "reward_std": 0.3219795897603035,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7406249940395355,
      "step": 772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.625,
      "epoch": 2.123456790123457,
      "grad_norm": 10.577530102589195,
      "kl": 0.16455078125,
      "learning_rate": 2.972727272727273e-07,
      "loss": 0.0066,
      "reward": 1.470312476158142,
      "reward_std": 0.5060344040393829,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7359375059604645,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.015625,
      "epoch": 2.1262002743484225,
      "grad_norm": 6.183379887663399,
      "kl": 0.140380859375,
      "learning_rate": 2.963636363636364e-07,
      "loss": 0.0056,
      "reward": 1.6015625,
      "reward_std": 0.7116466164588928,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8046875298023224,
      "step": 774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.359375,
      "epoch": 2.128943758573388,
      "grad_norm": 3.3452115484957266,
      "kl": 0.110595703125,
      "learning_rate": 2.9545454545454545e-07,
      "loss": 0.0044,
      "reward": 1.5499999523162842,
      "reward_std": 0.3921141056343913,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7843749523162842,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.96875,
      "epoch": 2.1316872427983538,
      "grad_norm": 3.8120103516787482,
      "kl": 0.10693359375,
      "learning_rate": 2.945454545454545e-07,
      "loss": 0.0043,
      "reward": 1.4554687142372131,
      "reward_std": 0.6374655365943909,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7367187440395355,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.046875,
      "epoch": 2.13443072702332,
      "grad_norm": 3.7087021187291227,
      "kl": 0.126953125,
      "learning_rate": 2.9363636363636364e-07,
      "loss": 0.0051,
      "reward": 1.8093749284744263,
      "reward_std": 0.40407513082027435,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.903124988079071,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.328125,
      "epoch": 2.1371742112482854,
      "grad_norm": 3.680864303291531,
      "kl": 0.118408203125,
      "learning_rate": 2.927272727272727e-07,
      "loss": 0.0047,
      "reward": 1.662500023841858,
      "reward_std": 0.3835296258330345,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8343750238418579,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.171875,
      "epoch": 2.139917695473251,
      "grad_norm": 5.814044481438807,
      "kl": 0.107666015625,
      "learning_rate": 2.918181818181818e-07,
      "loss": 0.0043,
      "reward": 1.0546875,
      "reward_std": 0.33707825164310634,
      "rewards/choice_reward": 0.515625,
      "rewards/roam_reward": 0.5390624403953552,
      "step": 779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.046875,
      "epoch": 2.1426611796982167,
      "grad_norm": 11.543603320171377,
      "kl": 0.17236328125,
      "learning_rate": 2.909090909090909e-07,
      "loss": 0.0069,
      "reward": 1.418749988079071,
      "reward_std": 0.4447284936904907,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.715624988079071,
      "step": 780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.796875,
      "epoch": 2.1454046639231823,
      "grad_norm": 4.414795879502755,
      "kl": 0.104248046875,
      "learning_rate": 2.9e-07,
      "loss": 0.0042,
      "reward": 1.6125000715255737,
      "reward_std": 0.4747316986322403,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8156249821186066,
      "step": 781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.484375,
      "epoch": 2.148148148148148,
      "grad_norm": 3.508188786631475,
      "kl": 0.1298828125,
      "learning_rate": 2.8909090909090905e-07,
      "loss": 0.0052,
      "reward": 1.658593773841858,
      "reward_std": 0.2683626413345337,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8304687738418579,
      "step": 782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.0625,
      "epoch": 2.150891632373114,
      "grad_norm": 4.240108432429567,
      "kl": 0.14697265625,
      "learning_rate": 2.8818181818181817e-07,
      "loss": 0.0059,
      "reward": 1.4374999403953552,
      "reward_std": 0.6624129414558411,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7187499701976776,
      "step": 783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.90625,
      "epoch": 2.1536351165980796,
      "grad_norm": 5.274311339934734,
      "kl": 0.1181640625,
      "learning_rate": 2.872727272727273e-07,
      "loss": 0.0047,
      "reward": 1.728124976158142,
      "reward_std": 0.09722718596458435,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8687499761581421,
      "step": 784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.640625,
      "epoch": 2.156378600823045,
      "grad_norm": 4.175305160972081,
      "kl": 0.13818359375,
      "learning_rate": 2.8636363636363637e-07,
      "loss": 0.0055,
      "reward": 1.618749976158142,
      "reward_std": 0.4070549011230469,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7906250059604645,
      "step": 785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.609375,
      "epoch": 2.159122085048011,
      "grad_norm": 3.509394846684709,
      "kl": 0.10498046875,
      "learning_rate": 2.8545454545454544e-07,
      "loss": 0.0042,
      "reward": 1.456250011920929,
      "reward_std": 0.3542475253343582,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.737500011920929,
      "step": 786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.609375,
      "epoch": 2.161865569272977,
      "grad_norm": 3.8637174394342217,
      "kl": 0.112548828125,
      "learning_rate": 2.845454545454545e-07,
      "loss": 0.0045,
      "reward": 1.7562499642372131,
      "reward_std": 0.36303091794252396,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8812499642372131,
      "step": 787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.625,
      "epoch": 2.1646090534979425,
      "grad_norm": 3.0933947112166003,
      "kl": 0.12158203125,
      "learning_rate": 2.8363636363636363e-07,
      "loss": 0.0049,
      "reward": 1.4867187142372131,
      "reward_std": 0.5142303705215454,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7523437142372131,
      "step": 788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.75,
      "epoch": 2.167352537722908,
      "grad_norm": 4.061168976745869,
      "kl": 0.09228515625,
      "learning_rate": 2.827272727272727e-07,
      "loss": 0.0037,
      "reward": 1.6757811903953552,
      "reward_std": 0.46660368144512177,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8320312201976776,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 55.5625,
      "epoch": 2.1700960219478738,
      "grad_norm": 1.2070475469082251,
      "kl": 0.11083984375,
      "learning_rate": 2.8181818181818177e-07,
      "loss": 0.0044,
      "reward": 1.732812523841858,
      "reward_std": 0.08396892994642258,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8734374642372131,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.734375,
      "epoch": 2.1728395061728394,
      "grad_norm": 4.913584657192109,
      "kl": 0.0986328125,
      "learning_rate": 2.809090909090909e-07,
      "loss": 0.0039,
      "reward": 1.5148437023162842,
      "reward_std": 0.358761690557003,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7648437321186066,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.40625,
      "epoch": 2.175582990397805,
      "grad_norm": 3.2707132647043875,
      "kl": 0.1279296875,
      "learning_rate": 2.8e-07,
      "loss": 0.0051,
      "reward": 1.603906273841858,
      "reward_std": 0.5658104568719864,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7914062440395355,
      "step": 792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.609375,
      "epoch": 2.178326474622771,
      "grad_norm": 4.3355832161580246,
      "kl": 0.126953125,
      "learning_rate": 2.790909090909091e-07,
      "loss": 0.0051,
      "reward": 1.7234375476837158,
      "reward_std": 0.47183680534362793,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.864062488079071,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 66.859375,
      "epoch": 2.1810699588477367,
      "grad_norm": 4.129358376134468,
      "kl": 0.108642578125,
      "learning_rate": 2.7818181818181816e-07,
      "loss": 0.0043,
      "reward": 1.830468773841858,
      "reward_std": 0.4118579924106598,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9085937440395355,
      "step": 794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.859375,
      "epoch": 2.1838134430727023,
      "grad_norm": 4.373027044183056,
      "kl": 0.11376953125,
      "learning_rate": 2.772727272727273e-07,
      "loss": 0.0046,
      "reward": 1.7937499284744263,
      "reward_std": 0.29688415490090847,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.871874988079071,
      "step": 795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.015625,
      "epoch": 2.186556927297668,
      "grad_norm": 2.5642796130070566,
      "kl": 0.0908203125,
      "learning_rate": 2.7636363636363635e-07,
      "loss": 0.0036,
      "reward": 1.6804687976837158,
      "reward_std": 0.2819945365190506,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.836718738079071,
      "step": 796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.484375,
      "epoch": 2.1893004115226335,
      "grad_norm": 5.357012535761609,
      "kl": 0.259521484375,
      "learning_rate": 2.754545454545454e-07,
      "loss": 0.0104,
      "reward": 1.837499976158142,
      "reward_std": 0.2781676575541496,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.8843750059604645,
      "step": 797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 69.671875,
      "epoch": 2.1920438957475996,
      "grad_norm": 4.22130164678084,
      "kl": 0.111083984375,
      "learning_rate": 2.7454545454545455e-07,
      "loss": 0.0044,
      "reward": 1.357812523841858,
      "reward_std": 0.5226652920246124,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859375238418579,
      "step": 798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.59375,
      "epoch": 2.1947873799725652,
      "grad_norm": 2.6780966138783278,
      "kl": 0.0872802734375,
      "learning_rate": 2.736363636363636e-07,
      "loss": 0.0035,
      "reward": 1.5828125476837158,
      "reward_std": 0.3197706267237663,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.801562488079071,
      "step": 799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.703125,
      "epoch": 2.197530864197531,
      "grad_norm": 3.2315439352160684,
      "kl": 0.09912109375,
      "learning_rate": 2.727272727272727e-07,
      "loss": 0.004,
      "reward": 1.696874976158142,
      "reward_std": 0.37345467507839203,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8531249761581421,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.609375,
      "epoch": 2.2002743484224965,
      "grad_norm": 6.333539078309606,
      "kl": 0.098388671875,
      "learning_rate": 2.7181818181818176e-07,
      "loss": 0.0039,
      "reward": 1.623437523841858,
      "reward_std": 0.5012295097112656,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8109375238418579,
      "step": 801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.4375,
      "epoch": 2.203017832647462,
      "grad_norm": 2.232547080842151,
      "kl": 0.095947265625,
      "learning_rate": 2.7090909090909094e-07,
      "loss": 0.0038,
      "reward": 1.8656249642372131,
      "reward_std": 0.21204502135515213,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9281249940395355,
      "step": 802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.4375,
      "epoch": 2.205761316872428,
      "grad_norm": 3.8566382494589813,
      "kl": 0.10107421875,
      "learning_rate": 2.7e-07,
      "loss": 0.004,
      "reward": 1.2835937142372131,
      "reward_std": 0.2889275439083576,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6585937142372131,
      "step": 803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.171875,
      "epoch": 2.2085048010973938,
      "grad_norm": 2.921891948516463,
      "kl": 0.114013671875,
      "learning_rate": 2.690909090909091e-07,
      "loss": 0.0046,
      "reward": 1.580468773841858,
      "reward_std": 0.3070296049118042,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7835937440395355,
      "step": 804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.703125,
      "epoch": 2.2112482853223594,
      "grad_norm": 6.092366474924012,
      "kl": 0.121826171875,
      "learning_rate": 2.6818181818181815e-07,
      "loss": 0.0049,
      "reward": 1.8203125,
      "reward_std": 0.44890232384204865,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.8828125,
      "step": 805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.90625,
      "epoch": 2.213991769547325,
      "grad_norm": 0.8292229427625117,
      "kl": 0.0849609375,
      "learning_rate": 2.672727272727273e-07,
      "loss": 0.0034,
      "reward": 1.731249988079071,
      "reward_std": 0.0578637570142746,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.856249988079071,
      "step": 806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.203125,
      "epoch": 2.2167352537722906,
      "grad_norm": 4.975513240528351,
      "kl": 0.1103515625,
      "learning_rate": 2.6636363636363634e-07,
      "loss": 0.0044,
      "reward": 1.4742187857627869,
      "reward_std": 0.46045906841754913,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7398437559604645,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.015625,
      "epoch": 2.2194787379972567,
      "grad_norm": 4.22241272304683,
      "kl": 0.1337890625,
      "learning_rate": 2.654545454545454e-07,
      "loss": 0.0054,
      "reward": 1.574999988079071,
      "reward_std": 0.35842348635196686,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.778124988079071,
      "step": 808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.4375,
      "epoch": 2.2222222222222223,
      "grad_norm": 4.555498756335306,
      "kl": 0.10791015625,
      "learning_rate": 2.6454545454545454e-07,
      "loss": 0.0043,
      "reward": 1.1343750357627869,
      "reward_std": 0.520625114440918,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5874999761581421,
      "step": 809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.5625,
      "epoch": 2.224965706447188,
      "grad_norm": 3.437950147489919,
      "kl": 0.09033203125,
      "learning_rate": 2.636363636363636e-07,
      "loss": 0.0036,
      "reward": 1.4007813334465027,
      "reward_std": 0.5314416140317917,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.6820312440395355,
      "step": 810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.765625,
      "epoch": 2.2277091906721536,
      "grad_norm": 3.8079860024144585,
      "kl": 0.10302734375,
      "learning_rate": 2.6272727272727273e-07,
      "loss": 0.0041,
      "reward": 1.717187523841858,
      "reward_std": 0.1220581904053688,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8578125238418579,
      "step": 811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.21875,
      "epoch": 2.230452674897119,
      "grad_norm": 3.0291385388945766,
      "kl": 0.123779296875,
      "learning_rate": 2.618181818181818e-07,
      "loss": 0.005,
      "reward": 1.734375,
      "reward_std": 0.3770694434642792,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.84375,
      "step": 812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.296875,
      "epoch": 2.2331961591220852,
      "grad_norm": 5.123582714680745,
      "kl": 0.104736328125,
      "learning_rate": 2.6090909090909093e-07,
      "loss": 0.0042,
      "reward": 1.821874976158142,
      "reward_std": 0.294886939227581,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9156250059604645,
      "step": 813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.96875,
      "epoch": 2.235939643347051,
      "grad_norm": 2.1805429532884464,
      "kl": 0.099609375,
      "learning_rate": 2.6e-07,
      "loss": 0.004,
      "reward": 1.7578125,
      "reward_std": 0.2911372510716319,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8828125,
      "step": 814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 71.359375,
      "epoch": 2.2386831275720165,
      "grad_norm": 14.093464621240487,
      "kl": 0.13134765625,
      "learning_rate": 2.5909090909090907e-07,
      "loss": 0.0052,
      "reward": 1.603124976158142,
      "reward_std": 0.37940096855163574,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8062500059604645,
      "step": 815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.96875,
      "epoch": 2.241426611796982,
      "grad_norm": 11.832522126618683,
      "kl": 0.10791015625,
      "learning_rate": 2.581818181818182e-07,
      "loss": 0.0043,
      "reward": 1.7578125596046448,
      "reward_std": 0.5320637822151184,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8671875,
      "step": 816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.65625,
      "epoch": 2.2441700960219477,
      "grad_norm": 6.779992242783354,
      "kl": 0.1259765625,
      "learning_rate": 2.5727272727272726e-07,
      "loss": 0.005,
      "reward": 1.647656261920929,
      "reward_std": 0.5611380934715271,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8195312321186066,
      "step": 817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.78125,
      "epoch": 2.246913580246914,
      "grad_norm": 5.229875349558636,
      "kl": 0.114990234375,
      "learning_rate": 2.5636363636363633e-07,
      "loss": 0.0046,
      "reward": 1.3835937976837158,
      "reward_std": 0.5313563942909241,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.696093738079071,
      "step": 818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.046875,
      "epoch": 2.2496570644718794,
      "grad_norm": 5.072122967999993,
      "kl": 0.14697265625,
      "learning_rate": 2.554545454545454e-07,
      "loss": 0.0059,
      "reward": 1.232812523841858,
      "reward_std": 0.5871487855911255,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6234374940395355,
      "step": 819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.703125,
      "epoch": 2.252400548696845,
      "grad_norm": 2.985874495956574,
      "kl": 0.13134765625,
      "learning_rate": 2.5454545454545453e-07,
      "loss": 0.0053,
      "reward": 1.8359375,
      "reward_std": 0.30598607659339905,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.9140624701976776,
      "step": 820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.703125,
      "epoch": 2.2551440329218106,
      "grad_norm": 3.556409499813488,
      "kl": 0.12451171875,
      "learning_rate": 2.5363636363636365e-07,
      "loss": 0.005,
      "reward": 1.55078125,
      "reward_std": 0.4785647839307785,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.78515625,
      "step": 821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.765625,
      "epoch": 2.2578875171467763,
      "grad_norm": 2.6368990361460876,
      "kl": 0.104736328125,
      "learning_rate": 2.527272727272727e-07,
      "loss": 0.0042,
      "reward": 1.315625011920929,
      "reward_std": 0.34501613676548004,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6749999821186066,
      "step": 822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.890625,
      "epoch": 2.260631001371742,
      "grad_norm": 4.607358551482156,
      "kl": 0.117431640625,
      "learning_rate": 2.518181818181818e-07,
      "loss": 0.0047,
      "reward": 1.374218761920929,
      "reward_std": 0.3684770464897156,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.7023437023162842,
      "step": 823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 72.265625,
      "epoch": 2.263374485596708,
      "grad_norm": 4.031602805941072,
      "kl": 0.098388671875,
      "learning_rate": 2.509090909090909e-07,
      "loss": 0.0039,
      "reward": 1.1226562857627869,
      "reward_std": 0.406386137008667,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5757812559604645,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.953125,
      "epoch": 2.2661179698216736,
      "grad_norm": 3.5991674682192696,
      "kl": 0.126953125,
      "learning_rate": 2.5e-07,
      "loss": 0.0051,
      "reward": 1.3109374642372131,
      "reward_std": 0.2621586322784424,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6703124344348907,
      "step": 825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.46875,
      "epoch": 2.268861454046639,
      "grad_norm": 4.67083394246974,
      "kl": 0.110107421875,
      "learning_rate": 2.4909090909090906e-07,
      "loss": 0.0044,
      "reward": 1.0859375,
      "reward_std": 0.6535941660404205,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5546875,
      "step": 826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.28125,
      "epoch": 2.271604938271605,
      "grad_norm": 1.9804141623697706,
      "kl": 0.09619140625,
      "learning_rate": 2.481818181818182e-07,
      "loss": 0.0039,
      "reward": 1.7890625,
      "reward_std": 0.09280776232481003,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8984375,
      "step": 827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.359375,
      "epoch": 2.274348422496571,
      "grad_norm": 9.729038956473259,
      "kl": 0.09228515625,
      "learning_rate": 2.4727272727272725e-07,
      "loss": 0.0037,
      "reward": 1.692187488079071,
      "reward_std": 0.3357216417789459,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8328125178813934,
      "step": 828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.75,
      "epoch": 2.2770919067215365,
      "grad_norm": 3.8855375569193304,
      "kl": 0.115966796875,
      "learning_rate": 2.463636363636364e-07,
      "loss": 0.0046,
      "reward": 1.4171874523162842,
      "reward_std": 0.7684605121612549,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7140624821186066,
      "step": 829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.875,
      "epoch": 2.279835390946502,
      "grad_norm": 3.15608198733615,
      "kl": 0.123046875,
      "learning_rate": 2.4545454545454545e-07,
      "loss": 0.0049,
      "reward": 1.608593761920929,
      "reward_std": 0.46275579929351807,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.811718761920929,
      "step": 830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.125,
      "epoch": 2.2825788751714677,
      "grad_norm": 3.225464493364868,
      "kl": 0.09423828125,
      "learning_rate": 2.445454545454545e-07,
      "loss": 0.0038,
      "reward": 1.564062476158142,
      "reward_std": 0.42860206216573715,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7828125059604645,
      "step": 831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.8125,
      "epoch": 2.2853223593964334,
      "grad_norm": 7.063770706719847,
      "kl": 0.1162109375,
      "learning_rate": 2.436363636363636e-07,
      "loss": 0.0046,
      "reward": 1.724218726158142,
      "reward_std": 0.274951308965683,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8492187559604645,
      "step": 832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.625,
      "epoch": 2.288065843621399,
      "grad_norm": 2.3658771387066144,
      "kl": 0.14892578125,
      "learning_rate": 2.427272727272727e-07,
      "loss": 0.006,
      "reward": 1.581250011920929,
      "reward_std": 0.3219865560531616,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.800000011920929,
      "step": 833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.609375,
      "epoch": 2.290809327846365,
      "grad_norm": 39.96062417184321,
      "kl": 0.108154296875,
      "learning_rate": 2.418181818181818e-07,
      "loss": 0.0043,
      "reward": 1.770312488079071,
      "reward_std": 0.48108595609664917,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.864062488079071,
      "step": 834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.203125,
      "epoch": 2.2935528120713307,
      "grad_norm": 1.548739464441859,
      "kl": 0.102783203125,
      "learning_rate": 2.409090909090909e-07,
      "loss": 0.0041,
      "reward": 1.567187488079071,
      "reward_std": 0.21341075748205185,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7859375178813934,
      "step": 835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.609375,
      "epoch": 2.2962962962962963,
      "grad_norm": 2.6372360900746297,
      "kl": 0.1005859375,
      "learning_rate": 2.4e-07,
      "loss": 0.004,
      "reward": 1.864843726158142,
      "reward_std": 0.2521742656826973,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9273437559604645,
      "step": 836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.40625,
      "epoch": 2.299039780521262,
      "grad_norm": 6.055698129525655,
      "kl": 0.12890625,
      "learning_rate": 2.390909090909091e-07,
      "loss": 0.0052,
      "reward": 1.535937488079071,
      "reward_std": 0.748941570520401,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7703125178813934,
      "step": 837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.625,
      "epoch": 2.3017832647462275,
      "grad_norm": 5.718085633049784,
      "kl": 0.102294921875,
      "learning_rate": 2.3818181818181817e-07,
      "loss": 0.0041,
      "reward": 1.934374988079071,
      "reward_std": 0.18561553210020065,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.9656250178813934,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.609375,
      "epoch": 2.3045267489711936,
      "grad_norm": 3.4220848985214447,
      "kl": 0.1708984375,
      "learning_rate": 2.3727272727272727e-07,
      "loss": 0.0068,
      "reward": 1.5148438215255737,
      "reward_std": 0.19026567973196507,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.764843761920929,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.859375,
      "epoch": 2.307270233196159,
      "grad_norm": 3.883757282531534,
      "kl": 0.124267578125,
      "learning_rate": 2.3636363636363634e-07,
      "loss": 0.005,
      "reward": 1.614062488079071,
      "reward_std": 0.28888290375471115,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.7859374582767487,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.0625,
      "epoch": 2.310013717421125,
      "grad_norm": 2.571179323855547,
      "kl": 0.083251953125,
      "learning_rate": 2.3545454545454544e-07,
      "loss": 0.0033,
      "reward": 1.641406238079071,
      "reward_std": 0.3707147389650345,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.813281238079071,
      "step": 841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.296875,
      "epoch": 2.3127572016460904,
      "grad_norm": 7.098184106639789,
      "kl": 0.117431640625,
      "learning_rate": 2.3454545454545454e-07,
      "loss": 0.0047,
      "reward": 1.6953125,
      "reward_std": 0.3832200914621353,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8515624701976776,
      "step": 842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 75.140625,
      "epoch": 2.315500685871056,
      "grad_norm": 4.19099378367587,
      "kl": 0.11181640625,
      "learning_rate": 2.3363636363636363e-07,
      "loss": 0.0045,
      "reward": 1.786718726158142,
      "reward_std": 0.33557650446891785,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8960937559604645,
      "step": 843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.78125,
      "epoch": 2.318244170096022,
      "grad_norm": 4.962168354226179,
      "kl": 0.117431640625,
      "learning_rate": 2.3272727272727273e-07,
      "loss": 0.0047,
      "reward": 1.205468773841858,
      "reward_std": 0.5368544310331345,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6117187738418579,
      "step": 844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.203125,
      "epoch": 2.3209876543209877,
      "grad_norm": 3.709767154383809,
      "kl": 0.104736328125,
      "learning_rate": 2.318181818181818e-07,
      "loss": 0.0042,
      "reward": 1.483593761920929,
      "reward_std": 0.32407404482364655,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7492187321186066,
      "step": 845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.375,
      "epoch": 2.3237311385459534,
      "grad_norm": 3.826413640525221,
      "kl": 0.08447265625,
      "learning_rate": 2.309090909090909e-07,
      "loss": 0.0034,
      "reward": 1.31640625,
      "reward_std": 0.5299557000398636,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.67578125,
      "step": 846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.578125,
      "epoch": 2.326474622770919,
      "grad_norm": 2.651734680987155,
      "kl": 0.10791015625,
      "learning_rate": 2.3e-07,
      "loss": 0.0043,
      "reward": 1.4578125476837158,
      "reward_std": 0.3175119236111641,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7390625178813934,
      "step": 847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.53125,
      "epoch": 2.3292181069958846,
      "grad_norm": 3.1907652990124857,
      "kl": 0.112060546875,
      "learning_rate": 2.290909090909091e-07,
      "loss": 0.0045,
      "reward": 1.5390625,
      "reward_std": 0.5349350571632385,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7734374701976776,
      "step": 848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.515625,
      "epoch": 2.3319615912208507,
      "grad_norm": 4.01003089380689,
      "kl": 0.111083984375,
      "learning_rate": 2.2818181818181816e-07,
      "loss": 0.0044,
      "reward": 1.785937488079071,
      "reward_std": 0.39371173083782196,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.8796875178813934,
      "step": 849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.15625,
      "epoch": 2.3347050754458163,
      "grad_norm": 51.61948966412654,
      "kl": 0.10498046875,
      "learning_rate": 2.2727272727272726e-07,
      "loss": 0.0042,
      "reward": 1.471875011920929,
      "reward_std": 0.30076853930950165,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7374999821186066,
      "step": 850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.015625,
      "epoch": 2.337448559670782,
      "grad_norm": 4.390409835085189,
      "kl": 0.090087890625,
      "learning_rate": 2.2636363636363633e-07,
      "loss": 0.0036,
      "reward": 1.4320312142372131,
      "reward_std": 0.22010089457035065,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7289062142372131,
      "step": 851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.578125,
      "epoch": 2.3401920438957475,
      "grad_norm": 3.194735158147514,
      "kl": 0.096435546875,
      "learning_rate": 2.2545454545454545e-07,
      "loss": 0.0039,
      "reward": 1.623437523841858,
      "reward_std": 0.325691282749176,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8109374940395355,
      "step": 852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.09375,
      "epoch": 2.342935528120713,
      "grad_norm": 3.239870186539037,
      "kl": 0.117431640625,
      "learning_rate": 2.2454545454545453e-07,
      "loss": 0.0047,
      "reward": 1.38671875,
      "reward_std": 0.4322517514228821,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.6992187649011612,
      "step": 853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.25,
      "epoch": 2.3456790123456788,
      "grad_norm": 4.085367729309037,
      "kl": 0.09765625,
      "learning_rate": 2.2363636363636362e-07,
      "loss": 0.0039,
      "reward": 1.532031238079071,
      "reward_std": 0.7208892107009888,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.766406238079071,
      "step": 854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.1875,
      "epoch": 2.348422496570645,
      "grad_norm": 20.88507515916802,
      "kl": 0.115966796875,
      "learning_rate": 2.2272727272727272e-07,
      "loss": 0.0046,
      "reward": 1.4679687023162842,
      "reward_std": 0.5433869957923889,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7335937023162842,
      "step": 855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.390625,
      "epoch": 2.3511659807956105,
      "grad_norm": 5.171802093144948,
      "kl": 0.131591796875,
      "learning_rate": 2.2181818181818182e-07,
      "loss": 0.0053,
      "reward": 1.754687488079071,
      "reward_std": 0.3516154810786247,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.879687488079071,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.765625,
      "epoch": 2.353909465020576,
      "grad_norm": 2.6822859489888233,
      "kl": 0.119384765625,
      "learning_rate": 2.2090909090909091e-07,
      "loss": 0.0048,
      "reward": 1.792187511920929,
      "reward_std": 0.33487100899219513,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.885937511920929,
      "step": 857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.203125,
      "epoch": 2.3566529492455417,
      "grad_norm": 3.852181269092699,
      "kl": 0.138427734375,
      "learning_rate": 2.1999999999999998e-07,
      "loss": 0.0055,
      "reward": 1.567187488079071,
      "reward_std": 0.43235016614198685,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.785937488079071,
      "step": 858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.203125,
      "epoch": 2.3593964334705078,
      "grad_norm": 9.283208203193293,
      "kl": 0.11376953125,
      "learning_rate": 2.1909090909090908e-07,
      "loss": 0.0046,
      "reward": 1.4859375357627869,
      "reward_std": 0.25532449781894684,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7359375059604645,
      "step": 859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.546875,
      "epoch": 2.3621399176954734,
      "grad_norm": 4.301864973090632,
      "kl": 0.0986328125,
      "learning_rate": 2.1818181818181815e-07,
      "loss": 0.0039,
      "reward": 1.053125023841858,
      "reward_std": 0.3161915987730026,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5218749940395355,
      "step": 860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.921875,
      "epoch": 2.364883401920439,
      "grad_norm": 3.942303526415412,
      "kl": 0.115234375,
      "learning_rate": 2.1727272727272728e-07,
      "loss": 0.0046,
      "reward": 1.7210937142372131,
      "reward_std": 0.3669208139181137,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8304687738418579,
      "step": 861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.53125,
      "epoch": 2.3676268861454046,
      "grad_norm": 4.143736465551136,
      "kl": 0.099609375,
      "learning_rate": 2.1636363636363635e-07,
      "loss": 0.004,
      "reward": 1.525781273841858,
      "reward_std": 0.15028471499681473,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7601562440395355,
      "step": 862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.5,
      "epoch": 2.3703703703703702,
      "grad_norm": 2.643244037430125,
      "kl": 0.10498046875,
      "learning_rate": 2.1545454545454544e-07,
      "loss": 0.0042,
      "reward": 1.635156273841858,
      "reward_std": 0.33017241954803467,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8226562738418579,
      "step": 863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.484375,
      "epoch": 2.373113854595336,
      "grad_norm": 3.12918081111299,
      "kl": 0.1162109375,
      "learning_rate": 2.1454545454545454e-07,
      "loss": 0.0046,
      "reward": 1.5859375,
      "reward_std": 0.27471774071455,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7890625,
      "step": 864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.328125,
      "epoch": 2.375857338820302,
      "grad_norm": 4.868710249582456,
      "kl": 0.109619140625,
      "learning_rate": 2.136363636363636e-07,
      "loss": 0.0044,
      "reward": 1.428125023841858,
      "reward_std": 0.5239690691232681,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7249999642372131,
      "step": 865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.5625,
      "epoch": 2.3786008230452675,
      "grad_norm": 6.729707203115704,
      "kl": 0.1025390625,
      "learning_rate": 2.1272727272727274e-07,
      "loss": 0.0041,
      "reward": 1.515625,
      "reward_std": 0.5596156716346741,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.765625,
      "step": 866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.953125,
      "epoch": 2.381344307270233,
      "grad_norm": 2.627118608764886,
      "kl": 0.0927734375,
      "learning_rate": 2.118181818181818e-07,
      "loss": 0.0037,
      "reward": 1.766406238079071,
      "reward_std": 0.14580436330288649,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.875781238079071,
      "step": 867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.21875,
      "epoch": 2.384087791495199,
      "grad_norm": 40.22424412671131,
      "kl": 0.1025390625,
      "learning_rate": 2.109090909090909e-07,
      "loss": 0.0041,
      "reward": 1.3320311903953552,
      "reward_std": 0.2978522703051567,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6757812201976776,
      "step": 868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.40625,
      "epoch": 2.386831275720165,
      "grad_norm": 2.801683774665442,
      "kl": 0.112060546875,
      "learning_rate": 2.0999999999999997e-07,
      "loss": 0.0045,
      "reward": 1.665624976158142,
      "reward_std": 0.37274664640426636,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8375000059604645,
      "step": 869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.046875,
      "epoch": 2.3895747599451305,
      "grad_norm": 7.799828848279018,
      "kl": 0.09423828125,
      "learning_rate": 2.0909090909090907e-07,
      "loss": 0.0038,
      "reward": 0.8023437634110451,
      "reward_std": 0.18869349360466003,
      "rewards/choice_reward": 0.375,
      "rewards/roam_reward": 0.4273437634110451,
      "step": 870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.0625,
      "epoch": 2.392318244170096,
      "grad_norm": 3.5958536385128066,
      "kl": 0.096923828125,
      "learning_rate": 2.0818181818181817e-07,
      "loss": 0.0039,
      "reward": 1.5695312023162842,
      "reward_std": 0.5512479394674301,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.7570312321186066,
      "step": 871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.234375,
      "epoch": 2.3950617283950617,
      "grad_norm": 5.192872918934563,
      "kl": 0.161376953125,
      "learning_rate": 2.0727272727272727e-07,
      "loss": 0.0065,
      "reward": 1.4531249403953552,
      "reward_std": 0.4561312794685364,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7343749701976776,
      "step": 872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.3125,
      "epoch": 2.3978052126200273,
      "grad_norm": 5.315526210318176,
      "kl": 0.13720703125,
      "learning_rate": 2.0636363636363636e-07,
      "loss": 0.0055,
      "reward": 1.651562511920929,
      "reward_std": 0.42224298417568207,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.823437511920929,
      "step": 873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.015625,
      "epoch": 2.400548696844993,
      "grad_norm": 2.0419602089837015,
      "kl": 0.108154296875,
      "learning_rate": 2.0545454545454543e-07,
      "loss": 0.0043,
      "reward": 1.901562511920929,
      "reward_std": 0.1383042959496379,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9484374821186066,
      "step": 874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.546875,
      "epoch": 2.403292181069959,
      "grad_norm": 10.813583170383565,
      "kl": 0.111083984375,
      "learning_rate": 2.0454545454545456e-07,
      "loss": 0.0044,
      "reward": 1.5453124642372131,
      "reward_std": 0.4527755454182625,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7796874940395355,
      "step": 875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 74.328125,
      "epoch": 2.4060356652949246,
      "grad_norm": 4.038706968687633,
      "kl": 0.10400390625,
      "learning_rate": 2.0363636363636363e-07,
      "loss": 0.0042,
      "reward": 1.754687488079071,
      "reward_std": 0.02209708746522665,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.879687488079071,
      "step": 876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.375,
      "epoch": 2.4087791495198903,
      "grad_norm": 1.3484577069751817,
      "kl": 0.071044921875,
      "learning_rate": 2.0272727272727273e-07,
      "loss": 0.0028,
      "reward": 1.7023437023162842,
      "reward_std": 0.20723089575767517,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.858593761920929,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.71875,
      "epoch": 2.411522633744856,
      "grad_norm": 12.510104389183923,
      "kl": 0.108154296875,
      "learning_rate": 2.018181818181818e-07,
      "loss": 0.0043,
      "reward": 1.364843726158142,
      "reward_std": 0.2217685431241989,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6929687261581421,
      "step": 878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.171875,
      "epoch": 2.4142661179698215,
      "grad_norm": 4.296791480235377,
      "kl": 0.11865234375,
      "learning_rate": 2.009090909090909e-07,
      "loss": 0.0047,
      "reward": 1.7554687857627869,
      "reward_std": 0.28783681988716125,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8648437559604645,
      "step": 879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 94.765625,
      "epoch": 2.4170096021947876,
      "grad_norm": 2.247367124041163,
      "kl": 0.09228515625,
      "learning_rate": 2e-07,
      "loss": 0.0037,
      "reward": 1.300000011920929,
      "reward_std": 0.2808035835623741,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.659375011920929,
      "step": 880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.796875,
      "epoch": 2.419753086419753,
      "grad_norm": 5.109391138156952,
      "kl": 0.1064453125,
      "learning_rate": 1.990909090909091e-07,
      "loss": 0.0043,
      "reward": 1.4718749523162842,
      "reward_std": 0.5331380069255829,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7374999821186066,
      "step": 881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.234375,
      "epoch": 2.422496570644719,
      "grad_norm": 4.962619557575083,
      "kl": 0.131103515625,
      "learning_rate": 1.9818181818181819e-07,
      "loss": 0.0052,
      "reward": 0.8562500178813934,
      "reward_std": 0.5578047335147858,
      "rewards/choice_reward": 0.40625,
      "rewards/roam_reward": 0.44999998807907104,
      "step": 882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.25,
      "epoch": 2.4252400548696844,
      "grad_norm": 7.189866516262367,
      "kl": 0.114013671875,
      "learning_rate": 1.9727272727272726e-07,
      "loss": 0.0046,
      "reward": 1.546093761920929,
      "reward_std": 0.35202687978744507,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7804687321186066,
      "step": 883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.5625,
      "epoch": 2.42798353909465,
      "grad_norm": 7.578308289122683,
      "kl": 0.102783203125,
      "learning_rate": 1.9636363636363635e-07,
      "loss": 0.0041,
      "reward": 1.589062511920929,
      "reward_std": 0.3643190935254097,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7921874821186066,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 68.796875,
      "epoch": 2.430727023319616,
      "grad_norm": 2.800001423540346,
      "kl": 0.109619140625,
      "learning_rate": 1.9545454545454545e-07,
      "loss": 0.0044,
      "reward": 1.7890625,
      "reward_std": 0.20274889469146729,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8984375,
      "step": 885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.875,
      "epoch": 2.4334705075445817,
      "grad_norm": 3.285631712678603,
      "kl": 0.095458984375,
      "learning_rate": 1.9454545454545455e-07,
      "loss": 0.0038,
      "reward": 1.2093749642372131,
      "reward_std": 0.3309120088815689,
      "rewards/choice_reward": 0.59375,
      "rewards/roam_reward": 0.6156249493360519,
      "step": 886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.15625,
      "epoch": 2.4362139917695473,
      "grad_norm": 2.3317520627287553,
      "kl": 0.105224609375,
      "learning_rate": 1.9363636363636362e-07,
      "loss": 0.0042,
      "reward": 1.7132812142372131,
      "reward_std": 0.3037515729665756,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8539062142372131,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.34375,
      "epoch": 2.438957475994513,
      "grad_norm": 1.8537767822147528,
      "kl": 0.097900390625,
      "learning_rate": 1.9272727272727272e-07,
      "loss": 0.0039,
      "reward": 1.666406273841858,
      "reward_std": 0.20769253838807344,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8382812738418579,
      "step": 888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.421875,
      "epoch": 2.4417009602194786,
      "grad_norm": 4.388051923324324,
      "kl": 0.10107421875,
      "learning_rate": 1.9181818181818179e-07,
      "loss": 0.004,
      "reward": 1.814062476158142,
      "reward_std": 0.399493470788002,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.8921874761581421,
      "step": 889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.796875,
      "epoch": 2.4444444444444446,
      "grad_norm": 6.489982708043214,
      "kl": 0.13427734375,
      "learning_rate": 1.909090909090909e-07,
      "loss": 0.0054,
      "reward": 1.6664062142372131,
      "reward_std": 0.21181846410036087,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8382812142372131,
      "step": 890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.03125,
      "epoch": 2.4471879286694103,
      "grad_norm": 4.975200941536546,
      "kl": 0.095703125,
      "learning_rate": 1.8999999999999998e-07,
      "loss": 0.0038,
      "reward": 1.666406273841858,
      "reward_std": 0.35487619042396545,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8382812440395355,
      "step": 891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.59375,
      "epoch": 2.449931412894376,
      "grad_norm": 1.9188236461190016,
      "kl": 0.110107421875,
      "learning_rate": 1.8909090909090908e-07,
      "loss": 0.0044,
      "reward": 1.607031226158142,
      "reward_std": 0.13777265278622508,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8101562559604645,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.1875,
      "epoch": 2.4526748971193415,
      "grad_norm": 3.5207228078772452,
      "kl": 0.16552734375,
      "learning_rate": 1.8818181818181818e-07,
      "loss": 0.0066,
      "reward": 1.6656250357627869,
      "reward_std": 0.42394691705703735,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8374999761581421,
      "step": 893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.015625,
      "epoch": 2.455418381344307,
      "grad_norm": 2.2571189778777634,
      "kl": 0.106201171875,
      "learning_rate": 1.8727272727272727e-07,
      "loss": 0.0042,
      "reward": 1.5390625,
      "reward_std": 0.27705324813723564,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7734374701976776,
      "step": 894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.625,
      "epoch": 2.4581618655692727,
      "grad_norm": 5.707502573988361,
      "kl": 0.1328125,
      "learning_rate": 1.8636363636363637e-07,
      "loss": 0.0053,
      "reward": 1.756250023841858,
      "reward_std": 0.3345056623220444,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.8812500238418579,
      "step": 895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.640625,
      "epoch": 2.460905349794239,
      "grad_norm": 5.383223650884088,
      "kl": 0.16357421875,
      "learning_rate": 1.8545454545454544e-07,
      "loss": 0.0065,
      "reward": 1.634374976158142,
      "reward_std": 0.3058193624019623,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8218749761581421,
      "step": 896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.859375,
      "epoch": 2.4636488340192044,
      "grad_norm": 2.264871070360021,
      "kl": 0.087646484375,
      "learning_rate": 1.8454545454545454e-07,
      "loss": 0.0035,
      "reward": 1.907812476158142,
      "reward_std": 0.192325159907341,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9546875059604645,
      "step": 897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.8125,
      "epoch": 2.46639231824417,
      "grad_norm": 6.510398650036107,
      "kl": 0.1025390625,
      "learning_rate": 1.836363636363636e-07,
      "loss": 0.0041,
      "reward": 1.6257811784744263,
      "reward_std": 0.3411605954170227,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.813281238079071,
      "step": 898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.15625,
      "epoch": 2.4691358024691357,
      "grad_norm": 5.050535777404141,
      "kl": 0.132080078125,
      "learning_rate": 1.8272727272727273e-07,
      "loss": 0.0053,
      "reward": 1.7132812142372131,
      "reward_std": 0.4274836480617523,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8539062738418579,
      "step": 899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.390625,
      "epoch": 2.4718792866941017,
      "grad_norm": 2.4150872091575435,
      "kl": 0.1044921875,
      "learning_rate": 1.818181818181818e-07,
      "loss": 0.0042,
      "reward": 1.7140624523162842,
      "reward_std": 0.1370019344612956,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8546874821186066,
      "step": 900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.984375,
      "epoch": 2.4746227709190673,
      "grad_norm": 2.9416235492274403,
      "kl": 0.087890625,
      "learning_rate": 1.809090909090909e-07,
      "loss": 0.0035,
      "reward": 1.66796875,
      "reward_std": 0.4173927679657936,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.83984375,
      "step": 901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 99.390625,
      "epoch": 2.477366255144033,
      "grad_norm": 3.0301688079902607,
      "kl": 0.106689453125,
      "learning_rate": 1.8e-07,
      "loss": 0.0043,
      "reward": 1.517187476158142,
      "reward_std": 0.6114129722118378,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7515624761581421,
      "step": 902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.9375,
      "epoch": 2.4801097393689986,
      "grad_norm": 2.3823712947325295,
      "kl": 0.086669921875,
      "learning_rate": 1.7909090909090907e-07,
      "loss": 0.0035,
      "reward": 1.821874976158142,
      "reward_std": 0.20688685774803162,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9156250059604645,
      "step": 903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.25,
      "epoch": 2.482853223593964,
      "grad_norm": 5.57538249725218,
      "kl": 0.137451171875,
      "learning_rate": 1.781818181818182e-07,
      "loss": 0.0055,
      "reward": 1.578125,
      "reward_std": 0.40212661027908325,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.796875,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.890625,
      "epoch": 2.48559670781893,
      "grad_norm": 2.6641129082049244,
      "kl": 0.115234375,
      "learning_rate": 1.7727272727272726e-07,
      "loss": 0.0046,
      "reward": 1.862500011920929,
      "reward_std": 0.2599198669195175,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.925000011920929,
      "step": 905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.875,
      "epoch": 2.488340192043896,
      "grad_norm": 3.0206434726292994,
      "kl": 0.1044921875,
      "learning_rate": 1.7636363636363636e-07,
      "loss": 0.0042,
      "reward": 1.579687476158142,
      "reward_std": 0.3286094591021538,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7984374463558197,
      "step": 906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.875,
      "epoch": 2.4910836762688615,
      "grad_norm": 4.455283714421392,
      "kl": 0.1044921875,
      "learning_rate": 1.7545454545454543e-07,
      "loss": 0.0042,
      "reward": 1.817187488079071,
      "reward_std": 0.3769528269767761,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.895312488079071,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.9375,
      "epoch": 2.493827160493827,
      "grad_norm": 2.6112679569674717,
      "kl": 0.093017578125,
      "learning_rate": 1.7454545454545455e-07,
      "loss": 0.0037,
      "reward": 1.1070312559604645,
      "reward_std": 0.11657025711610913,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5757812559604645,
      "step": 908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.265625,
      "epoch": 2.4965706447187928,
      "grad_norm": 8.190560736272449,
      "kl": 0.09326171875,
      "learning_rate": 1.7363636363636362e-07,
      "loss": 0.0037,
      "reward": 1.493749976158142,
      "reward_std": 0.4932164400815964,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7593750059604645,
      "step": 909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.8125,
      "epoch": 2.499314128943759,
      "grad_norm": 8.718805678669755,
      "kl": 0.107177734375,
      "learning_rate": 1.7272727272727272e-07,
      "loss": 0.0043,
      "reward": 1.600781261920929,
      "reward_std": 0.23684065975248814,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.8039062321186066,
      "step": 910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.546875,
      "epoch": 2.5020576131687244,
      "grad_norm": 4.198056415866492,
      "kl": 0.101318359375,
      "learning_rate": 1.7181818181818182e-07,
      "loss": 0.004,
      "reward": 1.310937523841858,
      "reward_std": 0.3555447608232498,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6703124642372131,
      "step": 911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.40625,
      "epoch": 2.50480109739369,
      "grad_norm": 4.373002370025674,
      "kl": 0.133056640625,
      "learning_rate": 1.709090909090909e-07,
      "loss": 0.0053,
      "reward": 1.3937500417232513,
      "reward_std": 0.5110341310501099,
      "rewards/choice_reward": 0.6875,
      "rewards/roam_reward": 0.7062499821186066,
      "step": 912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.421875,
      "epoch": 2.5075445816186557,
      "grad_norm": 28.22264100129003,
      "kl": 0.197265625,
      "learning_rate": 1.7000000000000001e-07,
      "loss": 0.0079,
      "reward": 1.8171875476837158,
      "reward_std": 0.391438253223896,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9109375476837158,
      "step": 913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.328125,
      "epoch": 2.5102880658436213,
      "grad_norm": 4.10992951027992,
      "kl": 0.116943359375,
      "learning_rate": 1.6909090909090908e-07,
      "loss": 0.0047,
      "reward": 1.6726561784744263,
      "reward_std": 0.45760543644428253,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.844531238079071,
      "step": 914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.421875,
      "epoch": 2.513031550068587,
      "grad_norm": 5.033545976301883,
      "kl": 0.08740234375,
      "learning_rate": 1.6818181818181818e-07,
      "loss": 0.0035,
      "reward": 1.4656250476837158,
      "reward_std": 0.5237148702144623,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7468750178813934,
      "step": 915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.15625,
      "epoch": 2.5157750342935525,
      "grad_norm": 4.4992682723749775,
      "kl": 0.09912109375,
      "learning_rate": 1.6727272727272725e-07,
      "loss": 0.004,
      "reward": 1.590624988079071,
      "reward_std": 0.2772092744708061,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.793749988079071,
      "step": 916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.953125,
      "epoch": 2.5185185185185186,
      "grad_norm": 7.067973308531796,
      "kl": 0.137939453125,
      "learning_rate": 1.6636363636363635e-07,
      "loss": 0.0055,
      "reward": 1.67578125,
      "reward_std": 0.46074244379997253,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.83203125,
      "step": 917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.78125,
      "epoch": 2.521262002743484,
      "grad_norm": 2.956470810154343,
      "kl": 0.101806640625,
      "learning_rate": 1.6545454545454545e-07,
      "loss": 0.0041,
      "reward": 1.2531249523162842,
      "reward_std": 0.5866954028606415,
      "rewards/choice_reward": 0.609375,
      "rewards/roam_reward": 0.6437499523162842,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.53125,
      "epoch": 2.52400548696845,
      "grad_norm": 6.591829552719709,
      "kl": 0.10205078125,
      "learning_rate": 1.6454545454545454e-07,
      "loss": 0.0041,
      "reward": 1.357812523841858,
      "reward_std": 0.47224710136651993,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6859374940395355,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.4375,
      "epoch": 2.526748971193416,
      "grad_norm": 7.515520905203439,
      "kl": 0.1015625,
      "learning_rate": 1.6363636363636364e-07,
      "loss": 0.0041,
      "reward": 1.455468773841858,
      "reward_std": 0.37790268659591675,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7367187738418579,
      "step": 920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.90625,
      "epoch": 2.5294924554183815,
      "grad_norm": 1.8682810884686853,
      "kl": 0.1240234375,
      "learning_rate": 1.627272727272727e-07,
      "loss": 0.0049,
      "reward": 1.6382812857627869,
      "reward_std": 0.28293294459581375,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8257812857627869,
      "step": 921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.25,
      "epoch": 2.532235939643347,
      "grad_norm": 3.8243588025411652,
      "kl": 0.0947265625,
      "learning_rate": 1.618181818181818e-07,
      "loss": 0.0038,
      "reward": 1.5343749523162842,
      "reward_std": 0.5920023918151855,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7374999821186066,
      "step": 922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.90625,
      "epoch": 2.5349794238683128,
      "grad_norm": 3.694037239049243,
      "kl": 0.093994140625,
      "learning_rate": 1.609090909090909e-07,
      "loss": 0.0038,
      "reward": 1.2859375178813934,
      "reward_std": 0.3922395408153534,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6609375178813934,
      "step": 923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.265625,
      "epoch": 2.5377229080932784,
      "grad_norm": 1.595043885674026,
      "kl": 0.09033203125,
      "learning_rate": 1.6e-07,
      "loss": 0.0036,
      "reward": 1.7015624642372131,
      "reward_std": 0.17235727235674858,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8421874940395355,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.34375,
      "epoch": 2.540466392318244,
      "grad_norm": 2.959211601757336,
      "kl": 0.095703125,
      "learning_rate": 1.5909090909090907e-07,
      "loss": 0.0038,
      "reward": 1.5499999523162842,
      "reward_std": 0.33859558403491974,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7843749821186066,
      "step": 925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 90.203125,
      "epoch": 2.5432098765432096,
      "grad_norm": 5.012725913458301,
      "kl": 0.154296875,
      "learning_rate": 1.5818181818181817e-07,
      "loss": 0.0062,
      "reward": 1.5078125,
      "reward_std": 0.4259311109781265,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7578125,
      "step": 926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.453125,
      "epoch": 2.5459533607681757,
      "grad_norm": 3.0997758801642994,
      "kl": 0.09130859375,
      "learning_rate": 1.5727272727272727e-07,
      "loss": 0.0037,
      "reward": 1.543749988079071,
      "reward_std": 0.5073673129081726,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.746874988079071,
      "step": 927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.078125,
      "epoch": 2.5486968449931413,
      "grad_norm": 5.2508460190751975,
      "kl": 0.1943359375,
      "learning_rate": 1.5636363636363637e-07,
      "loss": 0.0078,
      "reward": 1.306249976158142,
      "reward_std": 0.694450169801712,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6656250059604645,
      "step": 928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.703125,
      "epoch": 2.551440329218107,
      "grad_norm": 3.3798454745729902,
      "kl": 0.09765625,
      "learning_rate": 1.5545454545454544e-07,
      "loss": 0.0039,
      "reward": 1.760937511920929,
      "reward_std": 0.38189470767974854,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.870312511920929,
      "step": 929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.5,
      "epoch": 2.5541838134430725,
      "grad_norm": 10.457283078493456,
      "kl": 0.100830078125,
      "learning_rate": 1.5454545454545453e-07,
      "loss": 0.004,
      "reward": 1.623437523841858,
      "reward_std": 0.4348487854003906,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8109375238418579,
      "step": 930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.296875,
      "epoch": 2.5569272976680386,
      "grad_norm": 3.075962228788098,
      "kl": 0.09765625,
      "learning_rate": 1.5363636363636363e-07,
      "loss": 0.0039,
      "reward": 1.790624976158142,
      "reward_std": 0.2952752038836479,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.9000000059604645,
      "step": 931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 70.703125,
      "epoch": 2.5596707818930042,
      "grad_norm": 6.633142433143458,
      "kl": 0.147705078125,
      "learning_rate": 1.5272727272727273e-07,
      "loss": 0.0059,
      "reward": 1.8921875357627869,
      "reward_std": 0.2794111743569374,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9390625357627869,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.296875,
      "epoch": 2.56241426611797,
      "grad_norm": 2.1414242451237815,
      "kl": 0.0693359375,
      "learning_rate": 1.5181818181818183e-07,
      "loss": 0.0028,
      "reward": 1.7125000357627869,
      "reward_std": 0.1414213553071022,
      "rewards/choice_reward": 0.859375,
      "rewards/roam_reward": 0.8531249761581421,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.90625,
      "epoch": 2.5651577503429355,
      "grad_norm": 41.78678606748375,
      "kl": 0.092529296875,
      "learning_rate": 1.509090909090909e-07,
      "loss": 0.0037,
      "reward": 1.5468750596046448,
      "reward_std": 0.377840094268322,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.78125,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.0625,
      "epoch": 2.567901234567901,
      "grad_norm": 5.584232435399464,
      "kl": 0.1123046875,
      "learning_rate": 1.5e-07,
      "loss": 0.0045,
      "reward": 1.489843726158142,
      "reward_std": 0.44570471346378326,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7398437559604645,
      "step": 935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.96875,
      "epoch": 2.5706447187928667,
      "grad_norm": 2.791576141853459,
      "kl": 0.102294921875,
      "learning_rate": 1.4909090909090906e-07,
      "loss": 0.0041,
      "reward": 1.696874976158142,
      "reward_std": 0.43145138025283813,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8531250059604645,
      "step": 936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.578125,
      "epoch": 2.5733882030178328,
      "grad_norm": 3.053624480174356,
      "kl": 0.0888671875,
      "learning_rate": 1.481818181818182e-07,
      "loss": 0.0035,
      "reward": 1.5453124642372131,
      "reward_std": 0.26508478820323944,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7796875238418579,
      "step": 937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.171875,
      "epoch": 2.5761316872427984,
      "grad_norm": 2.973330455331203,
      "kl": 0.09765625,
      "learning_rate": 1.4727272727272726e-07,
      "loss": 0.0039,
      "reward": 1.344531238079071,
      "reward_std": 0.32123443484306335,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.672656238079071,
      "step": 938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.71875,
      "epoch": 2.578875171467764,
      "grad_norm": 5.163306118515068,
      "kl": 0.101318359375,
      "learning_rate": 1.4636363636363636e-07,
      "loss": 0.0041,
      "reward": 1.6968750357627869,
      "reward_std": 0.3943808674812317,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.8531250357627869,
      "step": 939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.515625,
      "epoch": 2.5816186556927296,
      "grad_norm": 3.5058106697132816,
      "kl": 0.106201171875,
      "learning_rate": 1.4545454545454545e-07,
      "loss": 0.0043,
      "reward": 1.544531226158142,
      "reward_std": 0.5131105482578278,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7789062261581421,
      "step": 940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 76.921875,
      "epoch": 2.5843621399176957,
      "grad_norm": 13.907951590750768,
      "kl": 0.146484375,
      "learning_rate": 1.4454545454545452e-07,
      "loss": 0.0058,
      "reward": 1.4218749403953552,
      "reward_std": 0.5525650680065155,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.703125,
      "step": 941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.265625,
      "epoch": 2.5871056241426613,
      "grad_norm": 14.992872788243073,
      "kl": 0.098876953125,
      "learning_rate": 1.4363636363636365e-07,
      "loss": 0.004,
      "reward": 1.815625011920929,
      "reward_std": 0.2820400223135948,
      "rewards/choice_reward": 0.921875,
      "rewards/roam_reward": 0.8937499821186066,
      "step": 942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.3125,
      "epoch": 2.589849108367627,
      "grad_norm": 4.11391804323829,
      "kl": 0.113037109375,
      "learning_rate": 1.4272727272727272e-07,
      "loss": 0.0045,
      "reward": 1.372656226158142,
      "reward_std": 0.4981390982866287,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.7007812261581421,
      "step": 943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.046875,
      "epoch": 2.5925925925925926,
      "grad_norm": 4.316485995076902,
      "kl": 0.105224609375,
      "learning_rate": 1.4181818181818182e-07,
      "loss": 0.0042,
      "reward": 1.342187523841858,
      "reward_std": 0.46965810656547546,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6859374642372131,
      "step": 944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.375,
      "epoch": 2.595336076817558,
      "grad_norm": 1.911352539641548,
      "kl": 0.089111328125,
      "learning_rate": 1.4090909090909089e-07,
      "loss": 0.0036,
      "reward": 1.547656238079071,
      "reward_std": 0.0964670218527317,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.782031238079071,
      "step": 945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.1875,
      "epoch": 2.598079561042524,
      "grad_norm": 2.4993048044470036,
      "kl": 0.10205078125,
      "learning_rate": 1.4e-07,
      "loss": 0.0041,
      "reward": 1.486718773841858,
      "reward_std": 0.33659204840660095,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7523437142372131,
      "step": 946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.578125,
      "epoch": 2.60082304526749,
      "grad_norm": 2.761243514278863,
      "kl": 0.096435546875,
      "learning_rate": 1.3909090909090908e-07,
      "loss": 0.0039,
      "reward": 1.755468726158142,
      "reward_std": 0.4127981811761856,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8648437559604645,
      "step": 947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.90625,
      "epoch": 2.6035665294924555,
      "grad_norm": 3.2114970692303935,
      "kl": 0.111328125,
      "learning_rate": 1.3818181818181818e-07,
      "loss": 0.0044,
      "reward": 1.672656238079071,
      "reward_std": 0.6133669316768646,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.828906238079071,
      "step": 948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.796875,
      "epoch": 2.606310013717421,
      "grad_norm": 3.318991870606774,
      "kl": 0.0966796875,
      "learning_rate": 1.3727272727272727e-07,
      "loss": 0.0039,
      "reward": 1.489062488079071,
      "reward_std": 0.38795000687241554,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.739062488079071,
      "step": 949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.1875,
      "epoch": 2.6090534979423867,
      "grad_norm": 4.713917493991348,
      "kl": 0.101806640625,
      "learning_rate": 1.3636363636363635e-07,
      "loss": 0.0041,
      "reward": 1.647656261920929,
      "reward_std": 0.4856095612049103,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8195312321186066,
      "step": 950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.796875,
      "epoch": 2.611796982167353,
      "grad_norm": 4.250557230931642,
      "kl": 0.098876953125,
      "learning_rate": 1.3545454545454547e-07,
      "loss": 0.004,
      "reward": 1.0906249582767487,
      "reward_std": 0.36680732667446136,
      "rewards/choice_reward": 0.53125,
      "rewards/roam_reward": 0.5593749582767487,
      "step": 951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 99.796875,
      "epoch": 2.6145404663923184,
      "grad_norm": 5.571591310306311,
      "kl": 0.09423828125,
      "learning_rate": 1.3454545454545454e-07,
      "loss": 0.0038,
      "reward": 1.456250011920929,
      "reward_std": 0.42433546483516693,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.7374999821186066,
      "step": 952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.28125,
      "epoch": 2.617283950617284,
      "grad_norm": 8.165042661619838,
      "kl": 0.121337890625,
      "learning_rate": 1.3363636363636364e-07,
      "loss": 0.0048,
      "reward": 1.5718750357627869,
      "reward_std": 0.5588430762290955,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7906250357627869,
      "step": 953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.765625,
      "epoch": 2.6200274348422496,
      "grad_norm": 3.7181708604261883,
      "kl": 0.093994140625,
      "learning_rate": 1.327272727272727e-07,
      "loss": 0.0038,
      "reward": 1.635937511920929,
      "reward_std": 0.3050750121474266,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8234374821186066,
      "step": 954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.71875,
      "epoch": 2.6227709190672153,
      "grad_norm": 4.792225130509411,
      "kl": 0.10546875,
      "learning_rate": 1.318181818181818e-07,
      "loss": 0.0042,
      "reward": 1.5187499523162842,
      "reward_std": 0.23608315736055374,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7687499523162842,
      "step": 955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.859375,
      "epoch": 2.625514403292181,
      "grad_norm": 2.071587920844544,
      "kl": 0.09326171875,
      "learning_rate": 1.309090909090909e-07,
      "loss": 0.0037,
      "reward": 1.565625011920929,
      "reward_std": 0.2502797245979309,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7843749821186066,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.0625,
      "epoch": 2.6282578875171465,
      "grad_norm": 3.2967853377032807,
      "kl": 0.10302734375,
      "learning_rate": 1.3e-07,
      "loss": 0.0041,
      "reward": 1.3421874642372131,
      "reward_std": 0.3264111652970314,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6859374940395355,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.09375,
      "epoch": 2.6310013717421126,
      "grad_norm": 4.646329496583205,
      "kl": 0.094482421875,
      "learning_rate": 1.290909090909091e-07,
      "loss": 0.0038,
      "reward": 1.40625,
      "reward_std": 0.5289385914802551,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7031249701976776,
      "step": 958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 93.8125,
      "epoch": 2.633744855967078,
      "grad_norm": 13.25104189197075,
      "kl": 0.087158203125,
      "learning_rate": 1.2818181818181817e-07,
      "loss": 0.0035,
      "reward": 1.466406226158142,
      "reward_std": 0.472397118806839,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7320312261581421,
      "step": 959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.40625,
      "epoch": 2.636488340192044,
      "grad_norm": 2.5446216685180474,
      "kl": 0.09423828125,
      "learning_rate": 1.2727272727272726e-07,
      "loss": 0.0038,
      "reward": 1.584375023841858,
      "reward_std": 0.4048280864953995,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.8031249642372131,
      "step": 960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 97.828125,
      "epoch": 2.63923182441701,
      "grad_norm": 3.555437790786787,
      "kl": 0.109130859375,
      "learning_rate": 1.2636363636363636e-07,
      "loss": 0.0044,
      "reward": 1.470312476158142,
      "reward_std": 0.35928191244602203,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7359375059604645,
      "step": 961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.671875,
      "epoch": 2.6419753086419755,
      "grad_norm": 3.6399867879843604,
      "kl": 0.09619140625,
      "learning_rate": 1.2545454545454546e-07,
      "loss": 0.0039,
      "reward": 1.364062488079071,
      "reward_std": 0.47104281187057495,
      "rewards/choice_reward": 0.671875,
      "rewards/roam_reward": 0.6921875178813934,
      "step": 962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.390625,
      "epoch": 2.644718792866941,
      "grad_norm": 3.586542860919556,
      "kl": 0.115478515625,
      "learning_rate": 1.2454545454545453e-07,
      "loss": 0.0046,
      "reward": 1.632031261920929,
      "reward_std": 0.32563401013612747,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8195312321186066,
      "step": 963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.21875,
      "epoch": 2.6474622770919067,
      "grad_norm": 9.179120368142398,
      "kl": 0.07470703125,
      "learning_rate": 1.2363636363636363e-07,
      "loss": 0.003,
      "reward": 0.890625,
      "reward_std": 0.4841189831495285,
      "rewards/choice_reward": 0.421875,
      "rewards/roam_reward": 0.4687499701976776,
      "step": 964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.890625,
      "epoch": 2.6502057613168724,
      "grad_norm": 6.378830292139383,
      "kl": 0.12109375,
      "learning_rate": 1.2272727272727272e-07,
      "loss": 0.0048,
      "reward": 1.5859375,
      "reward_std": 0.4652404338121414,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7890624403953552,
      "step": 965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.375,
      "epoch": 2.652949245541838,
      "grad_norm": 2.953950711020351,
      "kl": 0.0966796875,
      "learning_rate": 1.218181818181818e-07,
      "loss": 0.0039,
      "reward": 1.4562499523162842,
      "reward_std": 0.23340336233377457,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.737500011920929,
      "step": 966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 73.28125,
      "epoch": 2.6556927297668036,
      "grad_norm": 4.6014001745806015,
      "kl": 0.134765625,
      "learning_rate": 1.209090909090909e-07,
      "loss": 0.0054,
      "reward": 1.1312499642372131,
      "reward_std": 0.3415980935096741,
      "rewards/choice_reward": 0.546875,
      "rewards/roam_reward": 0.5843749940395355,
      "step": 967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.296875,
      "epoch": 2.6584362139917697,
      "grad_norm": 3.5615170855204568,
      "kl": 0.108642578125,
      "learning_rate": 1.2e-07,
      "loss": 0.0043,
      "reward": 1.737500011920929,
      "reward_std": 0.4942738860845566,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.862500011920929,
      "step": 968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.140625,
      "epoch": 2.6611796982167353,
      "grad_norm": 3.1516398632960962,
      "kl": 0.094482421875,
      "learning_rate": 1.1909090909090909e-07,
      "loss": 0.0038,
      "reward": 1.7671875357627869,
      "reward_std": 0.258515827357769,
      "rewards/choice_reward": 0.890625,
      "rewards/roam_reward": 0.8765625059604645,
      "step": 969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.4375,
      "epoch": 2.663923182441701,
      "grad_norm": 4.6750764993061455,
      "kl": 0.094482421875,
      "learning_rate": 1.1818181818181817e-07,
      "loss": 0.0038,
      "reward": 1.1898437142372131,
      "reward_std": 0.45025503635406494,
      "rewards/choice_reward": 0.578125,
      "rewards/roam_reward": 0.6117187142372131,
      "step": 970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 83.53125,
      "epoch": 2.6666666666666665,
      "grad_norm": 2.1423374157776482,
      "kl": 0.1005859375,
      "learning_rate": 1.1727272727272727e-07,
      "loss": 0.004,
      "reward": 1.879687488079071,
      "reward_std": 0.21130627393722534,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.9421875178813934,
      "step": 971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.859375,
      "epoch": 2.6694101508916326,
      "grad_norm": 3.7705135364888744,
      "kl": 0.11669921875,
      "learning_rate": 1.1636363636363636e-07,
      "loss": 0.0047,
      "reward": 1.311718761920929,
      "reward_std": 0.17687445972114801,
      "rewards/choice_reward": 0.65625,
      "rewards/roam_reward": 0.6554687321186066,
      "step": 972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.21875,
      "epoch": 2.672153635116598,
      "grad_norm": 2.6694912934157533,
      "kl": 0.09375,
      "learning_rate": 1.1545454545454545e-07,
      "loss": 0.0037,
      "reward": 1.9046874642372131,
      "reward_std": 0.23905985802412033,
      "rewards/choice_reward": 0.96875,
      "rewards/roam_reward": 0.9359374940395355,
      "step": 973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.1875,
      "epoch": 2.674897119341564,
      "grad_norm": 4.3604941363825125,
      "kl": 0.11083984375,
      "learning_rate": 1.1454545454545455e-07,
      "loss": 0.0044,
      "reward": 1.690625011920929,
      "reward_std": 0.5343562066555023,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.846875011920929,
      "step": 974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.265625,
      "epoch": 2.6776406035665294,
      "grad_norm": 6.6361785325827185,
      "kl": 0.100341796875,
      "learning_rate": 1.1363636363636363e-07,
      "loss": 0.004,
      "reward": 1.5109375715255737,
      "reward_std": 0.49013426899909973,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7453124821186066,
      "step": 975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 86.96875,
      "epoch": 2.680384087791495,
      "grad_norm": 2.9479493932753082,
      "kl": 0.09228515625,
      "learning_rate": 1.1272727272727273e-07,
      "loss": 0.0037,
      "reward": 1.653124988079071,
      "reward_std": 0.39804912731051445,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.824999988079071,
      "step": 976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.015625,
      "epoch": 2.6831275720164607,
      "grad_norm": 9.367804358161262,
      "kl": 0.133056640625,
      "learning_rate": 1.1181818181818181e-07,
      "loss": 0.0053,
      "reward": 1.471093773841858,
      "reward_std": 0.4260319769382477,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7367187142372131,
      "step": 977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.453125,
      "epoch": 2.6858710562414267,
      "grad_norm": 7.925544902541567,
      "kl": 0.14599609375,
      "learning_rate": 1.1090909090909091e-07,
      "loss": 0.0058,
      "reward": 1.69921875,
      "reward_std": 0.4324868619441986,
      "rewards/choice_reward": 0.84375,
      "rewards/roam_reward": 0.85546875,
      "step": 978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 79.9375,
      "epoch": 2.6886145404663924,
      "grad_norm": 6.647960389727826,
      "kl": 0.103515625,
      "learning_rate": 1.0999999999999999e-07,
      "loss": 0.0041,
      "reward": 1.461718738079071,
      "reward_std": 0.4704754278063774,
      "rewards/choice_reward": 0.71875,
      "rewards/roam_reward": 0.742968738079071,
      "step": 979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.421875,
      "epoch": 2.691358024691358,
      "grad_norm": 5.327850308136772,
      "kl": 0.12255859375,
      "learning_rate": 1.0909090909090908e-07,
      "loss": 0.0049,
      "reward": 1.482031226158142,
      "reward_std": 0.37536171078681946,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7320312261581421,
      "step": 980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.859375,
      "epoch": 2.6941015089163236,
      "grad_norm": 2.7605446866186165,
      "kl": 0.083740234375,
      "learning_rate": 1.0818181818181817e-07,
      "loss": 0.0034,
      "reward": 1.420312523841858,
      "reward_std": 0.4252743124961853,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7171874940395355,
      "step": 981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 80.9375,
      "epoch": 2.6968449931412897,
      "grad_norm": 3.5504893248826876,
      "kl": 0.0908203125,
      "learning_rate": 1.0727272727272727e-07,
      "loss": 0.0036,
      "reward": 1.534375011920929,
      "reward_std": 0.36864815652370453,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7687499821186066,
      "step": 982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 85.0625,
      "epoch": 2.6995884773662553,
      "grad_norm": 3.1240045755403387,
      "kl": 0.104736328125,
      "learning_rate": 1.0636363636363637e-07,
      "loss": 0.0042,
      "reward": 1.310937523841858,
      "reward_std": 0.1016465974971652,
      "rewards/choice_reward": 0.640625,
      "rewards/roam_reward": 0.6703124642372131,
      "step": 983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 91.609375,
      "epoch": 2.702331961591221,
      "grad_norm": 3.7008525745312175,
      "kl": 0.107421875,
      "learning_rate": 1.0545454545454545e-07,
      "loss": 0.0043,
      "reward": 1.4453125,
      "reward_std": 0.4314822629094124,
      "rewards/choice_reward": 0.734375,
      "rewards/roam_reward": 0.7109374701976776,
      "step": 984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 88.28125,
      "epoch": 2.7050754458161865,
      "grad_norm": 3.840966428690235,
      "kl": 0.112060546875,
      "learning_rate": 1.0454545454545454e-07,
      "loss": 0.0045,
      "reward": 1.416406273841858,
      "reward_std": 0.24614976346492767,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7132812142372131,
      "step": 985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.734375,
      "epoch": 2.707818930041152,
      "grad_norm": 2.917343856165196,
      "kl": 0.109375,
      "learning_rate": 1.0363636363636363e-07,
      "loss": 0.0044,
      "reward": 1.73828125,
      "reward_std": 0.3371606692671776,
      "rewards/choice_reward": 0.875,
      "rewards/roam_reward": 0.86328125,
      "step": 986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 89.71875,
      "epoch": 2.7105624142661178,
      "grad_norm": 3.098225905201064,
      "kl": 0.087646484375,
      "learning_rate": 1.0272727272727272e-07,
      "loss": 0.0035,
      "reward": 1.2874999642372131,
      "reward_std": 0.3168279901146889,
      "rewards/choice_reward": 0.625,
      "rewards/roam_reward": 0.6624999940395355,
      "step": 987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 96.296875,
      "epoch": 2.713305898491084,
      "grad_norm": 2.4222336122305737,
      "kl": 0.08349609375,
      "learning_rate": 1.0181818181818181e-07,
      "loss": 0.0033,
      "reward": 1.817968726158142,
      "reward_std": 0.24658063054084778,
      "rewards/choice_reward": 0.90625,
      "rewards/roam_reward": 0.9117187559604645,
      "step": 988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.78125,
      "epoch": 2.7160493827160495,
      "grad_norm": 4.645635153732883,
      "kl": 0.111083984375,
      "learning_rate": 1.009090909090909e-07,
      "loss": 0.0044,
      "reward": 1.634374976158142,
      "reward_std": 0.371658131480217,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8218750059604645,
      "step": 989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.4375,
      "epoch": 2.718792866941015,
      "grad_norm": 3.003399224538686,
      "kl": 0.09716796875,
      "learning_rate": 1e-07,
      "loss": 0.0039,
      "reward": 1.516406238079071,
      "reward_std": 0.42474687099456787,
      "rewards/choice_reward": 0.75,
      "rewards/roam_reward": 0.7664062678813934,
      "step": 990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.46875,
      "epoch": 2.7215363511659807,
      "grad_norm": 4.226013969835389,
      "kl": 0.115966796875,
      "learning_rate": 9.909090909090909e-08,
      "loss": 0.0046,
      "reward": 1.5312500596046448,
      "reward_std": 0.5685274600982666,
      "rewards/choice_reward": 0.765625,
      "rewards/roam_reward": 0.7656250298023224,
      "step": 991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 87.390625,
      "epoch": 2.7242798353909468,
      "grad_norm": 12.350588784104131,
      "kl": 0.099365234375,
      "learning_rate": 9.818181818181818e-08,
      "loss": 0.004,
      "reward": 1.4312500357627869,
      "reward_std": 0.5584160685539246,
      "rewards/choice_reward": 0.703125,
      "rewards/roam_reward": 0.7281250059604645,
      "step": 992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 82.515625,
      "epoch": 2.7270233196159124,
      "grad_norm": 2.46709204077942,
      "kl": 0.08056640625,
      "learning_rate": 9.727272727272727e-08,
      "loss": 0.0032,
      "reward": 1.591406226158142,
      "reward_std": 0.4659492149949074,
      "rewards/choice_reward": 0.796875,
      "rewards/roam_reward": 0.7945312559604645,
      "step": 993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 98.046875,
      "epoch": 2.729766803840878,
      "grad_norm": 2.6581896216315553,
      "kl": 0.0927734375,
      "learning_rate": 9.636363636363636e-08,
      "loss": 0.0037,
      "reward": 1.028906226158142,
      "reward_std": 0.48388032615184784,
      "rewards/choice_reward": 0.5,
      "rewards/roam_reward": 0.5289062261581421,
      "step": 994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 84.8125,
      "epoch": 2.7325102880658436,
      "grad_norm": 2.231189670238404,
      "kl": 0.098876953125,
      "learning_rate": 9.545454545454546e-08,
      "loss": 0.0039,
      "reward": 1.6640625,
      "reward_std": 0.2100028172135353,
      "rewards/choice_reward": 0.828125,
      "rewards/roam_reward": 0.8359375,
      "step": 995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 92.34375,
      "epoch": 2.7352537722908092,
      "grad_norm": 7.111804822235356,
      "kl": 0.1337890625,
      "learning_rate": 9.454545454545454e-08,
      "loss": 0.0054,
      "reward": 1.5531249046325684,
      "reward_std": 0.49822041392326355,
      "rewards/choice_reward": 0.78125,
      "rewards/roam_reward": 0.7718750238418579,
      "step": 996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.78125,
      "epoch": 2.737997256515775,
      "grad_norm": 2.1757372856376485,
      "kl": 0.091552734375,
      "learning_rate": 9.363636363636364e-08,
      "loss": 0.0037,
      "reward": 1.833593726158142,
      "reward_std": 0.22985859215259552,
      "rewards/choice_reward": 0.9375,
      "rewards/roam_reward": 0.8960937559604645,
      "step": 997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 77.234375,
      "epoch": 2.7407407407407405,
      "grad_norm": 6.25212799141905,
      "kl": 0.137451171875,
      "learning_rate": 9.272727272727272e-08,
      "loss": 0.0055,
      "reward": 1.6390624642372131,
      "reward_std": 0.2201451137661934,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8265624642372131,
      "step": 998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 78.03125,
      "epoch": 2.7434842249657065,
      "grad_norm": 3.0969955054568614,
      "kl": 0.09814453125,
      "learning_rate": 9.18181818181818e-08,
      "loss": 0.0039,
      "reward": 1.6343750357627869,
      "reward_std": 0.28497179597616196,
      "rewards/choice_reward": 0.8125,
      "rewards/roam_reward": 0.8218750357627869,
      "step": 999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 81.53125,
      "epoch": 2.746227709190672,
      "grad_norm": 2.700965617559501,
      "kl": 0.122314453125,
      "learning_rate": 9.09090909090909e-08,
      "loss": 0.0049,
      "reward": 1.903124988079071,
      "reward_std": 0.26436372846364975,
      "rewards/choice_reward": 0.953125,
      "rewards/roam_reward": 0.9500000178813934,
      "step": 1000
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 1100,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}