| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.746227709190672, |
| "eval_steps": 500, |
| "global_step": 1000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.09375, |
| "epoch": 0.0027434842249657062, |
| "grad_norm": 22.875594605913278, |
| "kl": 0.0, |
| "learning_rate": 9.990909090909091e-07, |
| "loss": 0.0, |
| "reward": 0.5414062440395355, |
| "reward_std": 0.4481600522994995, |
| "rewards/choice_reward": 0.28125, |
| "rewards/roam_reward": 0.2601562440395355, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.328125, |
| "epoch": 0.0054869684499314125, |
| "grad_norm": 23.368213544115484, |
| "kl": 0.0011501312255859375, |
| "learning_rate": 9.98181818181818e-07, |
| "loss": 0.0, |
| "reward": 0.7562499940395355, |
| "reward_std": 0.7361946105957031, |
| "rewards/choice_reward": 0.40625, |
| "rewards/roam_reward": 0.34999997913837433, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.5625, |
| "epoch": 0.00823045267489712, |
| "grad_norm": 6.439502461054405, |
| "kl": 0.001247406005859375, |
| "learning_rate": 9.972727272727273e-07, |
| "loss": 0.0, |
| "reward": 0.7953124940395355, |
| "reward_std": 0.8322779834270477, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.3421874940395355, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.21875, |
| "epoch": 0.010973936899862825, |
| "grad_norm": 5.055634258308147, |
| "kl": 0.002346038818359375, |
| "learning_rate": 9.963636363636362e-07, |
| "loss": 0.0001, |
| "reward": 0.813281238079071, |
| "reward_std": 0.6915583610534668, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.36015625298023224, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.046875, |
| "epoch": 0.013717421124828532, |
| "grad_norm": 5.4128284043265396, |
| "kl": 0.002582550048828125, |
| "learning_rate": 9.954545454545454e-07, |
| "loss": 0.0001, |
| "reward": 0.571093738079071, |
| "reward_std": 0.6553446054458618, |
| "rewards/choice_reward": 0.328125, |
| "rewards/roam_reward": 0.24296873062849045, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.9375, |
| "epoch": 0.01646090534979424, |
| "grad_norm": 4.376063841540647, |
| "kl": 0.00384521484375, |
| "learning_rate": 9.945454545454544e-07, |
| "loss": 0.0002, |
| "reward": 0.8460937142372131, |
| "reward_std": 0.6018270701169968, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.3773437440395355, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.390625, |
| "epoch": 0.019204389574759947, |
| "grad_norm": 7.101229751865706, |
| "kl": 0.01727294921875, |
| "learning_rate": 9.936363636363636e-07, |
| "loss": 0.0007, |
| "reward": 1.126562476158142, |
| "reward_std": 0.7818476557731628, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.5015625208616257, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.78125, |
| "epoch": 0.02194787379972565, |
| "grad_norm": 6.022296712805634, |
| "kl": 0.0061187744140625, |
| "learning_rate": 9.927272727272727e-07, |
| "loss": 0.0002, |
| "reward": 1.0984374284744263, |
| "reward_std": 0.8049026429653168, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.48906251788139343, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.3125, |
| "epoch": 0.024691358024691357, |
| "grad_norm": 4.245932293968646, |
| "kl": 0.01190185546875, |
| "learning_rate": 9.918181818181817e-07, |
| "loss": 0.0005, |
| "reward": 0.6945312321186066, |
| "reward_std": 0.3619903400540352, |
| "rewards/choice_reward": 0.359375, |
| "rewards/roam_reward": 0.33515624701976776, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.484375, |
| "epoch": 0.027434842249657063, |
| "grad_norm": 8.484504056440915, |
| "kl": 0.0076446533203125, |
| "learning_rate": 9.909090909090909e-07, |
| "loss": 0.0003, |
| "reward": 0.90625, |
| "reward_std": 0.45835646986961365, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.3906249850988388, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.03125, |
| "epoch": 0.03017832647462277, |
| "grad_norm": 7.742548302467443, |
| "kl": 0.0139007568359375, |
| "learning_rate": 9.9e-07, |
| "loss": 0.0006, |
| "reward": 0.8593749701976776, |
| "reward_std": 0.43932054936885834, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.390625, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.84375, |
| "epoch": 0.03292181069958848, |
| "grad_norm": 3.983053793753651, |
| "kl": 0.011077880859375, |
| "learning_rate": 9.89090909090909e-07, |
| "loss": 0.0004, |
| "reward": 1.0476561784744263, |
| "reward_std": 0.789838433265686, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.46953126788139343, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.25, |
| "epoch": 0.03566529492455418, |
| "grad_norm": 5.130214531045759, |
| "kl": 0.017059326171875, |
| "learning_rate": 9.881818181818182e-07, |
| "loss": 0.0007, |
| "reward": 0.9124999940395355, |
| "reward_std": 0.6435409486293793, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.3656250089406967, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.859375, |
| "epoch": 0.038408779149519894, |
| "grad_norm": 6.98501961751934, |
| "kl": 0.01129150390625, |
| "learning_rate": 9.872727272727272e-07, |
| "loss": 0.0005, |
| "reward": 0.9640624523162842, |
| "reward_std": 0.645575761795044, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.41718751192092896, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.046875, |
| "epoch": 0.0411522633744856, |
| "grad_norm": 7.147359479240712, |
| "kl": 0.0162200927734375, |
| "learning_rate": 9.863636363636363e-07, |
| "loss": 0.0006, |
| "reward": 1.0695312023162842, |
| "reward_std": 0.820113867521286, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.46015626192092896, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.859375, |
| "epoch": 0.0438957475994513, |
| "grad_norm": 5.664219699067135, |
| "kl": 0.015106201171875, |
| "learning_rate": 9.854545454545455e-07, |
| "loss": 0.0006, |
| "reward": 1.0304687321186066, |
| "reward_std": 0.6156916320323944, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.43671874701976776, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.296875, |
| "epoch": 0.04663923182441701, |
| "grad_norm": 4.598762294986751, |
| "kl": 0.028656005859375, |
| "learning_rate": 9.845454545454545e-07, |
| "loss": 0.0011, |
| "reward": 1.1671874523162842, |
| "reward_std": 0.717985987663269, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.49531251192092896, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.125, |
| "epoch": 0.04938271604938271, |
| "grad_norm": 14.244035690799146, |
| "kl": 0.016326904296875, |
| "learning_rate": 9.836363636363637e-07, |
| "loss": 0.0007, |
| "reward": 0.9195311665534973, |
| "reward_std": 0.6659009456634521, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.3882812261581421, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.71875, |
| "epoch": 0.05212620027434842, |
| "grad_norm": 10.540110657265467, |
| "kl": 0.1883544921875, |
| "learning_rate": 9.827272727272728e-07, |
| "loss": 0.0075, |
| "reward": 0.7343749701976776, |
| "reward_std": 0.4525277465581894, |
| "rewards/choice_reward": 0.4375, |
| "rewards/roam_reward": 0.2968749701976776, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.5, |
| "epoch": 0.05486968449931413, |
| "grad_norm": 7.327457221273048, |
| "kl": 0.0169677734375, |
| "learning_rate": 9.818181818181818e-07, |
| "loss": 0.0007, |
| "reward": 0.9648436903953552, |
| "reward_std": 0.7674175798892975, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.4335937350988388, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.875, |
| "epoch": 0.05761316872427984, |
| "grad_norm": 22.457323451419064, |
| "kl": 0.02655029296875, |
| "learning_rate": 9.80909090909091e-07, |
| "loss": 0.0011, |
| "reward": 0.9695312082767487, |
| "reward_std": 0.8863083720207214, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.43828125298023224, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.375, |
| "epoch": 0.06035665294924554, |
| "grad_norm": 5.7571881626029295, |
| "kl": 0.0379638671875, |
| "learning_rate": 9.8e-07, |
| "loss": 0.0015, |
| "reward": 1.309374988079071, |
| "reward_std": 0.54523104429245, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.543749988079071, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.859375, |
| "epoch": 0.06310013717421124, |
| "grad_norm": 11.09390596331488, |
| "kl": 0.0340576171875, |
| "learning_rate": 9.790909090909091e-07, |
| "loss": 0.0014, |
| "reward": 0.9507811963558197, |
| "reward_std": 0.604695588350296, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.4195312559604645, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.796875, |
| "epoch": 0.06584362139917696, |
| "grad_norm": 31.58108805295004, |
| "kl": 0.0621337890625, |
| "learning_rate": 9.78181818181818e-07, |
| "loss": 0.0025, |
| "reward": 1.024999976158142, |
| "reward_std": 0.6390542387962341, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.4625000059604645, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.46875, |
| "epoch": 0.06858710562414266, |
| "grad_norm": 11.336338033820835, |
| "kl": 0.034423828125, |
| "learning_rate": 9.772727272727273e-07, |
| "loss": 0.0014, |
| "reward": 0.6968749463558197, |
| "reward_std": 0.5415091067552567, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.3062499910593033, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.71875, |
| "epoch": 0.07133058984910837, |
| "grad_norm": 8.12783631250481, |
| "kl": 0.042724609375, |
| "learning_rate": 9.763636363636362e-07, |
| "loss": 0.0017, |
| "reward": 0.9734374582767487, |
| "reward_std": 0.7181236445903778, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.42656248807907104, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.8125, |
| "epoch": 0.07407407407407407, |
| "grad_norm": 4.5322020786066854, |
| "kl": 0.03717041015625, |
| "learning_rate": 9.754545454545454e-07, |
| "loss": 0.0015, |
| "reward": 0.8960937261581421, |
| "reward_std": 0.6352558135986328, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.3960937261581421, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.875, |
| "epoch": 0.07681755829903979, |
| "grad_norm": 6.2617401945392, |
| "kl": 0.0208740234375, |
| "learning_rate": 9.745454545454544e-07, |
| "loss": 0.0008, |
| "reward": 1.333593726158142, |
| "reward_std": 0.668065220117569, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.5523437559604645, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.734375, |
| "epoch": 0.07956104252400549, |
| "grad_norm": 4.8442832275116245, |
| "kl": 0.042236328125, |
| "learning_rate": 9.736363636363635e-07, |
| "loss": 0.0017, |
| "reward": 1.3671874403953552, |
| "reward_std": 0.7418243885040283, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.6015625, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.75, |
| "epoch": 0.0823045267489712, |
| "grad_norm": 4.996777022694269, |
| "kl": 0.01519775390625, |
| "learning_rate": 9.727272727272727e-07, |
| "loss": 0.0006, |
| "reward": 0.9710937142372131, |
| "reward_std": 0.5648946315050125, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.4085937589406967, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.796875, |
| "epoch": 0.0850480109739369, |
| "grad_norm": 6.927741488328218, |
| "kl": 0.0390625, |
| "learning_rate": 9.718181818181817e-07, |
| "loss": 0.0016, |
| "reward": 0.8914062678813934, |
| "reward_std": 0.7767362892627716, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.40703123807907104, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.96875, |
| "epoch": 0.0877914951989026, |
| "grad_norm": 21.37164743991267, |
| "kl": 0.02935791015625, |
| "learning_rate": 9.709090909090909e-07, |
| "loss": 0.0012, |
| "reward": 1.427343726158142, |
| "reward_std": 0.5377233773469925, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.6148437559604645, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.890625, |
| "epoch": 0.09053497942386832, |
| "grad_norm": 6.229072965781938, |
| "kl": 0.0120697021484375, |
| "learning_rate": 9.7e-07, |
| "loss": 0.0005, |
| "reward": 1.2453124523162842, |
| "reward_std": 0.5992177128791809, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.557812511920929, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.71875, |
| "epoch": 0.09327846364883402, |
| "grad_norm": 19.471019529635235, |
| "kl": 0.00970458984375, |
| "learning_rate": 9.69090909090909e-07, |
| "loss": 0.0004, |
| "reward": 0.9453125, |
| "reward_std": 0.5610494464635849, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.3984375, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.25, |
| "epoch": 0.09602194787379972, |
| "grad_norm": 3.510572314601594, |
| "kl": 0.015655517578125, |
| "learning_rate": 9.681818181818182e-07, |
| "loss": 0.0006, |
| "reward": 1.1757811903953552, |
| "reward_std": 0.34083548188209534, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.4882812649011612, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.921875, |
| "epoch": 0.09876543209876543, |
| "grad_norm": 3.6755713799066623, |
| "kl": 0.02471923828125, |
| "learning_rate": 9.672727272727271e-07, |
| "loss": 0.001, |
| "reward": 1.1796874403953552, |
| "reward_std": 0.4743943512439728, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.5234375149011612, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.1875, |
| "epoch": 0.10150891632373114, |
| "grad_norm": 5.910609278279588, |
| "kl": 0.0098876953125, |
| "learning_rate": 9.663636363636363e-07, |
| "loss": 0.0004, |
| "reward": 1.034375011920929, |
| "reward_std": 0.7065398991107941, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.45624999701976776, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.953125, |
| "epoch": 0.10425240054869685, |
| "grad_norm": 5.3764744844672006, |
| "kl": 0.02191162109375, |
| "learning_rate": 9.654545454545455e-07, |
| "loss": 0.0009, |
| "reward": 1.2164061665534973, |
| "reward_std": 0.5084110051393509, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.5289062708616257, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.28125, |
| "epoch": 0.10699588477366255, |
| "grad_norm": 4.693501392661649, |
| "kl": 0.071197509765625, |
| "learning_rate": 9.645454545454545e-07, |
| "loss": 0.0029, |
| "reward": 0.9593749642372131, |
| "reward_std": 0.2084776610136032, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.4437499940395355, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.203125, |
| "epoch": 0.10973936899862825, |
| "grad_norm": 5.331160500435366, |
| "kl": 0.062744140625, |
| "learning_rate": 9.636363636363636e-07, |
| "loss": 0.0025, |
| "reward": 0.9445311427116394, |
| "reward_std": 0.6358062028884888, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.42890623211860657, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.03125, |
| "epoch": 0.11248285322359397, |
| "grad_norm": 6.95362416315892, |
| "kl": 0.019287109375, |
| "learning_rate": 9.627272727272728e-07, |
| "loss": 0.0008, |
| "reward": 0.9070312082767487, |
| "reward_std": 0.6886500120162964, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.39140623807907104, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.703125, |
| "epoch": 0.11522633744855967, |
| "grad_norm": 3.3675354564501534, |
| "kl": 0.010711669921875, |
| "learning_rate": 9.618181818181818e-07, |
| "loss": 0.0004, |
| "reward": 1.071874976158142, |
| "reward_std": 0.29209593310952187, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.4624999910593033, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.671875, |
| "epoch": 0.11796982167352538, |
| "grad_norm": 3.9867453434532205, |
| "kl": 0.0181884765625, |
| "learning_rate": 9.60909090909091e-07, |
| "loss": 0.0007, |
| "reward": 0.7265624403953552, |
| "reward_std": 0.589001476764679, |
| "rewards/choice_reward": 0.40625, |
| "rewards/roam_reward": 0.3203124776482582, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.015625, |
| "epoch": 0.12071330589849108, |
| "grad_norm": 4.903322287174992, |
| "kl": 0.0218505859375, |
| "learning_rate": 9.6e-07, |
| "loss": 0.0009, |
| "reward": 1.0820311903953552, |
| "reward_std": 0.5496466904878616, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.48828125, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.390625, |
| "epoch": 0.12345679012345678, |
| "grad_norm": 8.089264703469837, |
| "kl": 0.018707275390625, |
| "learning_rate": 9.59090909090909e-07, |
| "loss": 0.0007, |
| "reward": 0.8726561665534973, |
| "reward_std": 0.47763732075691223, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.3726562410593033, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.015625, |
| "epoch": 0.1262002743484225, |
| "grad_norm": 9.349089691780627, |
| "kl": 0.02105712890625, |
| "learning_rate": 9.58181818181818e-07, |
| "loss": 0.0008, |
| "reward": 1.414843738079071, |
| "reward_std": 0.604925662279129, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.6179687678813934, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.015625, |
| "epoch": 0.1289437585733882, |
| "grad_norm": 308.00424029568256, |
| "kl": 0.018798828125, |
| "learning_rate": 9.572727272727272e-07, |
| "loss": 0.0008, |
| "reward": 1.1007811725139618, |
| "reward_std": 0.3999259024858475, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.49140626192092896, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.0, |
| "epoch": 0.13168724279835392, |
| "grad_norm": 4.176206112067573, |
| "kl": 0.01495361328125, |
| "learning_rate": 9.563636363636364e-07, |
| "loss": 0.0006, |
| "reward": 1.0499999523162842, |
| "reward_std": 0.45452696830034256, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.48749999701976776, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.34375, |
| "epoch": 0.13443072702331962, |
| "grad_norm": 8.442730409761806, |
| "kl": 0.0172119140625, |
| "learning_rate": 9.554545454545454e-07, |
| "loss": 0.0007, |
| "reward": 1.2593749165534973, |
| "reward_std": 0.5093583762645721, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.5562500208616257, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.3125, |
| "epoch": 0.13717421124828533, |
| "grad_norm": 7.629286049055737, |
| "kl": 0.016357421875, |
| "learning_rate": 9.545454545454546e-07, |
| "loss": 0.0007, |
| "reward": 1.1859374642372131, |
| "reward_std": 0.5876812934875488, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.5296874791383743, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.734375, |
| "epoch": 0.13991769547325103, |
| "grad_norm": 4.772489927214594, |
| "kl": 0.02093505859375, |
| "learning_rate": 9.536363636363636e-07, |
| "loss": 0.0008, |
| "reward": 1.1367187201976776, |
| "reward_std": 0.3796752989292145, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.4804687350988388, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.28125, |
| "epoch": 0.14266117969821673, |
| "grad_norm": 5.50996340780448, |
| "kl": 0.01727294921875, |
| "learning_rate": 9.527272727272727e-07, |
| "loss": 0.0007, |
| "reward": 0.9468749761581421, |
| "reward_std": 0.7070161998271942, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.4000000059604645, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.578125, |
| "epoch": 0.14540466392318244, |
| "grad_norm": 12.007296704335586, |
| "kl": 0.041259765625, |
| "learning_rate": 9.518181818181818e-07, |
| "loss": 0.0016, |
| "reward": 1.2179687023162842, |
| "reward_std": 0.5313120782375336, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.5617187768220901, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.375, |
| "epoch": 0.14814814814814814, |
| "grad_norm": 22.29430069726113, |
| "kl": 0.0205078125, |
| "learning_rate": 9.509090909090908e-07, |
| "loss": 0.0008, |
| "reward": 1.108593761920929, |
| "reward_std": 0.5563602149486542, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.48359373211860657, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.875, |
| "epoch": 0.15089163237311384, |
| "grad_norm": 7.735008497656213, |
| "kl": 0.013916015625, |
| "learning_rate": 9.499999999999999e-07, |
| "loss": 0.0006, |
| "reward": 0.9749999940395355, |
| "reward_std": 0.6277340650558472, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.4281249940395355, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.78125, |
| "epoch": 0.15363511659807957, |
| "grad_norm": 6.050105779999899, |
| "kl": 0.0223388671875, |
| "learning_rate": 9.490909090909091e-07, |
| "loss": 0.0009, |
| "reward": 0.9632812291383743, |
| "reward_std": 0.2669573575258255, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.3695312552154064, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.609375, |
| "epoch": 0.15637860082304528, |
| "grad_norm": 4.091151241083613, |
| "kl": 0.0247802734375, |
| "learning_rate": 9.481818181818182e-07, |
| "loss": 0.001, |
| "reward": 0.9515624642372131, |
| "reward_std": 0.4127080887556076, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.45156247913837433, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.0, |
| "epoch": 0.15912208504801098, |
| "grad_norm": 3.6710308570080694, |
| "kl": 0.0223388671875, |
| "learning_rate": 9.472727272727272e-07, |
| "loss": 0.0009, |
| "reward": 1.506250023841858, |
| "reward_std": 0.5399478673934937, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.6625000536441803, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.3125, |
| "epoch": 0.16186556927297668, |
| "grad_norm": 4.478479008678768, |
| "kl": 0.0220947265625, |
| "learning_rate": 9.463636363636364e-07, |
| "loss": 0.0009, |
| "reward": 0.8124999403953552, |
| "reward_std": 0.522526353597641, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.3593749850988388, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.6875, |
| "epoch": 0.1646090534979424, |
| "grad_norm": 5.956354120549674, |
| "kl": 0.0487060546875, |
| "learning_rate": 9.454545454545454e-07, |
| "loss": 0.0019, |
| "reward": 1.021093726158142, |
| "reward_std": 0.68604376912117, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.4429687559604645, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.0625, |
| "epoch": 0.1673525377229081, |
| "grad_norm": 4.817665871751061, |
| "kl": 0.02001953125, |
| "learning_rate": 9.445454545454545e-07, |
| "loss": 0.0008, |
| "reward": 1.3484374284744263, |
| "reward_std": 0.7283227443695068, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.5828125178813934, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.09375, |
| "epoch": 0.1700960219478738, |
| "grad_norm": 3.6242596404503176, |
| "kl": 0.0205078125, |
| "learning_rate": 9.436363636363636e-07, |
| "loss": 0.0008, |
| "reward": 1.244531273841858, |
| "reward_std": 0.46894803643226624, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.4789062440395355, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.53125, |
| "epoch": 0.1728395061728395, |
| "grad_norm": 3.7126985433932367, |
| "kl": 0.03619384765625, |
| "learning_rate": 9.427272727272727e-07, |
| "loss": 0.0014, |
| "reward": 0.8460937142372131, |
| "reward_std": 0.6942168772220612, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.37734372913837433, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.578125, |
| "epoch": 0.1755829903978052, |
| "grad_norm": 5.404480273297857, |
| "kl": 0.020751953125, |
| "learning_rate": 9.418181818181818e-07, |
| "loss": 0.0008, |
| "reward": 1.407031238079071, |
| "reward_std": 0.3813091591000557, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.5945312678813934, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.5625, |
| "epoch": 0.17832647462277093, |
| "grad_norm": 11.106716403601697, |
| "kl": 0.0284423828125, |
| "learning_rate": 9.409090909090908e-07, |
| "loss": 0.0011, |
| "reward": 1.239843726158142, |
| "reward_std": 0.2835831791162491, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.5523437559604645, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.671875, |
| "epoch": 0.18106995884773663, |
| "grad_norm": 5.339972405871843, |
| "kl": 0.02227783203125, |
| "learning_rate": 9.399999999999999e-07, |
| "loss": 0.0009, |
| "reward": 1.2054687142372131, |
| "reward_std": 0.4735162556171417, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.5648437142372131, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.078125, |
| "epoch": 0.18381344307270234, |
| "grad_norm": 6.59725887491622, |
| "kl": 0.026123046875, |
| "learning_rate": 9.390909090909091e-07, |
| "loss": 0.001, |
| "reward": 1.0601562857627869, |
| "reward_std": 0.5607233345508575, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.4351562559604645, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.171875, |
| "epoch": 0.18655692729766804, |
| "grad_norm": 5.148995458977506, |
| "kl": 0.0205078125, |
| "learning_rate": 9.381818181818181e-07, |
| "loss": 0.0008, |
| "reward": 1.079687476158142, |
| "reward_std": 0.67804354429245, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.4859375059604645, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.125, |
| "epoch": 0.18930041152263374, |
| "grad_norm": 5.44976353091056, |
| "kl": 0.02325439453125, |
| "learning_rate": 9.372727272727272e-07, |
| "loss": 0.0009, |
| "reward": 1.5710936784744263, |
| "reward_std": 0.525007963180542, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.680468738079071, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.5625, |
| "epoch": 0.19204389574759945, |
| "grad_norm": 4.727339965319695, |
| "kl": 0.0242919921875, |
| "learning_rate": 9.363636363636364e-07, |
| "loss": 0.001, |
| "reward": 1.239843726158142, |
| "reward_std": 0.559609979391098, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.5523437559604645, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.953125, |
| "epoch": 0.19478737997256515, |
| "grad_norm": 6.868511552604511, |
| "kl": 0.0240478515625, |
| "learning_rate": 9.354545454545454e-07, |
| "loss": 0.001, |
| "reward": 1.1156249642372131, |
| "reward_std": 0.7248922288417816, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.5062500387430191, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.625, |
| "epoch": 0.19753086419753085, |
| "grad_norm": 5.111425144500442, |
| "kl": 0.033935546875, |
| "learning_rate": 9.345454545454545e-07, |
| "loss": 0.0014, |
| "reward": 0.8828124701976776, |
| "reward_std": 0.5237676799297333, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.4140624701976776, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.84375, |
| "epoch": 0.20027434842249658, |
| "grad_norm": 6.375847552027292, |
| "kl": 0.0338134765625, |
| "learning_rate": 9.336363636363636e-07, |
| "loss": 0.0014, |
| "reward": 0.9273437261581421, |
| "reward_std": 0.6422051191329956, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.4273437112569809, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.59375, |
| "epoch": 0.2030178326474623, |
| "grad_norm": 6.3864872962971, |
| "kl": 0.034423828125, |
| "learning_rate": 9.327272727272727e-07, |
| "loss": 0.0014, |
| "reward": 1.5023437142372131, |
| "reward_std": 0.42026302218437195, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.6742187738418579, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.921875, |
| "epoch": 0.205761316872428, |
| "grad_norm": 4.324614987231985, |
| "kl": 0.02606201171875, |
| "learning_rate": 9.318181818181817e-07, |
| "loss": 0.001, |
| "reward": 0.7874999642372131, |
| "reward_std": 0.34173646569252014, |
| "rewards/choice_reward": 0.421875, |
| "rewards/roam_reward": 0.36562497913837433, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.46875, |
| "epoch": 0.2085048010973937, |
| "grad_norm": 7.876492304090726, |
| "kl": 0.02252197265625, |
| "learning_rate": 9.309090909090909e-07, |
| "loss": 0.0009, |
| "reward": 1.3250000476837158, |
| "reward_std": 0.6011227965354919, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.5593750178813934, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.9375, |
| "epoch": 0.2112482853223594, |
| "grad_norm": 8.904234886894526, |
| "kl": 0.04833984375, |
| "learning_rate": 9.3e-07, |
| "loss": 0.0019, |
| "reward": 1.142968773841858, |
| "reward_std": 0.41374167799949646, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.5335937440395355, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.234375, |
| "epoch": 0.2139917695473251, |
| "grad_norm": 5.0500367034566285, |
| "kl": 0.0286865234375, |
| "learning_rate": 9.290909090909091e-07, |
| "loss": 0.0011, |
| "reward": 1.395312488079071, |
| "reward_std": 0.5761922597885132, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.645312488079071, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.734375, |
| "epoch": 0.2167352537722908, |
| "grad_norm": 5.17109089115606, |
| "kl": 0.02337646484375, |
| "learning_rate": 9.281818181818181e-07, |
| "loss": 0.0009, |
| "reward": 1.2734374403953552, |
| "reward_std": 0.4153551012277603, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.5859375149011612, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.34375, |
| "epoch": 0.2194787379972565, |
| "grad_norm": 5.490569803700363, |
| "kl": 0.025634765625, |
| "learning_rate": 9.272727272727272e-07, |
| "loss": 0.001, |
| "reward": 0.8453124463558197, |
| "reward_std": 0.5321288704872131, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.37656253576278687, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.734375, |
| "epoch": 0.2222222222222222, |
| "grad_norm": 5.0754221806707305, |
| "kl": 0.04443359375, |
| "learning_rate": 9.263636363636364e-07, |
| "loss": 0.0018, |
| "reward": 1.7281250357627869, |
| "reward_std": 0.29361481964588165, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.7593750357627869, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.25, |
| "epoch": 0.22496570644718794, |
| "grad_norm": 3.7285408578284724, |
| "kl": 0.02777099609375, |
| "learning_rate": 9.254545454545453e-07, |
| "loss": 0.0011, |
| "reward": 1.30078125, |
| "reward_std": 0.35986968874931335, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.58203125, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.5625, |
| "epoch": 0.22770919067215364, |
| "grad_norm": 5.535866076818073, |
| "kl": 0.02117919921875, |
| "learning_rate": 9.245454545454545e-07, |
| "loss": 0.0008, |
| "reward": 1.393750011920929, |
| "reward_std": 0.2868022173643112, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.643750011920929, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.453125, |
| "epoch": 0.23045267489711935, |
| "grad_norm": 5.021842818824957, |
| "kl": 0.0277099609375, |
| "learning_rate": 9.236363636363636e-07, |
| "loss": 0.0011, |
| "reward": 1.263281226158142, |
| "reward_std": 0.47014766931533813, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.5601562559604645, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.34375, |
| "epoch": 0.23319615912208505, |
| "grad_norm": 5.028085915013138, |
| "kl": 0.02374267578125, |
| "learning_rate": 9.227272727272727e-07, |
| "loss": 0.001, |
| "reward": 0.977343738079071, |
| "reward_std": 0.3106689900159836, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.47734376788139343, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.203125, |
| "epoch": 0.23593964334705075, |
| "grad_norm": 14.449547520531839, |
| "kl": 0.03857421875, |
| "learning_rate": 9.218181818181818e-07, |
| "loss": 0.0015, |
| "reward": 0.8820312321186066, |
| "reward_std": 0.5157787501811981, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.42890623211860657, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.6875, |
| "epoch": 0.23868312757201646, |
| "grad_norm": 6.240824814588787, |
| "kl": 0.056884765625, |
| "learning_rate": 9.209090909090909e-07, |
| "loss": 0.0023, |
| "reward": 1.243749976158142, |
| "reward_std": 0.7348380982875824, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6031249761581421, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.0625, |
| "epoch": 0.24142661179698216, |
| "grad_norm": 5.99769488738683, |
| "kl": 0.0648193359375, |
| "learning_rate": 9.2e-07, |
| "loss": 0.0026, |
| "reward": 1.2578125, |
| "reward_std": 0.7988573312759399, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6015624701976776, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.890625, |
| "epoch": 0.24417009602194786, |
| "grad_norm": 4.495550953149352, |
| "kl": 0.0609130859375, |
| "learning_rate": 9.19090909090909e-07, |
| "loss": 0.0024, |
| "reward": 1.310937523841858, |
| "reward_std": 0.6118793487548828, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6234374940395355, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.203125, |
| "epoch": 0.24691358024691357, |
| "grad_norm": 14.959203165834163, |
| "kl": 0.0543212890625, |
| "learning_rate": 9.181818181818181e-07, |
| "loss": 0.0022, |
| "reward": 0.96875, |
| "reward_std": 0.5600316077470779, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.4843749701976776, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.140625, |
| "epoch": 0.2496570644718793, |
| "grad_norm": 8.605452241909484, |
| "kl": 0.0543212890625, |
| "learning_rate": 9.172727272727272e-07, |
| "loss": 0.0022, |
| "reward": 1.167187511920929, |
| "reward_std": 0.6275071799755096, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.5578124821186066, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.484375, |
| "epoch": 0.252400548696845, |
| "grad_norm": 12.223314100403433, |
| "kl": 0.0606689453125, |
| "learning_rate": 9.163636363636364e-07, |
| "loss": 0.0024, |
| "reward": 1.2749999165534973, |
| "reward_std": 0.6670045852661133, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6187500059604645, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.4375, |
| "epoch": 0.2551440329218107, |
| "grad_norm": 4.38653790036809, |
| "kl": 0.098876953125, |
| "learning_rate": 9.154545454545453e-07, |
| "loss": 0.004, |
| "reward": 1.211718738079071, |
| "reward_std": 0.4608733206987381, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.586718738079071, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.03125, |
| "epoch": 0.2578875171467764, |
| "grad_norm": 6.757753284366764, |
| "kl": 0.063232421875, |
| "learning_rate": 9.145454545454545e-07, |
| "loss": 0.0025, |
| "reward": 1.547656238079071, |
| "reward_std": 0.5729297697544098, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7507812082767487, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.5625, |
| "epoch": 0.2606310013717421, |
| "grad_norm": 5.077342693634887, |
| "kl": 0.0767822265625, |
| "learning_rate": 9.136363636363637e-07, |
| "loss": 0.0031, |
| "reward": 0.9843749701976776, |
| "reward_std": 0.7534768581390381, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.4843749701976776, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.34375, |
| "epoch": 0.26337448559670784, |
| "grad_norm": 4.650632802200209, |
| "kl": 0.0548095703125, |
| "learning_rate": 9.127272727272727e-07, |
| "loss": 0.0022, |
| "reward": 1.3523436784744263, |
| "reward_std": 0.809281975030899, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.617968738079071, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.46875, |
| "epoch": 0.2661179698216735, |
| "grad_norm": 10.070287917248892, |
| "kl": 0.06982421875, |
| "learning_rate": 9.118181818181818e-07, |
| "loss": 0.0028, |
| "reward": 1.1124999523162842, |
| "reward_std": 0.4468783587217331, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.5343749970197678, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.59375, |
| "epoch": 0.26886145404663925, |
| "grad_norm": 14.869262822828924, |
| "kl": 0.10107421875, |
| "learning_rate": 9.109090909090909e-07, |
| "loss": 0.004, |
| "reward": 1.078906238079071, |
| "reward_std": 0.8109665513038635, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5320312678813934, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.09375, |
| "epoch": 0.2716049382716049, |
| "grad_norm": 8.524354382622187, |
| "kl": 0.078857421875, |
| "learning_rate": 9.1e-07, |
| "loss": 0.0032, |
| "reward": 1.3359375, |
| "reward_std": 0.8216530680656433, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6640624701976776, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.390625, |
| "epoch": 0.27434842249657065, |
| "grad_norm": 4.866005700071, |
| "kl": 0.0703125, |
| "learning_rate": 9.09090909090909e-07, |
| "loss": 0.0028, |
| "reward": 1.3390624523162842, |
| "reward_std": 0.40313820540905, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6671874523162842, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.390625, |
| "epoch": 0.27709190672153633, |
| "grad_norm": 11.598271374848927, |
| "kl": 0.077880859375, |
| "learning_rate": 9.081818181818181e-07, |
| "loss": 0.0031, |
| "reward": 1.548437476158142, |
| "reward_std": 0.6047469079494476, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7359375357627869, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.796875, |
| "epoch": 0.27983539094650206, |
| "grad_norm": 4.238458907315475, |
| "kl": 0.0662841796875, |
| "learning_rate": 9.072727272727272e-07, |
| "loss": 0.0027, |
| "reward": 1.456250011920929, |
| "reward_std": 0.6197087466716766, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.706250011920929, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.484375, |
| "epoch": 0.2825788751714678, |
| "grad_norm": 4.893207308232037, |
| "kl": 0.0733642578125, |
| "learning_rate": 9.063636363636364e-07, |
| "loss": 0.0029, |
| "reward": 1.110937476158142, |
| "reward_std": 0.36215464770793915, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5484375059604645, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.453125, |
| "epoch": 0.28532235939643347, |
| "grad_norm": 3.444057641618053, |
| "kl": 0.056640625, |
| "learning_rate": 9.054545454545453e-07, |
| "loss": 0.0023, |
| "reward": 1.409375011920929, |
| "reward_std": 0.43059414625167847, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7062499821186066, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.0625, |
| "epoch": 0.2880658436213992, |
| "grad_norm": 6.563920276782772, |
| "kl": 0.05712890625, |
| "learning_rate": 9.045454545454545e-07, |
| "loss": 0.0023, |
| "reward": 1.3742187023162842, |
| "reward_std": 0.6756561994552612, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6867187321186066, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.109375, |
| "epoch": 0.2908093278463649, |
| "grad_norm": 5.004219209273814, |
| "kl": 0.0635986328125, |
| "learning_rate": 9.036363636363637e-07, |
| "loss": 0.0025, |
| "reward": 1.0578124970197678, |
| "reward_std": 0.543773353099823, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5421874970197678, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.84375, |
| "epoch": 0.2935528120713306, |
| "grad_norm": 5.006064034352587, |
| "kl": 0.0760498046875, |
| "learning_rate": 9.027272727272726e-07, |
| "loss": 0.003, |
| "reward": 1.364062488079071, |
| "reward_std": 0.4392424523830414, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6765624582767487, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.828125, |
| "epoch": 0.2962962962962963, |
| "grad_norm": 6.230846226086625, |
| "kl": 0.07080078125, |
| "learning_rate": 9.018181818181818e-07, |
| "loss": 0.0028, |
| "reward": 1.2906250357627869, |
| "reward_std": 0.7027235329151154, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6343750059604645, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.078125, |
| "epoch": 0.299039780521262, |
| "grad_norm": 17.840305616448358, |
| "kl": 0.0703125, |
| "learning_rate": 9.009090909090909e-07, |
| "loss": 0.0028, |
| "reward": 1.500781238079071, |
| "reward_std": 0.6326878219842911, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.735156238079071, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.46875, |
| "epoch": 0.3017832647462277, |
| "grad_norm": 4.0753012242949564, |
| "kl": 0.0653076171875, |
| "learning_rate": 9e-07, |
| "loss": 0.0026, |
| "reward": 0.8124999701976776, |
| "reward_std": 0.8824117183685303, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.421875, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.328125, |
| "epoch": 0.3045267489711934, |
| "grad_norm": 3.228177073766033, |
| "kl": 0.06787109375, |
| "learning_rate": 8.99090909090909e-07, |
| "loss": 0.0027, |
| "reward": 1.228906273841858, |
| "reward_std": 0.5324103534221649, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6195312738418579, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.828125, |
| "epoch": 0.30727023319615915, |
| "grad_norm": 4.50841039714633, |
| "kl": 0.06982421875, |
| "learning_rate": 8.981818181818181e-07, |
| "loss": 0.0028, |
| "reward": 1.30859375, |
| "reward_std": 0.7017756104469299, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6523437201976776, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.5, |
| "epoch": 0.3100137174211248, |
| "grad_norm": 4.325246595315257, |
| "kl": 0.09228515625, |
| "learning_rate": 8.972727272727272e-07, |
| "loss": 0.0037, |
| "reward": 1.007031261920929, |
| "reward_std": 0.5978388488292694, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.49140623211860657, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.34375, |
| "epoch": 0.31275720164609055, |
| "grad_norm": 4.994040116821786, |
| "kl": 0.080078125, |
| "learning_rate": 8.963636363636363e-07, |
| "loss": 0.0032, |
| "reward": 1.610156238079071, |
| "reward_std": 0.3702356815338135, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.797656238079071, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.765625, |
| "epoch": 0.31550068587105623, |
| "grad_norm": 3.2675458624595204, |
| "kl": 0.087890625, |
| "learning_rate": 8.954545454545454e-07, |
| "loss": 0.0035, |
| "reward": 1.7054687142372131, |
| "reward_std": 0.3386332616209984, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8460937440395355, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.21875, |
| "epoch": 0.31824417009602196, |
| "grad_norm": 6.459049917145571, |
| "kl": 0.0908203125, |
| "learning_rate": 8.945454545454545e-07, |
| "loss": 0.0036, |
| "reward": 0.7921874821186066, |
| "reward_std": 0.6447290182113647, |
| "rewards/choice_reward": 0.375, |
| "rewards/roam_reward": 0.41718748211860657, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.359375, |
| "epoch": 0.32098765432098764, |
| "grad_norm": 3.464024839590293, |
| "kl": 0.07177734375, |
| "learning_rate": 8.936363636363637e-07, |
| "loss": 0.0029, |
| "reward": 1.1210937798023224, |
| "reward_std": 0.46119415760040283, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5742186903953552, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.84375, |
| "epoch": 0.32373113854595337, |
| "grad_norm": 3.787838016059412, |
| "kl": 0.0732421875, |
| "learning_rate": 8.927272727272726e-07, |
| "loss": 0.0029, |
| "reward": 1.0484374463558197, |
| "reward_std": 0.6449593305587769, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5328124761581421, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.796875, |
| "epoch": 0.32647462277091904, |
| "grad_norm": 4.158322704150245, |
| "kl": 0.10693359375, |
| "learning_rate": 8.918181818181818e-07, |
| "loss": 0.0043, |
| "reward": 0.9898437261581421, |
| "reward_std": 0.6245564520359039, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.5054687261581421, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.953125, |
| "epoch": 0.3292181069958848, |
| "grad_norm": 5.328842138867986, |
| "kl": 0.082275390625, |
| "learning_rate": 8.909090909090909e-07, |
| "loss": 0.0033, |
| "reward": 1.060937523841858, |
| "reward_std": 0.774547666311264, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5140624493360519, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.25, |
| "epoch": 0.3319615912208505, |
| "grad_norm": 4.398455266312539, |
| "kl": 0.074462890625, |
| "learning_rate": 8.9e-07, |
| "loss": 0.003, |
| "reward": 1.529687523841858, |
| "reward_std": 0.6608569622039795, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7484374642372131, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.09375, |
| "epoch": 0.3347050754458162, |
| "grad_norm": 3.6986348942117826, |
| "kl": 0.0673828125, |
| "learning_rate": 8.89090909090909e-07, |
| "loss": 0.0027, |
| "reward": 1.6007813215255737, |
| "reward_std": 0.6274507939815521, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7882812321186066, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.109375, |
| "epoch": 0.3374485596707819, |
| "grad_norm": 5.404402499390665, |
| "kl": 0.0672607421875, |
| "learning_rate": 8.881818181818181e-07, |
| "loss": 0.0027, |
| "reward": 1.2890624403953552, |
| "reward_std": 0.6910946071147919, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6484375, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.359375, |
| "epoch": 0.3401920438957476, |
| "grad_norm": 6.0013254416888335, |
| "kl": 0.07373046875, |
| "learning_rate": 8.872727272727273e-07, |
| "loss": 0.003, |
| "reward": 0.9984374940395355, |
| "reward_std": 0.695489376783371, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.5140624642372131, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.09375, |
| "epoch": 0.3429355281207133, |
| "grad_norm": 3.900033333400682, |
| "kl": 0.06494140625, |
| "learning_rate": 8.863636363636363e-07, |
| "loss": 0.0026, |
| "reward": 1.207812488079071, |
| "reward_std": 0.29828448593616486, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.582812488079071, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.203125, |
| "epoch": 0.345679012345679, |
| "grad_norm": 4.849697720686882, |
| "kl": 0.0731201171875, |
| "learning_rate": 8.854545454545454e-07, |
| "loss": 0.0029, |
| "reward": 1.1765624284744263, |
| "reward_std": 0.7504218816757202, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.5828124582767487, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.546875, |
| "epoch": 0.3484224965706447, |
| "grad_norm": 3.7293956851296635, |
| "kl": 0.080810546875, |
| "learning_rate": 8.845454545454545e-07, |
| "loss": 0.0032, |
| "reward": 1.224218726158142, |
| "reward_std": 0.45756466686725616, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6148437559604645, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.640625, |
| "epoch": 0.3511659807956104, |
| "grad_norm": 4.20915141452958, |
| "kl": 0.0579833984375, |
| "learning_rate": 8.836363636363637e-07, |
| "loss": 0.0023, |
| "reward": 1.764062523841858, |
| "reward_std": 0.4473951756954193, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8734375238418579, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.71875, |
| "epoch": 0.35390946502057613, |
| "grad_norm": 3.3123008804712497, |
| "kl": 0.0567626953125, |
| "learning_rate": 8.827272727272726e-07, |
| "loss": 0.0023, |
| "reward": 1.3398438096046448, |
| "reward_std": 0.5563405156135559, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6679687201976776, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.578125, |
| "epoch": 0.35665294924554186, |
| "grad_norm": 4.994418127246765, |
| "kl": 0.0604248046875, |
| "learning_rate": 8.818181818181818e-07, |
| "loss": 0.0024, |
| "reward": 1.596093773841858, |
| "reward_std": 0.5808419585227966, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7992187440395355, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.28125, |
| "epoch": 0.35939643347050754, |
| "grad_norm": 3.2360790785258327, |
| "kl": 0.063720703125, |
| "learning_rate": 8.809090909090909e-07, |
| "loss": 0.0026, |
| "reward": 1.8734374642372131, |
| "reward_std": 0.25403610337525606, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9203124642372131, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.21875, |
| "epoch": 0.36213991769547327, |
| "grad_norm": 7.248695098065959, |
| "kl": 0.078125, |
| "learning_rate": 8.799999999999999e-07, |
| "loss": 0.0031, |
| "reward": 1.436718761920929, |
| "reward_std": 0.6495934128761292, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.717968761920929, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.265625, |
| "epoch": 0.36488340192043894, |
| "grad_norm": 8.378430132685443, |
| "kl": 0.064697265625, |
| "learning_rate": 8.790909090909091e-07, |
| "loss": 0.0026, |
| "reward": 1.046093761920929, |
| "reward_std": 0.9277720153331757, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5304687172174454, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.78125, |
| "epoch": 0.3676268861454047, |
| "grad_norm": 3.1589997148498195, |
| "kl": 0.0614013671875, |
| "learning_rate": 8.781818181818182e-07, |
| "loss": 0.0025, |
| "reward": 1.125, |
| "reward_std": 0.5783344358205795, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5781249701976776, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.171875, |
| "epoch": 0.37037037037037035, |
| "grad_norm": 5.042679339537896, |
| "kl": 0.0548095703125, |
| "learning_rate": 8.772727272727273e-07, |
| "loss": 0.0022, |
| "reward": 1.153124988079071, |
| "reward_std": 0.7899528741836548, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.590624988079071, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.015625, |
| "epoch": 0.3731138545953361, |
| "grad_norm": 4.251656339164905, |
| "kl": 0.055908203125, |
| "learning_rate": 8.763636363636363e-07, |
| "loss": 0.0022, |
| "reward": 1.579687476158142, |
| "reward_std": 0.4745950400829315, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7828125357627869, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.671875, |
| "epoch": 0.37585733882030176, |
| "grad_norm": 4.776801386893953, |
| "kl": 0.073486328125, |
| "learning_rate": 8.754545454545454e-07, |
| "loss": 0.0029, |
| "reward": 1.482812523841858, |
| "reward_std": 0.49166353046894073, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7328124940395355, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.53125, |
| "epoch": 0.3786008230452675, |
| "grad_norm": 5.061327970873458, |
| "kl": 0.066162109375, |
| "learning_rate": 8.745454545454545e-07, |
| "loss": 0.0027, |
| "reward": 1.675000011920929, |
| "reward_std": 0.44719746708869934, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.831250011920929, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.75, |
| "epoch": 0.3813443072702332, |
| "grad_norm": 3.6735769907735163, |
| "kl": 0.0628662109375, |
| "learning_rate": 8.736363636363636e-07, |
| "loss": 0.0025, |
| "reward": 1.249218761920929, |
| "reward_std": 0.6481688320636749, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6398437321186066, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.796875, |
| "epoch": 0.3840877914951989, |
| "grad_norm": 2.6505790004359864, |
| "kl": 0.060546875, |
| "learning_rate": 8.727272727272726e-07, |
| "loss": 0.0024, |
| "reward": 1.449999988079071, |
| "reward_std": 0.4319717437028885, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7156250178813934, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.03125, |
| "epoch": 0.3868312757201646, |
| "grad_norm": 4.147421746031423, |
| "kl": 0.0592041015625, |
| "learning_rate": 8.718181818181818e-07, |
| "loss": 0.0024, |
| "reward": 1.5, |
| "reward_std": 0.3329978957772255, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.75, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.5625, |
| "epoch": 0.3895747599451303, |
| "grad_norm": 10.931878070060389, |
| "kl": 0.07763671875, |
| "learning_rate": 8.709090909090909e-07, |
| "loss": 0.0031, |
| "reward": 1.0132812559604645, |
| "reward_std": 0.6477212905883789, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.5289062112569809, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.390625, |
| "epoch": 0.39231824417009603, |
| "grad_norm": 4.543133727421236, |
| "kl": 0.078125, |
| "learning_rate": 8.699999999999999e-07, |
| "loss": 0.0031, |
| "reward": 1.245312511920929, |
| "reward_std": 0.5095339268445969, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6359374821186066, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.5, |
| "epoch": 0.3950617283950617, |
| "grad_norm": 6.799154945627812, |
| "kl": 0.06591796875, |
| "learning_rate": 8.690909090909091e-07, |
| "loss": 0.0026, |
| "reward": 1.2093749642372131, |
| "reward_std": 0.7542276382446289, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6156249940395355, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.890625, |
| "epoch": 0.39780521262002744, |
| "grad_norm": 21.830417679362668, |
| "kl": 0.067626953125, |
| "learning_rate": 8.681818181818182e-07, |
| "loss": 0.0027, |
| "reward": 0.8671875, |
| "reward_std": 0.6834112703800201, |
| "rewards/choice_reward": 0.4375, |
| "rewards/roam_reward": 0.4296874850988388, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.5, |
| "epoch": 0.40054869684499317, |
| "grad_norm": 4.962962840054633, |
| "kl": 0.090087890625, |
| "learning_rate": 8.672727272727272e-07, |
| "loss": 0.0036, |
| "reward": 1.069531261920929, |
| "reward_std": 0.42628343403339386, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5539062321186066, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.59375, |
| "epoch": 0.40329218106995884, |
| "grad_norm": 3.1575892724965473, |
| "kl": 0.066650390625, |
| "learning_rate": 8.663636363636363e-07, |
| "loss": 0.0027, |
| "reward": 1.569531261920929, |
| "reward_std": 0.49635447561740875, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.772656261920929, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.828125, |
| "epoch": 0.4060356652949246, |
| "grad_norm": 6.6179935670184085, |
| "kl": 0.069580078125, |
| "learning_rate": 8.654545454545454e-07, |
| "loss": 0.0028, |
| "reward": 1.360937476158142, |
| "reward_std": 0.4443059712648392, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6890624463558197, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.734375, |
| "epoch": 0.40877914951989025, |
| "grad_norm": 5.892963936248546, |
| "kl": 0.070556640625, |
| "learning_rate": 8.645454545454545e-07, |
| "loss": 0.0028, |
| "reward": 1.543749988079071, |
| "reward_std": 0.3481606990098953, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7625000178813934, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.328125, |
| "epoch": 0.411522633744856, |
| "grad_norm": 6.263730759309839, |
| "kl": 0.081787109375, |
| "learning_rate": 8.636363636363636e-07, |
| "loss": 0.0033, |
| "reward": 1.300000011920929, |
| "reward_std": 0.7723948359489441, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6437499821186066, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.96875, |
| "epoch": 0.41426611796982166, |
| "grad_norm": 4.024542032878697, |
| "kl": 0.147705078125, |
| "learning_rate": 8.627272727272726e-07, |
| "loss": 0.0059, |
| "reward": 1.3515625, |
| "reward_std": 0.3730863481760025, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6796874701976776, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.359375, |
| "epoch": 0.4170096021947874, |
| "grad_norm": 5.399045676631981, |
| "kl": 0.05712890625, |
| "learning_rate": 8.618181818181818e-07, |
| "loss": 0.0023, |
| "reward": 0.90234375, |
| "reward_std": 0.39868150651454926, |
| "rewards/choice_reward": 0.4375, |
| "rewards/roam_reward": 0.46484375, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.03125, |
| "epoch": 0.41975308641975306, |
| "grad_norm": 5.7191847832606, |
| "kl": 0.06103515625, |
| "learning_rate": 8.60909090909091e-07, |
| "loss": 0.0024, |
| "reward": 1.348437488079071, |
| "reward_std": 0.35833945870399475, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6765624582767487, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.1875, |
| "epoch": 0.4224965706447188, |
| "grad_norm": 2.661374077804329, |
| "kl": 0.06787109375, |
| "learning_rate": 8.599999999999999e-07, |
| "loss": 0.0027, |
| "reward": 1.7312500476837158, |
| "reward_std": 0.04797553177922964, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8562500178813934, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.015625, |
| "epoch": 0.4252400548696845, |
| "grad_norm": 3.9622328522717303, |
| "kl": 0.060302734375, |
| "learning_rate": 8.590909090909091e-07, |
| "loss": 0.0024, |
| "reward": 1.4890624284744263, |
| "reward_std": 0.5478314012289047, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7390625178813934, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.671875, |
| "epoch": 0.4279835390946502, |
| "grad_norm": 3.273955933162255, |
| "kl": 0.0582275390625, |
| "learning_rate": 8.581818181818182e-07, |
| "loss": 0.0023, |
| "reward": 1.2171874642372131, |
| "reward_std": 0.6891123056411743, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6078124940395355, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.984375, |
| "epoch": 0.43072702331961593, |
| "grad_norm": 4.7893666691359345, |
| "kl": 0.0482177734375, |
| "learning_rate": 8.572727272727272e-07, |
| "loss": 0.0019, |
| "reward": 1.54296875, |
| "reward_std": 0.5729685127735138, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.73046875, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.75, |
| "epoch": 0.4334705075445816, |
| "grad_norm": 5.119948994612539, |
| "kl": 0.087890625, |
| "learning_rate": 8.563636363636363e-07, |
| "loss": 0.0035, |
| "reward": 1.5859375, |
| "reward_std": 0.35503947734832764, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7734374701976776, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.140625, |
| "epoch": 0.43621399176954734, |
| "grad_norm": 5.3369385279264705, |
| "kl": 0.052978515625, |
| "learning_rate": 8.554545454545454e-07, |
| "loss": 0.0021, |
| "reward": 1.34375, |
| "reward_std": 0.7487562596797943, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.671875, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.578125, |
| "epoch": 0.438957475994513, |
| "grad_norm": 6.963922269114714, |
| "kl": 0.0548095703125, |
| "learning_rate": 8.545454545454544e-07, |
| "loss": 0.0022, |
| "reward": 1.1132811903953552, |
| "reward_std": 0.5710795968770981, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.51953125, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.125, |
| "epoch": 0.44170096021947874, |
| "grad_norm": 5.0024696069552075, |
| "kl": 0.059326171875, |
| "learning_rate": 8.536363636363636e-07, |
| "loss": 0.0024, |
| "reward": 1.1171875596046448, |
| "reward_std": 0.5813678801059723, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5546875, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.765625, |
| "epoch": 0.4444444444444444, |
| "grad_norm": 6.863926677541896, |
| "kl": 0.072265625, |
| "learning_rate": 8.527272727272727e-07, |
| "loss": 0.0029, |
| "reward": 1.3398436903953552, |
| "reward_std": 0.45532071590423584, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.65234375, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.34375, |
| "epoch": 0.44718792866941015, |
| "grad_norm": 4.309835209354842, |
| "kl": 0.062255859375, |
| "learning_rate": 8.518181818181818e-07, |
| "loss": 0.0025, |
| "reward": 1.2171874642372131, |
| "reward_std": 0.5791269540786743, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6078124940395355, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.21875, |
| "epoch": 0.4499314128943759, |
| "grad_norm": 4.814305223557731, |
| "kl": 0.0640869140625, |
| "learning_rate": 8.509090909090909e-07, |
| "loss": 0.0026, |
| "reward": 1.4398437142372131, |
| "reward_std": 0.19878476113080978, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7210937440395355, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.03125, |
| "epoch": 0.45267489711934156, |
| "grad_norm": 4.324305104508143, |
| "kl": 0.066650390625, |
| "learning_rate": 8.499999999999999e-07, |
| "loss": 0.0027, |
| "reward": 1.706250011920929, |
| "reward_std": 0.45024340599775314, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.831250011920929, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.25, |
| "epoch": 0.4554183813443073, |
| "grad_norm": 12.515244809329793, |
| "kl": 0.081298828125, |
| "learning_rate": 8.490909090909091e-07, |
| "loss": 0.0033, |
| "reward": 1.435156226158142, |
| "reward_std": 0.5398507863283157, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7007812261581421, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.1875, |
| "epoch": 0.45816186556927296, |
| "grad_norm": 8.8304789346465, |
| "kl": 0.062255859375, |
| "learning_rate": 8.481818181818182e-07, |
| "loss": 0.0025, |
| "reward": 1.694531261920929, |
| "reward_std": 0.6171333193778992, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8195312321186066, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.53125, |
| "epoch": 0.4609053497942387, |
| "grad_norm": 5.0298882453907705, |
| "kl": 0.064697265625, |
| "learning_rate": 8.472727272727272e-07, |
| "loss": 0.0026, |
| "reward": 1.306249976158142, |
| "reward_std": 0.5636239349842072, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6343749463558197, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.1875, |
| "epoch": 0.46364883401920437, |
| "grad_norm": 6.918591227186971, |
| "kl": 0.053955078125, |
| "learning_rate": 8.463636363636363e-07, |
| "loss": 0.0022, |
| "reward": 1.1968750655651093, |
| "reward_std": 0.3086411654949188, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6031249761581421, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.078125, |
| "epoch": 0.4663923182441701, |
| "grad_norm": 10.229800465554694, |
| "kl": 0.0631103515625, |
| "learning_rate": 8.454545454545454e-07, |
| "loss": 0.0025, |
| "reward": 1.167187511920929, |
| "reward_std": 0.40548501908779144, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.557812511920929, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.109375, |
| "epoch": 0.4691358024691358, |
| "grad_norm": 4.895522138743598, |
| "kl": 0.0555419921875, |
| "learning_rate": 8.445454545454545e-07, |
| "loss": 0.0022, |
| "reward": 0.9687499701976776, |
| "reward_std": 0.36868916451931, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.4843749850988388, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.703125, |
| "epoch": 0.4718792866941015, |
| "grad_norm": 8.387517289112681, |
| "kl": 0.05908203125, |
| "learning_rate": 8.436363636363636e-07, |
| "loss": 0.0024, |
| "reward": 1.2609374523162842, |
| "reward_std": 0.47968146204948425, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.620312511920929, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.03125, |
| "epoch": 0.47462277091906724, |
| "grad_norm": 8.177680301301846, |
| "kl": 0.0472412109375, |
| "learning_rate": 8.427272727272727e-07, |
| "loss": 0.0019, |
| "reward": 1.29296875, |
| "reward_std": 0.6639520227909088, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6523436903953552, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.03125, |
| "epoch": 0.4773662551440329, |
| "grad_norm": 4.238237025234008, |
| "kl": 0.06640625, |
| "learning_rate": 8.418181818181818e-07, |
| "loss": 0.0027, |
| "reward": 1.607031226158142, |
| "reward_std": 0.2946796417236328, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7789062559604645, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.515625, |
| "epoch": 0.48010973936899864, |
| "grad_norm": 9.88349820542714, |
| "kl": 0.0687255859375, |
| "learning_rate": 8.409090909090909e-07, |
| "loss": 0.0027, |
| "reward": 1.362500011920929, |
| "reward_std": 0.4368693381547928, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.675000011920929, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.59375, |
| "epoch": 0.4828532235939643, |
| "grad_norm": 4.996971343930396, |
| "kl": 0.064453125, |
| "learning_rate": 8.399999999999999e-07, |
| "loss": 0.0026, |
| "reward": 1.498437523841858, |
| "reward_std": 0.5075877904891968, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7484375238418579, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.171875, |
| "epoch": 0.48559670781893005, |
| "grad_norm": 3.9600821078570516, |
| "kl": 0.0634765625, |
| "learning_rate": 8.390909090909091e-07, |
| "loss": 0.0025, |
| "reward": 1.4000000357627869, |
| "reward_std": 0.5038798004388809, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6812499910593033, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.171875, |
| "epoch": 0.4883401920438957, |
| "grad_norm": 3.1362251817973954, |
| "kl": 0.06396484375, |
| "learning_rate": 8.381818181818181e-07, |
| "loss": 0.0026, |
| "reward": 1.4093750715255737, |
| "reward_std": 0.38460302352905273, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6906249821186066, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.265625, |
| "epoch": 0.49108367626886146, |
| "grad_norm": 25.078130616204984, |
| "kl": 0.062744140625, |
| "learning_rate": 8.372727272727272e-07, |
| "loss": 0.0025, |
| "reward": 1.28515625, |
| "reward_std": 0.808452844619751, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.62890625, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.265625, |
| "epoch": 0.49382716049382713, |
| "grad_norm": 4.035103162287522, |
| "kl": 0.06103515625, |
| "learning_rate": 8.363636363636363e-07, |
| "loss": 0.0024, |
| "reward": 1.5062499642372131, |
| "reward_std": 0.5272868573665619, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7406249642372131, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.78125, |
| "epoch": 0.49657064471879286, |
| "grad_norm": 7.1345410663907, |
| "kl": 0.0576171875, |
| "learning_rate": 8.354545454545455e-07, |
| "loss": 0.0023, |
| "reward": 1.501562476158142, |
| "reward_std": 0.4868186265230179, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7515625059604645, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.234375, |
| "epoch": 0.4993141289437586, |
| "grad_norm": 5.1332471380641245, |
| "kl": 0.06201171875, |
| "learning_rate": 8.345454545454545e-07, |
| "loss": 0.0025, |
| "reward": 1.438281238079071, |
| "reward_std": 0.5540306866168976, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.703906238079071, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.625, |
| "epoch": 0.5020576131687243, |
| "grad_norm": 3.6923216925479827, |
| "kl": 0.063232421875, |
| "learning_rate": 8.336363636363636e-07, |
| "loss": 0.0025, |
| "reward": 1.921875, |
| "reward_std": 0.1922881193459034, |
| "rewards/choice_reward": 0.984375, |
| "rewards/roam_reward": 0.9375, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.96875, |
| "epoch": 0.50480109739369, |
| "grad_norm": 10.401860032506013, |
| "kl": 0.081298828125, |
| "learning_rate": 8.327272727272727e-07, |
| "loss": 0.0033, |
| "reward": 1.055468738079071, |
| "reward_std": 0.506570965051651, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5398437082767487, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.5625, |
| "epoch": 0.5075445816186557, |
| "grad_norm": 11.418018723924762, |
| "kl": 0.0673828125, |
| "learning_rate": 8.318181818181817e-07, |
| "loss": 0.0027, |
| "reward": 0.9195312261581421, |
| "reward_std": 0.7840481698513031, |
| "rewards/choice_reward": 0.4375, |
| "rewards/roam_reward": 0.4820312112569809, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.546875, |
| "epoch": 0.5102880658436214, |
| "grad_norm": 8.196839368231336, |
| "kl": 0.0618896484375, |
| "learning_rate": 8.309090909090909e-07, |
| "loss": 0.0025, |
| "reward": 1.266406238079071, |
| "reward_std": 0.6124914884567261, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.641406238079071, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.4375, |
| "epoch": 0.5130315500685871, |
| "grad_norm": 5.765281950846985, |
| "kl": 0.07470703125, |
| "learning_rate": 8.299999999999999e-07, |
| "loss": 0.003, |
| "reward": 1.354687511920929, |
| "reward_std": 0.3992236405611038, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.682812511920929, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.28125, |
| "epoch": 0.5157750342935528, |
| "grad_norm": 3.7172993929568325, |
| "kl": 0.0555419921875, |
| "learning_rate": 8.290909090909091e-07, |
| "loss": 0.0022, |
| "reward": 1.3773437142372131, |
| "reward_std": 0.5645609945058823, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6898437142372131, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.109375, |
| "epoch": 0.5185185185185185, |
| "grad_norm": 16.786414154053386, |
| "kl": 0.068359375, |
| "learning_rate": 8.281818181818181e-07, |
| "loss": 0.0027, |
| "reward": 1.662500023841858, |
| "reward_std": 0.5854551792144775, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8343749940395355, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.421875, |
| "epoch": 0.5212620027434842, |
| "grad_norm": 4.276593973859327, |
| "kl": 0.08642578125, |
| "learning_rate": 8.272727272727272e-07, |
| "loss": 0.0035, |
| "reward": 1.5609374642372131, |
| "reward_std": 0.4817267805337906, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7796874940395355, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.640625, |
| "epoch": 0.52400548696845, |
| "grad_norm": 11.234841144738262, |
| "kl": 0.074462890625, |
| "learning_rate": 8.263636363636364e-07, |
| "loss": 0.003, |
| "reward": 1.4281249642372131, |
| "reward_std": 0.13276604190468788, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7249999642372131, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.96875, |
| "epoch": 0.5267489711934157, |
| "grad_norm": 4.108221314020624, |
| "kl": 0.0673828125, |
| "learning_rate": 8.254545454545455e-07, |
| "loss": 0.0027, |
| "reward": 1.456250011920929, |
| "reward_std": 0.35238832235336304, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7218749821186066, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.671875, |
| "epoch": 0.5294924554183813, |
| "grad_norm": 16.0986145516457, |
| "kl": 0.106201171875, |
| "learning_rate": 8.245454545454545e-07, |
| "loss": 0.0042, |
| "reward": 1.378125011920929, |
| "reward_std": 0.44103650748729706, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.690625011920929, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.671875, |
| "epoch": 0.532235939643347, |
| "grad_norm": 5.763755062215557, |
| "kl": 0.077392578125, |
| "learning_rate": 8.236363636363636e-07, |
| "loss": 0.0031, |
| "reward": 1.1351562142372131, |
| "reward_std": 0.5356723964214325, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5882812142372131, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.78125, |
| "epoch": 0.5349794238683128, |
| "grad_norm": 3.782326754008591, |
| "kl": 0.091064453125, |
| "learning_rate": 8.227272727272727e-07, |
| "loss": 0.0036, |
| "reward": 1.3101562857627869, |
| "reward_std": 0.5293387621641159, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6382812261581421, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.5625, |
| "epoch": 0.5377229080932785, |
| "grad_norm": 39.2966810629021, |
| "kl": 0.077880859375, |
| "learning_rate": 8.218181818181817e-07, |
| "loss": 0.0031, |
| "reward": 1.4812500476837158, |
| "reward_std": 0.5564833879470825, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.715624988079071, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.203125, |
| "epoch": 0.5404663923182441, |
| "grad_norm": 4.055896823642352, |
| "kl": 0.09765625, |
| "learning_rate": 8.209090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.590624988079071, |
| "reward_std": 0.7486997246742249, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7937500178813934, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.4375, |
| "epoch": 0.5432098765432098, |
| "grad_norm": 4.254385588670383, |
| "kl": 0.1015625, |
| "learning_rate": 8.199999999999999e-07, |
| "loss": 0.0041, |
| "reward": 1.4296874403953552, |
| "reward_std": 0.54799535125494, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7265624701976776, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.265625, |
| "epoch": 0.5459533607681756, |
| "grad_norm": 5.646190691397499, |
| "kl": 0.096435546875, |
| "learning_rate": 8.190909090909091e-07, |
| "loss": 0.0039, |
| "reward": 1.3124999403953552, |
| "reward_std": 0.7107203602790833, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.65625, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.515625, |
| "epoch": 0.5486968449931413, |
| "grad_norm": 12.874617507070692, |
| "kl": 0.07421875, |
| "learning_rate": 8.181818181818182e-07, |
| "loss": 0.003, |
| "reward": 1.3499999642372131, |
| "reward_std": 0.7221298813819885, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6468750238418579, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.6875, |
| "epoch": 0.551440329218107, |
| "grad_norm": 3.1856851559121817, |
| "kl": 0.075439453125, |
| "learning_rate": 8.172727272727272e-07, |
| "loss": 0.003, |
| "reward": 1.6726562976837158, |
| "reward_std": 0.47229228913784027, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.844531238079071, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.546875, |
| "epoch": 0.5541838134430727, |
| "grad_norm": 2.3982091402370185, |
| "kl": 0.085205078125, |
| "learning_rate": 8.163636363636364e-07, |
| "loss": 0.0034, |
| "reward": 1.5078125, |
| "reward_std": 0.3067757934331894, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7578125, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.34375, |
| "epoch": 0.5569272976680384, |
| "grad_norm": 3.0016047967579027, |
| "kl": 0.08203125, |
| "learning_rate": 8.154545454545454e-07, |
| "loss": 0.0033, |
| "reward": 1.48828125, |
| "reward_std": 0.4044075235724449, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.75390625, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.125, |
| "epoch": 0.5596707818930041, |
| "grad_norm": 11.362915172474983, |
| "kl": 0.08251953125, |
| "learning_rate": 8.145454545454545e-07, |
| "loss": 0.0033, |
| "reward": 1.375781238079071, |
| "reward_std": 0.6488424837589264, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.703906238079071, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.09375, |
| "epoch": 0.5624142661179699, |
| "grad_norm": 10.498621862531559, |
| "kl": 0.072265625, |
| "learning_rate": 8.136363636363636e-07, |
| "loss": 0.0029, |
| "reward": 1.2671875357627869, |
| "reward_std": 0.7070984542369843, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6421874761581421, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.234375, |
| "epoch": 0.5651577503429356, |
| "grad_norm": 3.5297036527459893, |
| "kl": 0.0771484375, |
| "learning_rate": 8.127272727272727e-07, |
| "loss": 0.0031, |
| "reward": 1.071874976158142, |
| "reward_std": 0.7598352134227753, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5562500059604645, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.890625, |
| "epoch": 0.5679012345679012, |
| "grad_norm": 6.090925733370347, |
| "kl": 0.0631103515625, |
| "learning_rate": 8.118181818181817e-07, |
| "loss": 0.0025, |
| "reward": 1.0101562440395355, |
| "reward_std": 0.5675522089004517, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5101562589406967, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.53125, |
| "epoch": 0.5706447187928669, |
| "grad_norm": 3.4062820304676844, |
| "kl": 0.085205078125, |
| "learning_rate": 8.109090909090909e-07, |
| "loss": 0.0034, |
| "reward": 1.087499976158142, |
| "reward_std": 0.6073361337184906, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5562499761581421, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.078125, |
| "epoch": 0.5733882030178327, |
| "grad_norm": 5.021924382672697, |
| "kl": 0.07666015625, |
| "learning_rate": 8.1e-07, |
| "loss": 0.0031, |
| "reward": 1.4632812142372131, |
| "reward_std": 0.5444166660308838, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7132812142372131, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.59375, |
| "epoch": 0.5761316872427984, |
| "grad_norm": 4.83366950976083, |
| "kl": 0.0662841796875, |
| "learning_rate": 8.09090909090909e-07, |
| "loss": 0.0027, |
| "reward": 1.278124988079071, |
| "reward_std": 0.4413192719221115, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6531249582767487, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.65625, |
| "epoch": 0.578875171467764, |
| "grad_norm": 7.426211553447796, |
| "kl": 0.072509765625, |
| "learning_rate": 8.081818181818182e-07, |
| "loss": 0.0029, |
| "reward": 1.670312523841858, |
| "reward_std": 0.5201479196548462, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8265624940395355, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.421875, |
| "epoch": 0.5816186556927297, |
| "grad_norm": 6.151006851159569, |
| "kl": 0.070068359375, |
| "learning_rate": 8.072727272727272e-07, |
| "loss": 0.0028, |
| "reward": 1.505468726158142, |
| "reward_std": 0.7296687662601471, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7554687261581421, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.546875, |
| "epoch": 0.5843621399176955, |
| "grad_norm": 4.997037481883234, |
| "kl": 0.0791015625, |
| "learning_rate": 8.063636363636364e-07, |
| "loss": 0.0032, |
| "reward": 1.4304687976837158, |
| "reward_std": 0.3970513939857483, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.6960937082767487, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.703125, |
| "epoch": 0.5871056241426612, |
| "grad_norm": 9.389905559594292, |
| "kl": 0.07861328125, |
| "learning_rate": 8.054545454545454e-07, |
| "loss": 0.0032, |
| "reward": 1.30859375, |
| "reward_std": 0.5475015193223953, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.65234375, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.421875, |
| "epoch": 0.5898491083676269, |
| "grad_norm": 9.63225445426572, |
| "kl": 0.072998046875, |
| "learning_rate": 8.045454545454545e-07, |
| "loss": 0.0029, |
| "reward": 1.8234374523162842, |
| "reward_std": 0.4097355008125305, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.901562511920929, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.578125, |
| "epoch": 0.5925925925925926, |
| "grad_norm": 2.9208152120219633, |
| "kl": 0.09912109375, |
| "learning_rate": 8.036363636363636e-07, |
| "loss": 0.004, |
| "reward": 0.934374988079071, |
| "reward_std": 0.3858245126903057, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.48124995827674866, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.0625, |
| "epoch": 0.5953360768175583, |
| "grad_norm": 4.505664414649176, |
| "kl": 0.06494140625, |
| "learning_rate": 8.027272727272726e-07, |
| "loss": 0.0026, |
| "reward": 1.4281249642372131, |
| "reward_std": 0.508673369884491, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7249999642372131, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.171875, |
| "epoch": 0.598079561042524, |
| "grad_norm": 9.316782090306322, |
| "kl": 0.095947265625, |
| "learning_rate": 8.018181818181817e-07, |
| "loss": 0.0038, |
| "reward": 0.9671874642372131, |
| "reward_std": 0.5713367760181427, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.48281247913837433, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.609375, |
| "epoch": 0.6008230452674898, |
| "grad_norm": 2.855058328976504, |
| "kl": 0.071533203125, |
| "learning_rate": 8.009090909090909e-07, |
| "loss": 0.0029, |
| "reward": 1.116406261920929, |
| "reward_std": 0.344180703163147, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5695312172174454, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.03125, |
| "epoch": 0.6035665294924554, |
| "grad_norm": 6.719437428317107, |
| "kl": 0.0653076171875, |
| "learning_rate": 8e-07, |
| "loss": 0.0026, |
| "reward": 1.3828125, |
| "reward_std": 0.6445798277854919, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6796874701976776, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.453125, |
| "epoch": 0.6063100137174211, |
| "grad_norm": 71.48990096623994, |
| "kl": 0.074462890625, |
| "learning_rate": 7.99090909090909e-07, |
| "loss": 0.003, |
| "reward": 1.6539062857627869, |
| "reward_std": 0.3329023942351341, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8257812559604645, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.625, |
| "epoch": 0.6090534979423868, |
| "grad_norm": 3.0987243394875694, |
| "kl": 0.0572509765625, |
| "learning_rate": 7.981818181818182e-07, |
| "loss": 0.0023, |
| "reward": 1.564843773841858, |
| "reward_std": 0.5028156638145447, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7679687440395355, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.90625, |
| "epoch": 0.6117969821673526, |
| "grad_norm": 4.11857447430343, |
| "kl": 0.0654296875, |
| "learning_rate": 7.972727272727272e-07, |
| "loss": 0.0026, |
| "reward": 0.9781249761581421, |
| "reward_std": 0.12847728352062404, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.5093749761581421, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.765625, |
| "epoch": 0.6145404663923183, |
| "grad_norm": 6.078673620884672, |
| "kl": 0.078369140625, |
| "learning_rate": 7.963636363636364e-07, |
| "loss": 0.0031, |
| "reward": 1.6046874523162842, |
| "reward_std": 0.3992674648761749, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8078124821186066, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.890625, |
| "epoch": 0.6172839506172839, |
| "grad_norm": 8.644719384891074, |
| "kl": 0.071533203125, |
| "learning_rate": 7.954545454545454e-07, |
| "loss": 0.0029, |
| "reward": 1.5695313215255737, |
| "reward_std": 0.49713802337646484, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.772656261920929, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.140625, |
| "epoch": 0.6200274348422496, |
| "grad_norm": 3.2040195903210864, |
| "kl": 0.07763671875, |
| "learning_rate": 7.945454545454545e-07, |
| "loss": 0.0031, |
| "reward": 1.139843761920929, |
| "reward_std": 0.5067263841629028, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.561718761920929, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.890625, |
| "epoch": 0.6227709190672154, |
| "grad_norm": 4.010879810986008, |
| "kl": 0.0733642578125, |
| "learning_rate": 7.936363636363636e-07, |
| "loss": 0.0029, |
| "reward": 0.8140625059604645, |
| "reward_std": 0.5690027624368668, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.4234374761581421, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.734375, |
| "epoch": 0.6255144032921811, |
| "grad_norm": 9.581974697201934, |
| "kl": 0.08203125, |
| "learning_rate": 7.927272727272727e-07, |
| "loss": 0.0033, |
| "reward": 1.751562476158142, |
| "reward_std": 0.33712854981422424, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8765625059604645, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.171875, |
| "epoch": 0.6282578875171467, |
| "grad_norm": 4.489954093125781, |
| "kl": 0.084716796875, |
| "learning_rate": 7.918181818181818e-07, |
| "loss": 0.0034, |
| "reward": 1.2937500476837158, |
| "reward_std": 0.5399462282657623, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6374999582767487, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.25, |
| "epoch": 0.6310013717421125, |
| "grad_norm": 10.636316970075013, |
| "kl": 0.12744140625, |
| "learning_rate": 7.909090909090909e-07, |
| "loss": 0.0051, |
| "reward": 1.693750023841858, |
| "reward_std": 0.5591640174388885, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8343750238418579, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.421875, |
| "epoch": 0.6337448559670782, |
| "grad_norm": 2.8163130027865257, |
| "kl": 0.0859375, |
| "learning_rate": 7.9e-07, |
| "loss": 0.0034, |
| "reward": 1.4468750357627869, |
| "reward_std": 0.28992322087287903, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7281249761581421, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.46875, |
| "epoch": 0.6364883401920439, |
| "grad_norm": 5.0001878798824695, |
| "kl": 0.077392578125, |
| "learning_rate": 7.89090909090909e-07, |
| "loss": 0.0031, |
| "reward": 0.835156261920929, |
| "reward_std": 0.5924297571182251, |
| "rewards/choice_reward": 0.40625, |
| "rewards/roam_reward": 0.4289062172174454, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.65625, |
| "epoch": 0.6392318244170097, |
| "grad_norm": 3.9331776114517534, |
| "kl": 0.104248046875, |
| "learning_rate": 7.881818181818182e-07, |
| "loss": 0.0042, |
| "reward": 0.967968761920929, |
| "reward_std": 0.537762463092804, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.49921873211860657, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.53125, |
| "epoch": 0.6419753086419753, |
| "grad_norm": 12.025679525832018, |
| "kl": 0.076416015625, |
| "learning_rate": 7.872727272727272e-07, |
| "loss": 0.0031, |
| "reward": 1.5515625476837158, |
| "reward_std": 0.6109479367733002, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7859375178813934, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.484375, |
| "epoch": 0.644718792866941, |
| "grad_norm": 4.602797841328001, |
| "kl": 0.092529296875, |
| "learning_rate": 7.863636363636363e-07, |
| "loss": 0.0037, |
| "reward": 1.405468761920929, |
| "reward_std": 0.6358656883239746, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7179687321186066, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.234375, |
| "epoch": 0.6474622770919067, |
| "grad_norm": 16.49968920373472, |
| "kl": 0.096923828125, |
| "learning_rate": 7.854545454545454e-07, |
| "loss": 0.0039, |
| "reward": 1.0156249403953552, |
| "reward_std": 0.3995538502931595, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.53125, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.78125, |
| "epoch": 0.6502057613168725, |
| "grad_norm": 5.31859764210797, |
| "kl": 0.102294921875, |
| "learning_rate": 7.845454545454545e-07, |
| "loss": 0.0041, |
| "reward": 1.419531226158142, |
| "reward_std": 0.7316969633102417, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7164062261581421, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.265625, |
| "epoch": 0.6529492455418381, |
| "grad_norm": 3.9773992042537007, |
| "kl": 0.09814453125, |
| "learning_rate": 7.836363636363637e-07, |
| "loss": 0.0039, |
| "reward": 1.442187488079071, |
| "reward_std": 0.3996770977973938, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.723437488079071, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.625, |
| "epoch": 0.6556927297668038, |
| "grad_norm": 22.07054390449565, |
| "kl": 0.13623046875, |
| "learning_rate": 7.827272727272727e-07, |
| "loss": 0.0055, |
| "reward": 1.612500011920929, |
| "reward_std": 0.40521635860204697, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.815625011920929, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.25, |
| "epoch": 0.6584362139917695, |
| "grad_norm": 5.980129650176006, |
| "kl": 0.097412109375, |
| "learning_rate": 7.818181818181818e-07, |
| "loss": 0.0039, |
| "reward": 1.268750011920929, |
| "reward_std": 0.6887725591659546, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6437499821186066, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.921875, |
| "epoch": 0.6611796982167353, |
| "grad_norm": 8.202068432584896, |
| "kl": 0.0947265625, |
| "learning_rate": 7.809090909090909e-07, |
| "loss": 0.0038, |
| "reward": 0.9382812082767487, |
| "reward_std": 0.7795107066631317, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.48515623807907104, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.484375, |
| "epoch": 0.663923182441701, |
| "grad_norm": 8.414940184713855, |
| "kl": 0.09765625, |
| "learning_rate": 7.799999999999999e-07, |
| "loss": 0.0039, |
| "reward": 1.387499988079071, |
| "reward_std": 0.5833183825016022, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.699999988079071, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.078125, |
| "epoch": 0.6666666666666666, |
| "grad_norm": 2.0305731063352193, |
| "kl": 0.1015625, |
| "learning_rate": 7.79090909090909e-07, |
| "loss": 0.0041, |
| "reward": 1.864062488079071, |
| "reward_std": 0.28691305220127106, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9265625178813934, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.34375, |
| "epoch": 0.6694101508916324, |
| "grad_norm": 2.7281908126720817, |
| "kl": 0.10302734375, |
| "learning_rate": 7.781818181818182e-07, |
| "loss": 0.0041, |
| "reward": 1.1234374642372131, |
| "reward_std": 0.5269819796085358, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5765624642372131, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.375, |
| "epoch": 0.6721536351165981, |
| "grad_norm": 25.08389462493231, |
| "kl": 0.098388671875, |
| "learning_rate": 7.772727272727272e-07, |
| "loss": 0.0039, |
| "reward": 1.0125000178813934, |
| "reward_std": 0.6459892988204956, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.528124988079071, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.875, |
| "epoch": 0.6748971193415638, |
| "grad_norm": 9.640690247273248, |
| "kl": 0.109619140625, |
| "learning_rate": 7.763636363636363e-07, |
| "loss": 0.0044, |
| "reward": 0.7359375059604645, |
| "reward_std": 0.7127074599266052, |
| "rewards/choice_reward": 0.34375, |
| "rewards/roam_reward": 0.3921874761581421, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.28125, |
| "epoch": 0.6776406035665294, |
| "grad_norm": 9.572287534215459, |
| "kl": 0.14697265625, |
| "learning_rate": 7.754545454545455e-07, |
| "loss": 0.0059, |
| "reward": 1.31640625, |
| "reward_std": 0.5501902848482132, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6757812798023224, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.765625, |
| "epoch": 0.6803840877914952, |
| "grad_norm": 9.930117524851914, |
| "kl": 0.1484375, |
| "learning_rate": 7.745454545454545e-07, |
| "loss": 0.0059, |
| "reward": 1.3507812023162842, |
| "reward_std": 0.5230868756771088, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6789062321186066, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.453125, |
| "epoch": 0.6831275720164609, |
| "grad_norm": 2.8552077332389056, |
| "kl": 0.121826171875, |
| "learning_rate": 7.736363636363637e-07, |
| "loss": 0.0049, |
| "reward": 1.6515625715255737, |
| "reward_std": 0.5274887830018997, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.807812511920929, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.140625, |
| "epoch": 0.6858710562414266, |
| "grad_norm": 3.978831679062336, |
| "kl": 0.10205078125, |
| "learning_rate": 7.727272727272727e-07, |
| "loss": 0.0041, |
| "reward": 1.3421874642372131, |
| "reward_std": 0.54762202501297, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.484375, |
| "epoch": 0.6886145404663924, |
| "grad_norm": 7.637263581573902, |
| "kl": 0.091552734375, |
| "learning_rate": 7.718181818181818e-07, |
| "loss": 0.0037, |
| "reward": 1.490625023841858, |
| "reward_std": 0.6624627113342285, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7406249940395355, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.25, |
| "epoch": 0.691358024691358, |
| "grad_norm": 3.9888732110964122, |
| "kl": 0.08740234375, |
| "learning_rate": 7.709090909090909e-07, |
| "loss": 0.0035, |
| "reward": 1.66796875, |
| "reward_std": 0.42886100709438324, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.83984375, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.40625, |
| "epoch": 0.6941015089163237, |
| "grad_norm": 4.590621836385763, |
| "kl": 0.0966796875, |
| "learning_rate": 7.699999999999999e-07, |
| "loss": 0.0039, |
| "reward": 1.596875011920929, |
| "reward_std": 0.47170980274677277, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7999999821186066, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.890625, |
| "epoch": 0.6968449931412894, |
| "grad_norm": 4.347474133639677, |
| "kl": 0.100341796875, |
| "learning_rate": 7.69090909090909e-07, |
| "loss": 0.004, |
| "reward": 1.478124976158142, |
| "reward_std": 0.593116819858551, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7437499761581421, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.171875, |
| "epoch": 0.6995884773662552, |
| "grad_norm": 8.503027719373268, |
| "kl": 0.082763671875, |
| "learning_rate": 7.681818181818182e-07, |
| "loss": 0.0033, |
| "reward": 1.448437511920929, |
| "reward_std": 0.47390592098236084, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7296874821186066, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.28125, |
| "epoch": 0.7023319615912208, |
| "grad_norm": 4.592452577929141, |
| "kl": 0.09130859375, |
| "learning_rate": 7.672727272727271e-07, |
| "loss": 0.0036, |
| "reward": 1.4421875476837158, |
| "reward_std": 0.1951931044459343, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.723437488079071, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.875, |
| "epoch": 0.7050754458161865, |
| "grad_norm": 6.177062965711192, |
| "kl": 0.0947265625, |
| "learning_rate": 7.663636363636363e-07, |
| "loss": 0.0038, |
| "reward": 1.3125, |
| "reward_std": 0.33411741256713867, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6718749701976776, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.171875, |
| "epoch": 0.7078189300411523, |
| "grad_norm": 3.819488973591032, |
| "kl": 0.09521484375, |
| "learning_rate": 7.654545454545455e-07, |
| "loss": 0.0038, |
| "reward": 1.234375, |
| "reward_std": 0.3691229335963726, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.625, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.75, |
| "epoch": 0.710562414266118, |
| "grad_norm": 8.505930688018147, |
| "kl": 0.09716796875, |
| "learning_rate": 7.645454545454545e-07, |
| "loss": 0.0039, |
| "reward": 0.8343750238418579, |
| "reward_std": 0.43087735772132874, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.4437499940395355, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.15625, |
| "epoch": 0.7133058984910837, |
| "grad_norm": 2.619379040722451, |
| "kl": 0.081298828125, |
| "learning_rate": 7.636363636363636e-07, |
| "loss": 0.0032, |
| "reward": 1.250781238079071, |
| "reward_std": 0.09419287834316492, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6414062082767487, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.078125, |
| "epoch": 0.7160493827160493, |
| "grad_norm": 24.31183400021532, |
| "kl": 0.086181640625, |
| "learning_rate": 7.627272727272727e-07, |
| "loss": 0.0035, |
| "reward": 1.541406273841858, |
| "reward_std": 0.3669806718826294, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7601561844348907, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.25, |
| "epoch": 0.7187928669410151, |
| "grad_norm": 4.330377980133479, |
| "kl": 0.085205078125, |
| "learning_rate": 7.618181818181818e-07, |
| "loss": 0.0034, |
| "reward": 1.778124988079071, |
| "reward_std": 0.42067764699459076, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8875000178813934, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.4375, |
| "epoch": 0.7215363511659808, |
| "grad_norm": 2.8368583838801182, |
| "kl": 0.067626953125, |
| "learning_rate": 7.609090909090909e-07, |
| "loss": 0.0027, |
| "reward": 1.1468749642372131, |
| "reward_std": 0.4227535352110863, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5843749642372131, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.640625, |
| "epoch": 0.7242798353909465, |
| "grad_norm": 5.131099302078508, |
| "kl": 0.078369140625, |
| "learning_rate": 7.599999999999999e-07, |
| "loss": 0.0031, |
| "reward": 1.178906261920929, |
| "reward_std": 0.49694086611270905, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.6007812470197678, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.703125, |
| "epoch": 0.7270233196159122, |
| "grad_norm": 7.5594308912964765, |
| "kl": 0.093505859375, |
| "learning_rate": 7.59090909090909e-07, |
| "loss": 0.0037, |
| "reward": 1.3468750715255737, |
| "reward_std": 0.42676912248134613, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6906249225139618, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.359375, |
| "epoch": 0.7297668038408779, |
| "grad_norm": 3.699739470898038, |
| "kl": 0.081298828125, |
| "learning_rate": 7.581818181818182e-07, |
| "loss": 0.0033, |
| "reward": 1.2773436903953552, |
| "reward_std": 0.6463219225406647, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6523437798023224, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.75, |
| "epoch": 0.7325102880658436, |
| "grad_norm": 5.612634899949188, |
| "kl": 0.096923828125, |
| "learning_rate": 7.572727272727272e-07, |
| "loss": 0.0039, |
| "reward": 0.8046875, |
| "reward_std": 0.4974658638238907, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.4140624403953552, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.34375, |
| "epoch": 0.7352537722908093, |
| "grad_norm": 3.9276511293328675, |
| "kl": 0.1044921875, |
| "learning_rate": 7.563636363636363e-07, |
| "loss": 0.0042, |
| "reward": 1.5101561546325684, |
| "reward_std": 0.4631221741437912, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7132812440395355, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.390625, |
| "epoch": 0.7379972565157751, |
| "grad_norm": 7.060213562987752, |
| "kl": 0.090087890625, |
| "learning_rate": 7.554545454545455e-07, |
| "loss": 0.0036, |
| "reward": 1.3593750596046448, |
| "reward_std": 0.586755782365799, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6874999403953552, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.9375, |
| "epoch": 0.7407407407407407, |
| "grad_norm": 5.23495205214961, |
| "kl": 0.0888671875, |
| "learning_rate": 7.545454545454545e-07, |
| "loss": 0.0036, |
| "reward": 1.020312488079071, |
| "reward_std": 0.5529607236385345, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.48906248807907104, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.796875, |
| "epoch": 0.7434842249657064, |
| "grad_norm": 4.078711855133753, |
| "kl": 0.126220703125, |
| "learning_rate": 7.536363636363636e-07, |
| "loss": 0.0051, |
| "reward": 1.4265625476837158, |
| "reward_std": 0.6099326908588409, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7234375178813934, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.234375, |
| "epoch": 0.7462277091906722, |
| "grad_norm": 5.828898668341959, |
| "kl": 0.0631103515625, |
| "learning_rate": 7.527272727272727e-07, |
| "loss": 0.0025, |
| "reward": 1.5179687142372131, |
| "reward_std": 0.5480734258890152, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7523437142372131, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.484375, |
| "epoch": 0.7489711934156379, |
| "grad_norm": 6.955776732075884, |
| "kl": 0.087890625, |
| "learning_rate": 7.518181818181818e-07, |
| "loss": 0.0035, |
| "reward": 1.2203125357627869, |
| "reward_std": 0.40472525358200073, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6265624761581421, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.140625, |
| "epoch": 0.7517146776406035, |
| "grad_norm": 3.884958639901895, |
| "kl": 0.08935546875, |
| "learning_rate": 7.509090909090908e-07, |
| "loss": 0.0036, |
| "reward": 1.6171875596046448, |
| "reward_std": 0.43695899844169617, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8046874701976776, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.703125, |
| "epoch": 0.7544581618655692, |
| "grad_norm": 3.7953559845276397, |
| "kl": 0.089111328125, |
| "learning_rate": 7.5e-07, |
| "loss": 0.0036, |
| "reward": 1.545312523841858, |
| "reward_std": 0.5323244035243988, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7796874940395355, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.625, |
| "epoch": 0.757201646090535, |
| "grad_norm": 4.135276215930814, |
| "kl": 0.100830078125, |
| "learning_rate": 7.490909090909091e-07, |
| "loss": 0.004, |
| "reward": 1.137499988079071, |
| "reward_std": 0.5229451805353165, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.574999988079071, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.28125, |
| "epoch": 0.7599451303155007, |
| "grad_norm": 3.7538164014579056, |
| "kl": 0.091552734375, |
| "learning_rate": 7.481818181818182e-07, |
| "loss": 0.0037, |
| "reward": 1.39453125, |
| "reward_std": 0.6234084516763687, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6914062798023224, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.359375, |
| "epoch": 0.7626886145404664, |
| "grad_norm": 5.0738501302107855, |
| "kl": 0.0986328125, |
| "learning_rate": 7.472727272727272e-07, |
| "loss": 0.0039, |
| "reward": 1.0726562142372131, |
| "reward_std": 0.8105998933315277, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5414062440395355, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.765625, |
| "epoch": 0.7654320987654321, |
| "grad_norm": 3.458442593359515, |
| "kl": 0.0830078125, |
| "learning_rate": 7.463636363636363e-07, |
| "loss": 0.0033, |
| "reward": 1.3437500596046448, |
| "reward_std": 0.642494797706604, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6874999701976776, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.265625, |
| "epoch": 0.7681755829903978, |
| "grad_norm": 4.725759896770685, |
| "kl": 0.083251953125, |
| "learning_rate": 7.454545454545455e-07, |
| "loss": 0.0033, |
| "reward": 1.1656250357627869, |
| "reward_std": 0.3226143643260002, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.6031249761581421, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.25, |
| "epoch": 0.7709190672153635, |
| "grad_norm": 3.678317888450733, |
| "kl": 0.1083984375, |
| "learning_rate": 7.445454545454544e-07, |
| "loss": 0.0043, |
| "reward": 1.1148437559604645, |
| "reward_std": 0.4589616358280182, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5679687410593033, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.921875, |
| "epoch": 0.7736625514403292, |
| "grad_norm": 14.099429377447535, |
| "kl": 0.08740234375, |
| "learning_rate": 7.436363636363636e-07, |
| "loss": 0.0035, |
| "reward": 1.4703125357627869, |
| "reward_std": 0.4954650402069092, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7359375059604645, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.359375, |
| "epoch": 0.7764060356652949, |
| "grad_norm": 3.358461975282723, |
| "kl": 0.085693359375, |
| "learning_rate": 7.427272727272727e-07, |
| "loss": 0.0034, |
| "reward": 1.1437499523162842, |
| "reward_std": 0.45597282610833645, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.581250011920929, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.359375, |
| "epoch": 0.7791495198902606, |
| "grad_norm": 5.233594477059757, |
| "kl": 0.095947265625, |
| "learning_rate": 7.418181818181818e-07, |
| "loss": 0.0038, |
| "reward": 1.33984375, |
| "reward_std": 0.6549213230609894, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6835937201976776, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.46875, |
| "epoch": 0.7818930041152263, |
| "grad_norm": 2.478879945186959, |
| "kl": 0.101318359375, |
| "learning_rate": 7.409090909090909e-07, |
| "loss": 0.0041, |
| "reward": 1.7281250357627869, |
| "reward_std": 0.37530940771102905, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8531250059604645, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.0625, |
| "epoch": 0.7846364883401921, |
| "grad_norm": 2.9609063225313297, |
| "kl": 0.099609375, |
| "learning_rate": 7.4e-07, |
| "loss": 0.004, |
| "reward": 1.2867187857627869, |
| "reward_std": 0.5633288323879242, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6617186963558197, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.59375, |
| "epoch": 0.7873799725651578, |
| "grad_norm": 3.84297620567891, |
| "kl": 0.09228515625, |
| "learning_rate": 7.390909090909091e-07, |
| "loss": 0.0037, |
| "reward": 1.453125, |
| "reward_std": 0.5088267177343369, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.734375, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.484375, |
| "epoch": 0.7901234567901234, |
| "grad_norm": 1.732041600478566, |
| "kl": 0.114013671875, |
| "learning_rate": 7.381818181818182e-07, |
| "loss": 0.0046, |
| "reward": 1.253125011920929, |
| "reward_std": 0.20044078677892685, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6437499821186066, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.890625, |
| "epoch": 0.7928669410150891, |
| "grad_norm": 3.494611350142717, |
| "kl": 0.098388671875, |
| "learning_rate": 7.372727272727272e-07, |
| "loss": 0.0039, |
| "reward": 1.0382812321186066, |
| "reward_std": 0.7175245881080627, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5382812023162842, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.03125, |
| "epoch": 0.7956104252400549, |
| "grad_norm": 4.409782914550581, |
| "kl": 0.09814453125, |
| "learning_rate": 7.363636363636363e-07, |
| "loss": 0.0039, |
| "reward": 1.0171875357627869, |
| "reward_std": 0.4266436994075775, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.5328124761581421, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.3125, |
| "epoch": 0.7983539094650206, |
| "grad_norm": 2.8626066477859005, |
| "kl": 0.09521484375, |
| "learning_rate": 7.354545454545455e-07, |
| "loss": 0.0038, |
| "reward": 1.479687511920929, |
| "reward_std": 0.4619990587234497, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.745312511920929, |
| "step": 291 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.609375, |
| "epoch": 0.8010973936899863, |
| "grad_norm": 4.176612663324125, |
| "kl": 0.095703125, |
| "learning_rate": 7.345454545454544e-07, |
| "loss": 0.0038, |
| "reward": 1.2265625, |
| "reward_std": 0.554766446352005, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6328125, |
| "step": 292 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.90625, |
| "epoch": 0.803840877914952, |
| "grad_norm": 2.4249557893758578, |
| "kl": 0.109130859375, |
| "learning_rate": 7.336363636363636e-07, |
| "loss": 0.0044, |
| "reward": 1.7718749642372131, |
| "reward_std": 0.21178293507546186, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8812499940395355, |
| "step": 293 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.703125, |
| "epoch": 0.8065843621399177, |
| "grad_norm": 2.871803780416522, |
| "kl": 0.09130859375, |
| "learning_rate": 7.327272727272728e-07, |
| "loss": 0.0037, |
| "reward": 1.3429687321186066, |
| "reward_std": 0.31595955044031143, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6710937321186066, |
| "step": 294 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.3125, |
| "epoch": 0.8093278463648834, |
| "grad_norm": 10.806987704832263, |
| "kl": 0.1357421875, |
| "learning_rate": 7.318181818181818e-07, |
| "loss": 0.0054, |
| "reward": 1.2625000476837158, |
| "reward_std": 0.6731216311454773, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6375000178813934, |
| "step": 295 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.375, |
| "epoch": 0.8120713305898491, |
| "grad_norm": 3.46173976943176, |
| "kl": 0.12841796875, |
| "learning_rate": 7.309090909090909e-07, |
| "loss": 0.0051, |
| "reward": 1.702343761920929, |
| "reward_std": 0.3345009386539459, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8585937321186066, |
| "step": 296 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.453125, |
| "epoch": 0.8148148148148148, |
| "grad_norm": 3.9733750299455375, |
| "kl": 0.081298828125, |
| "learning_rate": 7.3e-07, |
| "loss": 0.0033, |
| "reward": 1.6203124523162842, |
| "reward_std": 0.5902264714241028, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7921874821186066, |
| "step": 297 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.25, |
| "epoch": 0.8175582990397805, |
| "grad_norm": 12.925700937011783, |
| "kl": 0.088623046875, |
| "learning_rate": 7.290909090909091e-07, |
| "loss": 0.0035, |
| "reward": 1.739843726158142, |
| "reward_std": 0.4693824499845505, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8648437261581421, |
| "step": 298 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.453125, |
| "epoch": 0.8203017832647462, |
| "grad_norm": 2.1252410795267442, |
| "kl": 0.08349609375, |
| "learning_rate": 7.281818181818181e-07, |
| "loss": 0.0034, |
| "reward": 1.6500000357627869, |
| "reward_std": 0.365024596452713, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8218750059604645, |
| "step": 299 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.375, |
| "epoch": 0.823045267489712, |
| "grad_norm": 2.9274319672566915, |
| "kl": 0.091552734375, |
| "learning_rate": 7.272727272727272e-07, |
| "loss": 0.0037, |
| "reward": 1.592187523841858, |
| "reward_std": 0.3340279906988144, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7953124940395355, |
| "step": 300 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.046875, |
| "epoch": 0.8257887517146777, |
| "grad_norm": 3.5682926071799645, |
| "kl": 0.094482421875, |
| "learning_rate": 7.263636363636363e-07, |
| "loss": 0.0038, |
| "reward": 1.622656226158142, |
| "reward_std": 0.5170932710170746, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8101562559604645, |
| "step": 301 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.1875, |
| "epoch": 0.8285322359396433, |
| "grad_norm": 4.223087291480602, |
| "kl": 0.084228515625, |
| "learning_rate": 7.254545454545455e-07, |
| "loss": 0.0034, |
| "reward": 1.1343750357627869, |
| "reward_std": 0.49241510033607483, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5874999463558197, |
| "step": 302 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.359375, |
| "epoch": 0.831275720164609, |
| "grad_norm": 2.86803311983425, |
| "kl": 0.103271484375, |
| "learning_rate": 7.245454545454544e-07, |
| "loss": 0.0041, |
| "reward": 1.5828125476837158, |
| "reward_std": 0.43118855357170105, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.801562488079071, |
| "step": 303 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.609375, |
| "epoch": 0.8340192043895748, |
| "grad_norm": 2.5900651993580857, |
| "kl": 0.093505859375, |
| "learning_rate": 7.236363636363636e-07, |
| "loss": 0.0037, |
| "reward": 1.3578124642372131, |
| "reward_std": 0.3414458632469177, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 304 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.109375, |
| "epoch": 0.8367626886145405, |
| "grad_norm": 19.974214898027473, |
| "kl": 0.086181640625, |
| "learning_rate": 7.227272727272728e-07, |
| "loss": 0.0034, |
| "reward": 1.682812511920929, |
| "reward_std": 0.46156272292137146, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.839062511920929, |
| "step": 305 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.59375, |
| "epoch": 0.8395061728395061, |
| "grad_norm": 4.462795625676432, |
| "kl": 0.104736328125, |
| "learning_rate": 7.218181818181817e-07, |
| "loss": 0.0042, |
| "reward": 1.474218726158142, |
| "reward_std": 0.4678943157196045, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7398437261581421, |
| "step": 306 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.515625, |
| "epoch": 0.8422496570644719, |
| "grad_norm": 5.604391149460443, |
| "kl": 0.084228515625, |
| "learning_rate": 7.209090909090909e-07, |
| "loss": 0.0034, |
| "reward": 1.217968761920929, |
| "reward_std": 0.5433058440685272, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6085937023162842, |
| "step": 307 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.109375, |
| "epoch": 0.8449931412894376, |
| "grad_norm": 6.693685254800663, |
| "kl": 0.09228515625, |
| "learning_rate": 7.2e-07, |
| "loss": 0.0037, |
| "reward": 1.3624999523162842, |
| "reward_std": 0.7833757400512695, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.690625011920929, |
| "step": 308 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.640625, |
| "epoch": 0.8477366255144033, |
| "grad_norm": 8.54674736781978, |
| "kl": 0.079833984375, |
| "learning_rate": 7.190909090909091e-07, |
| "loss": 0.0032, |
| "reward": 1.043749988079071, |
| "reward_std": 0.457779660820961, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5437500178813934, |
| "step": 309 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.84375, |
| "epoch": 0.850480109739369, |
| "grad_norm": 2.4299194682326593, |
| "kl": 0.080078125, |
| "learning_rate": 7.181818181818181e-07, |
| "loss": 0.0032, |
| "reward": 1.550000011920929, |
| "reward_std": 0.36771687865257263, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.784375011920929, |
| "step": 310 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.421875, |
| "epoch": 0.8532235939643347, |
| "grad_norm": 9.62096693731744, |
| "kl": 0.12158203125, |
| "learning_rate": 7.172727272727272e-07, |
| "loss": 0.0049, |
| "reward": 1.089062511920929, |
| "reward_std": 0.784207671880722, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5578124821186066, |
| "step": 311 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.375, |
| "epoch": 0.8559670781893004, |
| "grad_norm": 3.791843879642304, |
| "kl": 0.09521484375, |
| "learning_rate": 7.163636363636363e-07, |
| "loss": 0.0038, |
| "reward": 1.3015625476837158, |
| "reward_std": 0.3239322304725647, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.660937488079071, |
| "step": 312 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.46875, |
| "epoch": 0.8587105624142661, |
| "grad_norm": 4.566693887033749, |
| "kl": 0.08251953125, |
| "learning_rate": 7.154545454545454e-07, |
| "loss": 0.0033, |
| "reward": 1.403124988079071, |
| "reward_std": 0.6231254190206528, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.715624988079071, |
| "step": 313 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.65625, |
| "epoch": 0.8614540466392319, |
| "grad_norm": 18.23495938082825, |
| "kl": 0.117431640625, |
| "learning_rate": 7.145454545454545e-07, |
| "loss": 0.0047, |
| "reward": 1.0859375, |
| "reward_std": 0.6859345436096191, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5546874403953552, |
| "step": 314 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.234375, |
| "epoch": 0.8641975308641975, |
| "grad_norm": 4.832892304043935, |
| "kl": 0.100341796875, |
| "learning_rate": 7.136363636363636e-07, |
| "loss": 0.004, |
| "reward": 1.6624999642372131, |
| "reward_std": 0.5350576937198639, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8343749940395355, |
| "step": 315 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.40625, |
| "epoch": 0.8669410150891632, |
| "grad_norm": 3.296079249308525, |
| "kl": 0.103759765625, |
| "learning_rate": 7.127272727272728e-07, |
| "loss": 0.0041, |
| "reward": 1.396875023841858, |
| "reward_std": 0.6269551515579224, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7093749642372131, |
| "step": 316 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.859375, |
| "epoch": 0.869684499314129, |
| "grad_norm": 3.8568552276327703, |
| "kl": 0.0849609375, |
| "learning_rate": 7.118181818181817e-07, |
| "loss": 0.0034, |
| "reward": 1.890625, |
| "reward_std": 0.20640429854393005, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9375, |
| "step": 317 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.546875, |
| "epoch": 0.8724279835390947, |
| "grad_norm": 4.024591910607265, |
| "kl": 0.09814453125, |
| "learning_rate": 7.109090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.6203124523162842, |
| "reward_std": 0.611024409532547, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7921874821186066, |
| "step": 318 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.0, |
| "epoch": 0.8751714677640604, |
| "grad_norm": 4.977602479562242, |
| "kl": 0.10791015625, |
| "learning_rate": 7.1e-07, |
| "loss": 0.0043, |
| "reward": 1.431249976158142, |
| "reward_std": 0.7265175879001617, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.6968749761581421, |
| "step": 319 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.671875, |
| "epoch": 0.877914951989026, |
| "grad_norm": 12.201487507644098, |
| "kl": 0.09423828125, |
| "learning_rate": 7.09090909090909e-07, |
| "loss": 0.0038, |
| "reward": 1.667187511920929, |
| "reward_std": 0.4393137991428375, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.823437511920929, |
| "step": 320 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.609375, |
| "epoch": 0.8806584362139918, |
| "grad_norm": 5.6109455126827426, |
| "kl": 0.095947265625, |
| "learning_rate": 7.081818181818181e-07, |
| "loss": 0.0038, |
| "reward": 1.578125, |
| "reward_std": 0.4120820192620158, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7968749701976776, |
| "step": 321 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.28125, |
| "epoch": 0.8834019204389575, |
| "grad_norm": 3.029237010880203, |
| "kl": 0.079833984375, |
| "learning_rate": 7.072727272727272e-07, |
| "loss": 0.0032, |
| "reward": 1.500781238079071, |
| "reward_std": 0.599192276597023, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7351562678813934, |
| "step": 322 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.90625, |
| "epoch": 0.8861454046639232, |
| "grad_norm": 3.0855938992202625, |
| "kl": 0.094970703125, |
| "learning_rate": 7.063636363636364e-07, |
| "loss": 0.0038, |
| "reward": 1.278906226158142, |
| "reward_std": 0.3176858276128769, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6382812559604645, |
| "step": 323 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.328125, |
| "epoch": 0.8888888888888888, |
| "grad_norm": 6.174475362725212, |
| "kl": 0.104736328125, |
| "learning_rate": 7.054545454545454e-07, |
| "loss": 0.0042, |
| "reward": 1.3726562857627869, |
| "reward_std": 0.6849194169044495, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.7007812559604645, |
| "step": 324 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.578125, |
| "epoch": 0.8916323731138546, |
| "grad_norm": 4.340367067952867, |
| "kl": 0.09912109375, |
| "learning_rate": 7.045454545454545e-07, |
| "loss": 0.004, |
| "reward": 1.42578125, |
| "reward_std": 0.645444929599762, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7226562201976776, |
| "step": 325 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.9375, |
| "epoch": 0.8943758573388203, |
| "grad_norm": 2.8142124440241765, |
| "kl": 0.099853515625, |
| "learning_rate": 7.036363636363636e-07, |
| "loss": 0.004, |
| "reward": 1.2023437321186066, |
| "reward_std": 0.4010866731405258, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6085937321186066, |
| "step": 326 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.984375, |
| "epoch": 0.897119341563786, |
| "grad_norm": 9.240569190949303, |
| "kl": 0.097412109375, |
| "learning_rate": 7.027272727272728e-07, |
| "loss": 0.0039, |
| "reward": 1.3351562023162842, |
| "reward_std": 0.7516996264457703, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6789062321186066, |
| "step": 327 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.0625, |
| "epoch": 0.8998628257887518, |
| "grad_norm": 4.041779286763977, |
| "kl": 0.119873046875, |
| "learning_rate": 7.018181818181817e-07, |
| "loss": 0.0048, |
| "reward": 1.420312523841858, |
| "reward_std": 0.3459370732307434, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7171874940395355, |
| "step": 328 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.796875, |
| "epoch": 0.9026063100137174, |
| "grad_norm": 4.242021387631853, |
| "kl": 0.093017578125, |
| "learning_rate": 7.009090909090909e-07, |
| "loss": 0.0037, |
| "reward": 1.607812523841858, |
| "reward_std": 0.4460008144378662, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7953125238418579, |
| "step": 329 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.609375, |
| "epoch": 0.9053497942386831, |
| "grad_norm": 4.9465256422264945, |
| "kl": 0.1025390625, |
| "learning_rate": 7e-07, |
| "loss": 0.0041, |
| "reward": 1.5195311903953552, |
| "reward_std": 0.3289015144109726, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7695312201976776, |
| "step": 330 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.78125, |
| "epoch": 0.9080932784636488, |
| "grad_norm": 3.4771150742508716, |
| "kl": 0.083984375, |
| "learning_rate": 6.99090909090909e-07, |
| "loss": 0.0034, |
| "reward": 1.7789062857627869, |
| "reward_std": 0.4107031673192978, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8882812857627869, |
| "step": 331 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.3125, |
| "epoch": 0.9108367626886146, |
| "grad_norm": 3.4677948106320637, |
| "kl": 0.085693359375, |
| "learning_rate": 6.981818181818182e-07, |
| "loss": 0.0034, |
| "reward": 1.2460936903953552, |
| "reward_std": 0.6076911389827728, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.63671875, |
| "step": 332 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.0, |
| "epoch": 0.9135802469135802, |
| "grad_norm": 23.080171367895154, |
| "kl": 0.130615234375, |
| "learning_rate": 6.972727272727273e-07, |
| "loss": 0.0052, |
| "reward": 1.426562488079071, |
| "reward_std": 0.5667466223239899, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7234375178813934, |
| "step": 333 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.53125, |
| "epoch": 0.9163237311385459, |
| "grad_norm": 3.5573105855329077, |
| "kl": 0.101318359375, |
| "learning_rate": 6.963636363636364e-07, |
| "loss": 0.0041, |
| "reward": 1.153124988079071, |
| "reward_std": 0.442140594124794, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5906249582767487, |
| "step": 334 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.3125, |
| "epoch": 0.9190672153635117, |
| "grad_norm": 16.20583521957908, |
| "kl": 0.09375, |
| "learning_rate": 6.954545454545454e-07, |
| "loss": 0.0037, |
| "reward": 1.2859374284744263, |
| "reward_std": 0.5812228620052338, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6453124582767487, |
| "step": 335 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.859375, |
| "epoch": 0.9218106995884774, |
| "grad_norm": 8.148470377319255, |
| "kl": 0.0859375, |
| "learning_rate": 6.945454545454545e-07, |
| "loss": 0.0034, |
| "reward": 1.262499988079071, |
| "reward_std": 0.5951336622238159, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6218749582767487, |
| "step": 336 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.234375, |
| "epoch": 0.9245541838134431, |
| "grad_norm": 2.4416096060166383, |
| "kl": 0.09228515625, |
| "learning_rate": 6.936363636363636e-07, |
| "loss": 0.0037, |
| "reward": 1.668749988079071, |
| "reward_std": 0.3197062835097313, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.824999988079071, |
| "step": 337 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.796875, |
| "epoch": 0.9272976680384087, |
| "grad_norm": 3.2162665539035724, |
| "kl": 0.0927734375, |
| "learning_rate": 6.927272727272727e-07, |
| "loss": 0.0037, |
| "reward": 1.6093750596046448, |
| "reward_std": 0.4507560729980469, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8125, |
| "step": 338 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.046875, |
| "epoch": 0.9300411522633745, |
| "grad_norm": 17.23422076000473, |
| "kl": 0.080810546875, |
| "learning_rate": 6.918181818181817e-07, |
| "loss": 0.0032, |
| "reward": 1.639843761920929, |
| "reward_std": 0.292666919529438, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.827343761920929, |
| "step": 339 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.125, |
| "epoch": 0.9327846364883402, |
| "grad_norm": 6.520218077277935, |
| "kl": 0.1044921875, |
| "learning_rate": 6.909090909090909e-07, |
| "loss": 0.0042, |
| "reward": 1.5898436903953552, |
| "reward_std": 0.5847151577472687, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7929687201976776, |
| "step": 340 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.984375, |
| "epoch": 0.9355281207133059, |
| "grad_norm": 3.632307809145378, |
| "kl": 0.103759765625, |
| "learning_rate": 6.9e-07, |
| "loss": 0.0041, |
| "reward": 0.9554687440395355, |
| "reward_std": 0.5614143311977386, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.5023437440395355, |
| "step": 341 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.28125, |
| "epoch": 0.9382716049382716, |
| "grad_norm": 6.791169700208845, |
| "kl": 0.107421875, |
| "learning_rate": 6.89090909090909e-07, |
| "loss": 0.0043, |
| "reward": 1.4710937142372131, |
| "reward_std": 0.30878862738609314, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7367187440395355, |
| "step": 342 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.15625, |
| "epoch": 0.9410150891632373, |
| "grad_norm": 9.180934723679384, |
| "kl": 0.102294921875, |
| "learning_rate": 6.881818181818182e-07, |
| "loss": 0.0041, |
| "reward": 1.016406238079071, |
| "reward_std": 0.5839601010084152, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.532031238079071, |
| "step": 343 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.375, |
| "epoch": 0.943758573388203, |
| "grad_norm": 3.256295819561447, |
| "kl": 0.108154296875, |
| "learning_rate": 6.872727272727273e-07, |
| "loss": 0.0043, |
| "reward": 1.2203125357627869, |
| "reward_std": 0.5046594589948654, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6265624761581421, |
| "step": 344 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.453125, |
| "epoch": 0.9465020576131687, |
| "grad_norm": 3.6115730033643945, |
| "kl": 0.1015625, |
| "learning_rate": 6.863636363636363e-07, |
| "loss": 0.0041, |
| "reward": 1.5234375, |
| "reward_std": 0.6625894606113434, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7734375, |
| "step": 345 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.078125, |
| "epoch": 0.9492455418381345, |
| "grad_norm": 4.100928270883839, |
| "kl": 0.11083984375, |
| "learning_rate": 6.854545454545454e-07, |
| "loss": 0.0044, |
| "reward": 1.309374988079071, |
| "reward_std": 0.3420407697558403, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6531250178813934, |
| "step": 346 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.796875, |
| "epoch": 0.9519890260631001, |
| "grad_norm": 6.918298580051922, |
| "kl": 0.106689453125, |
| "learning_rate": 6.845454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.548437476158142, |
| "reward_std": 0.20821654796600342, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7671874761581421, |
| "step": 347 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.90625, |
| "epoch": 0.9547325102880658, |
| "grad_norm": 6.920880779530441, |
| "kl": 0.10791015625, |
| "learning_rate": 6.836363636363636e-07, |
| "loss": 0.0043, |
| "reward": 1.535937488079071, |
| "reward_std": 0.7484138607978821, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.770312488079071, |
| "step": 348 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.390625, |
| "epoch": 0.9574759945130316, |
| "grad_norm": 3.228855877922397, |
| "kl": 0.125, |
| "learning_rate": 6.827272727272727e-07, |
| "loss": 0.005, |
| "reward": 1.6398437023162842, |
| "reward_std": 0.4850577563047409, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8273437321186066, |
| "step": 349 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.265625, |
| "epoch": 0.9602194787379973, |
| "grad_norm": 1.4808004667582153, |
| "kl": 0.11572265625, |
| "learning_rate": 6.818181818181817e-07, |
| "loss": 0.0046, |
| "reward": 1.821874976158142, |
| "reward_std": 0.10994113236665726, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9156250059604645, |
| "step": 350 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.140625, |
| "epoch": 0.9629629629629629, |
| "grad_norm": 3.4206053989322953, |
| "kl": 0.115234375, |
| "learning_rate": 6.809090909090909e-07, |
| "loss": 0.0046, |
| "reward": 1.239062488079071, |
| "reward_std": 0.6298282146453857, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6296875178813934, |
| "step": 351 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.375, |
| "epoch": 0.9657064471879286, |
| "grad_norm": 3.82268090438329, |
| "kl": 0.1162109375, |
| "learning_rate": 6.800000000000001e-07, |
| "loss": 0.0047, |
| "reward": 1.609375, |
| "reward_std": 0.6609872579574585, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8125, |
| "step": 352 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.734375, |
| "epoch": 0.9684499314128944, |
| "grad_norm": 4.857664777910766, |
| "kl": 0.14697265625, |
| "learning_rate": 6.79090909090909e-07, |
| "loss": 0.0059, |
| "reward": 1.412500023841858, |
| "reward_std": 0.46737395226955414, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7093750238418579, |
| "step": 353 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.421875, |
| "epoch": 0.9711934156378601, |
| "grad_norm": 3.6194631196809253, |
| "kl": 0.105712890625, |
| "learning_rate": 6.781818181818182e-07, |
| "loss": 0.0042, |
| "reward": 1.5984375476837158, |
| "reward_std": 0.5569853782653809, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.801562488079071, |
| "step": 354 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.078125, |
| "epoch": 0.9739368998628258, |
| "grad_norm": 4.709451670730105, |
| "kl": 0.10400390625, |
| "learning_rate": 6.772727272727273e-07, |
| "loss": 0.0042, |
| "reward": 1.487500011920929, |
| "reward_std": 0.5324640423059464, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.721875011920929, |
| "step": 355 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.375, |
| "epoch": 0.9766803840877915, |
| "grad_norm": 6.2815158563240825, |
| "kl": 0.12548828125, |
| "learning_rate": 6.763636363636363e-07, |
| "loss": 0.005, |
| "reward": 1.3414062559604645, |
| "reward_std": 0.4287596568465233, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6851562559604645, |
| "step": 356 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.046875, |
| "epoch": 0.9794238683127572, |
| "grad_norm": 4.754282552821982, |
| "kl": 0.13671875, |
| "learning_rate": 6.754545454545454e-07, |
| "loss": 0.0055, |
| "reward": 1.1171875, |
| "reward_std": 0.480400413274765, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5703125, |
| "step": 357 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.03125, |
| "epoch": 0.9821673525377229, |
| "grad_norm": 3.161618547824301, |
| "kl": 0.1015625, |
| "learning_rate": 6.745454545454545e-07, |
| "loss": 0.0041, |
| "reward": 1.801562488079071, |
| "reward_std": 0.3133033514022827, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8953125178813934, |
| "step": 358 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.3125, |
| "epoch": 0.9849108367626886, |
| "grad_norm": 3.137174508984925, |
| "kl": 0.096435546875, |
| "learning_rate": 6.736363636363635e-07, |
| "loss": 0.0039, |
| "reward": 1.4281249940395355, |
| "reward_std": 0.37352126836776733, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7249999940395355, |
| "step": 359 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.859375, |
| "epoch": 0.9876543209876543, |
| "grad_norm": 5.442071087429494, |
| "kl": 0.10009765625, |
| "learning_rate": 6.727272727272727e-07, |
| "loss": 0.004, |
| "reward": 0.9671874940395355, |
| "reward_std": 0.6916260123252869, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.49843746423721313, |
| "step": 360 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.21875, |
| "epoch": 0.99039780521262, |
| "grad_norm": 6.404313944657274, |
| "kl": 0.0986328125, |
| "learning_rate": 6.718181818181818e-07, |
| "loss": 0.0039, |
| "reward": 1.4789062142372131, |
| "reward_std": 0.6432763636112213, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7445312440395355, |
| "step": 361 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.765625, |
| "epoch": 0.9931412894375857, |
| "grad_norm": 11.533715639852097, |
| "kl": 0.114990234375, |
| "learning_rate": 6.709090909090909e-07, |
| "loss": 0.0046, |
| "reward": 1.285937488079071, |
| "reward_std": 0.7446453869342804, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6609374582767487, |
| "step": 362 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.171875, |
| "epoch": 0.9958847736625515, |
| "grad_norm": 7.3920091754927375, |
| "kl": 0.095947265625, |
| "learning_rate": 6.7e-07, |
| "loss": 0.0038, |
| "reward": 1.6062500476837158, |
| "reward_std": 0.5926486253738403, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8093749582767487, |
| "step": 363 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.0, |
| "epoch": 0.9986282578875172, |
| "grad_norm": 3.98245690956048, |
| "kl": 0.085693359375, |
| "learning_rate": 6.69090909090909e-07, |
| "loss": 0.0038, |
| "reward": 1.050000000745058, |
| "reward_std": 0.999468058347702, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5500000007450581, |
| "step": 364 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.71875, |
| "epoch": 1.0027434842249656, |
| "grad_norm": 2.911604760532795, |
| "kl": 0.09619140625, |
| "learning_rate": 6.681818181818182e-07, |
| "loss": 0.0038, |
| "reward": 1.442968726158142, |
| "reward_std": 0.35979723185300827, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7242186963558197, |
| "step": 365 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.203125, |
| "epoch": 1.0054869684499315, |
| "grad_norm": 7.010322344814963, |
| "kl": 0.08837890625, |
| "learning_rate": 6.672727272727273e-07, |
| "loss": 0.0035, |
| "reward": 1.3468749523162842, |
| "reward_std": 0.5796718001365662, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6906249821186066, |
| "step": 366 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.625, |
| "epoch": 1.008230452674897, |
| "grad_norm": 4.912622392676742, |
| "kl": 0.08544921875, |
| "learning_rate": 6.663636363636363e-07, |
| "loss": 0.0034, |
| "reward": 1.3984375, |
| "reward_std": 0.5179575830698013, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6953125, |
| "step": 367 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.65625, |
| "epoch": 1.010973936899863, |
| "grad_norm": 2.0977130478706805, |
| "kl": 0.12109375, |
| "learning_rate": 6.654545454545454e-07, |
| "loss": 0.0048, |
| "reward": 1.6640625, |
| "reward_std": 0.2591197192668915, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8359375, |
| "step": 368 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.5625, |
| "epoch": 1.0137174211248285, |
| "grad_norm": 3.2382017857278753, |
| "kl": 0.1162109375, |
| "learning_rate": 6.645454545454545e-07, |
| "loss": 0.0047, |
| "reward": 1.5703125, |
| "reward_std": 0.5361149609088898, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7890624701976776, |
| "step": 369 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.046875, |
| "epoch": 1.0164609053497942, |
| "grad_norm": 7.678867298043466, |
| "kl": 0.100341796875, |
| "learning_rate": 6.636363636363636e-07, |
| "loss": 0.004, |
| "reward": 1.40234375, |
| "reward_std": 0.6340524554252625, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7148437201976776, |
| "step": 370 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.734375, |
| "epoch": 1.01920438957476, |
| "grad_norm": 4.289702688201466, |
| "kl": 0.08544921875, |
| "learning_rate": 6.627272727272727e-07, |
| "loss": 0.0034, |
| "reward": 1.598437488079071, |
| "reward_std": 0.34804536402225494, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.801562488079071, |
| "step": 371 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.53125, |
| "epoch": 1.0219478737997256, |
| "grad_norm": 8.3854367781373, |
| "kl": 0.080810546875, |
| "learning_rate": 6.618181818181818e-07, |
| "loss": 0.0032, |
| "reward": 1.3406250476837158, |
| "reward_std": 0.6063534319400787, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.684374988079071, |
| "step": 372 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.46875, |
| "epoch": 1.0246913580246915, |
| "grad_norm": 2.8046480040356845, |
| "kl": 0.08154296875, |
| "learning_rate": 6.609090909090909e-07, |
| "loss": 0.0033, |
| "reward": 1.193750023841858, |
| "reward_std": 0.30354222655296326, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.5843749940395355, |
| "step": 373 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.0, |
| "epoch": 1.027434842249657, |
| "grad_norm": 2.762046165937544, |
| "kl": 0.095703125, |
| "learning_rate": 6.6e-07, |
| "loss": 0.0038, |
| "reward": 1.550000011920929, |
| "reward_std": 0.09536151215434074, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.784375011920929, |
| "step": 374 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.171875, |
| "epoch": 1.0301783264746227, |
| "grad_norm": 12.076907213539458, |
| "kl": 0.0986328125, |
| "learning_rate": 6.59090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.504687488079071, |
| "reward_std": 0.5868997871875763, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.754687488079071, |
| "step": 375 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.265625, |
| "epoch": 1.0329218106995885, |
| "grad_norm": 3.580412178061941, |
| "kl": 0.091064453125, |
| "learning_rate": 6.581818181818182e-07, |
| "loss": 0.0036, |
| "reward": 1.498437523841858, |
| "reward_std": 0.6327075660228729, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7328124642372131, |
| "step": 376 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.390625, |
| "epoch": 1.0356652949245542, |
| "grad_norm": 13.372269942081257, |
| "kl": 0.113525390625, |
| "learning_rate": 6.572727272727272e-07, |
| "loss": 0.0045, |
| "reward": 1.045312523841858, |
| "reward_std": 0.3369172066450119, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5453125089406967, |
| "step": 377 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.328125, |
| "epoch": 1.0384087791495198, |
| "grad_norm": 5.734247996169968, |
| "kl": 0.112548828125, |
| "learning_rate": 6.563636363636363e-07, |
| "loss": 0.0045, |
| "reward": 1.5640625357627869, |
| "reward_std": 0.2916586846113205, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7828125059604645, |
| "step": 378 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.65625, |
| "epoch": 1.0411522633744856, |
| "grad_norm": 3.5003013068620623, |
| "kl": 0.098388671875, |
| "learning_rate": 6.554545454545454e-07, |
| "loss": 0.0039, |
| "reward": 1.4835937023162842, |
| "reward_std": 0.49376851320266724, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7492187023162842, |
| "step": 379 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.578125, |
| "epoch": 1.0438957475994513, |
| "grad_norm": 3.90417718511622, |
| "kl": 0.105712890625, |
| "learning_rate": 6.545454545454546e-07, |
| "loss": 0.0042, |
| "reward": 1.2640624642372131, |
| "reward_std": 0.7536377906799316, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6390624642372131, |
| "step": 380 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.640625, |
| "epoch": 1.046639231824417, |
| "grad_norm": 2.516289316470427, |
| "kl": 0.07958984375, |
| "learning_rate": 6.536363636363636e-07, |
| "loss": 0.0032, |
| "reward": 1.553125023841858, |
| "reward_std": 0.42002078890800476, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7875000238418579, |
| "step": 381 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.046875, |
| "epoch": 1.0493827160493827, |
| "grad_norm": 8.187450982168903, |
| "kl": 0.114013671875, |
| "learning_rate": 6.527272727272727e-07, |
| "loss": 0.0046, |
| "reward": 1.834375023841858, |
| "reward_std": 0.36994873732328415, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9125000238418579, |
| "step": 382 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.90625, |
| "epoch": 1.0521262002743483, |
| "grad_norm": 4.6540049962198795, |
| "kl": 0.10986328125, |
| "learning_rate": 6.518181818181818e-07, |
| "loss": 0.0044, |
| "reward": 1.489062488079071, |
| "reward_std": 0.3555447794497013, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.754687488079071, |
| "step": 383 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.921875, |
| "epoch": 1.0548696844993142, |
| "grad_norm": 2.9326006901858266, |
| "kl": 0.148193359375, |
| "learning_rate": 6.509090909090908e-07, |
| "loss": 0.0059, |
| "reward": 0.985156238079071, |
| "reward_std": 0.43424905836582184, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.5164062082767487, |
| "step": 384 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.421875, |
| "epoch": 1.0576131687242798, |
| "grad_norm": 13.627648816241393, |
| "kl": 0.14697265625, |
| "learning_rate": 6.5e-07, |
| "loss": 0.0059, |
| "reward": 1.3671875, |
| "reward_std": 0.4714278131723404, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6953125, |
| "step": 385 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.59375, |
| "epoch": 1.0603566529492456, |
| "grad_norm": 2.028231122702671, |
| "kl": 0.09228515625, |
| "learning_rate": 6.49090909090909e-07, |
| "loss": 0.0037, |
| "reward": 1.903124988079071, |
| "reward_std": 0.27400387078523636, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.949999988079071, |
| "step": 386 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.859375, |
| "epoch": 1.0631001371742113, |
| "grad_norm": 2.3011736950684885, |
| "kl": 0.121337890625, |
| "learning_rate": 6.481818181818182e-07, |
| "loss": 0.0048, |
| "reward": 1.550000011920929, |
| "reward_std": 0.3014083132147789, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7687499821186066, |
| "step": 387 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.21875, |
| "epoch": 1.0658436213991769, |
| "grad_norm": 4.479044650293866, |
| "kl": 0.152099609375, |
| "learning_rate": 6.472727272727272e-07, |
| "loss": 0.0061, |
| "reward": 1.276562511920929, |
| "reward_std": 0.5062314867973328, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6515624523162842, |
| "step": 388 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.15625, |
| "epoch": 1.0685871056241427, |
| "grad_norm": 6.413333872117701, |
| "kl": 0.095947265625, |
| "learning_rate": 6.463636363636363e-07, |
| "loss": 0.0038, |
| "reward": 1.745312511920929, |
| "reward_std": 0.4144056588411331, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.870312511920929, |
| "step": 389 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.4375, |
| "epoch": 1.0713305898491083, |
| "grad_norm": 3.9246839250901453, |
| "kl": 0.091064453125, |
| "learning_rate": 6.454545454545455e-07, |
| "loss": 0.0036, |
| "reward": 1.639062523841858, |
| "reward_std": 0.5730085074901581, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.7953124940395355, |
| "step": 390 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.84375, |
| "epoch": 1.074074074074074, |
| "grad_norm": 5.849058308896624, |
| "kl": 0.122314453125, |
| "learning_rate": 6.445454545454546e-07, |
| "loss": 0.0049, |
| "reward": 1.5796875357627869, |
| "reward_std": 0.5560208112001419, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7984374761581421, |
| "step": 391 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.171875, |
| "epoch": 1.0768175582990398, |
| "grad_norm": 4.424568502709933, |
| "kl": 0.10693359375, |
| "learning_rate": 6.436363636363636e-07, |
| "loss": 0.0043, |
| "reward": 1.756250023841858, |
| "reward_std": 0.3756431043148041, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8499999940395355, |
| "step": 392 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.09375, |
| "epoch": 1.0795610425240054, |
| "grad_norm": 3.8477460328846846, |
| "kl": 0.1474609375, |
| "learning_rate": 6.427272727272727e-07, |
| "loss": 0.0059, |
| "reward": 1.471875011920929, |
| "reward_std": 0.4663201868534088, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.721875011920929, |
| "step": 393 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.75, |
| "epoch": 1.0823045267489713, |
| "grad_norm": 1.7667673078114647, |
| "kl": 0.14453125, |
| "learning_rate": 6.418181818181818e-07, |
| "loss": 0.0058, |
| "reward": 1.610937476158142, |
| "reward_std": 0.11885568499565125, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8140625059604645, |
| "step": 394 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.015625, |
| "epoch": 1.0850480109739369, |
| "grad_norm": 4.041008655001557, |
| "kl": 0.120849609375, |
| "learning_rate": 6.409090909090908e-07, |
| "loss": 0.0048, |
| "reward": 1.4820312857627869, |
| "reward_std": 0.6050707995891571, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7476562261581421, |
| "step": 395 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.015625, |
| "epoch": 1.0877914951989025, |
| "grad_norm": 3.9773345438051457, |
| "kl": 0.14453125, |
| "learning_rate": 6.4e-07, |
| "loss": 0.0058, |
| "reward": 1.545312523841858, |
| "reward_std": 0.5625161677598953, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7640624940395355, |
| "step": 396 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.375, |
| "epoch": 1.0905349794238683, |
| "grad_norm": 3.6785156309909492, |
| "kl": 0.134765625, |
| "learning_rate": 6.39090909090909e-07, |
| "loss": 0.0054, |
| "reward": 1.5703124403953552, |
| "reward_std": 0.5366352498531342, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7734375, |
| "step": 397 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.234375, |
| "epoch": 1.093278463648834, |
| "grad_norm": 9.913354874557868, |
| "kl": 0.1259765625, |
| "learning_rate": 6.381818181818182e-07, |
| "loss": 0.005, |
| "reward": 1.0429687201976776, |
| "reward_std": 0.3137985020875931, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5429687201976776, |
| "step": 398 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.890625, |
| "epoch": 1.0960219478737998, |
| "grad_norm": 5.590408639861752, |
| "kl": 0.121826171875, |
| "learning_rate": 6.372727272727273e-07, |
| "loss": 0.0049, |
| "reward": 1.198437511920929, |
| "reward_std": 0.7430685758590698, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.604687511920929, |
| "step": 399 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.625, |
| "epoch": 1.0987654320987654, |
| "grad_norm": 5.793826424725683, |
| "kl": 0.124755859375, |
| "learning_rate": 6.363636363636363e-07, |
| "loss": 0.005, |
| "reward": 1.696093738079071, |
| "reward_std": 0.3521786089986563, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.852343738079071, |
| "step": 400 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.4375, |
| "epoch": 1.101508916323731, |
| "grad_norm": 6.137194041317214, |
| "kl": 0.125, |
| "learning_rate": 6.354545454545455e-07, |
| "loss": 0.005, |
| "reward": 1.3140625357627869, |
| "reward_std": 0.519576907157898, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6734374761581421, |
| "step": 401 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.3125, |
| "epoch": 1.1042524005486969, |
| "grad_norm": 4.912481398404394, |
| "kl": 0.130859375, |
| "learning_rate": 6.345454545454545e-07, |
| "loss": 0.0052, |
| "reward": 1.2992187142372131, |
| "reward_std": 0.8708893656730652, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6585937440395355, |
| "step": 402 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.953125, |
| "epoch": 1.1069958847736625, |
| "grad_norm": 4.70721200434814, |
| "kl": 0.122314453125, |
| "learning_rate": 6.336363636363636e-07, |
| "loss": 0.0049, |
| "reward": 1.2437500357627869, |
| "reward_std": 0.4800318330526352, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6343749761581421, |
| "step": 403 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.796875, |
| "epoch": 1.1097393689986284, |
| "grad_norm": 3.8086599476638066, |
| "kl": 0.128173828125, |
| "learning_rate": 6.327272727272727e-07, |
| "loss": 0.0051, |
| "reward": 1.467968761920929, |
| "reward_std": 0.4594145566225052, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.717968761920929, |
| "step": 404 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.75, |
| "epoch": 1.112482853223594, |
| "grad_norm": 7.097538837072882, |
| "kl": 0.154296875, |
| "learning_rate": 6.318181818181818e-07, |
| "loss": 0.0062, |
| "reward": 1.57421875, |
| "reward_std": 0.5617561936378479, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.79296875, |
| "step": 405 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.890625, |
| "epoch": 1.1152263374485596, |
| "grad_norm": 4.337147165160251, |
| "kl": 0.130859375, |
| "learning_rate": 6.309090909090908e-07, |
| "loss": 0.0052, |
| "reward": 1.4664062857627869, |
| "reward_std": 0.38797758519649506, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7320312857627869, |
| "step": 406 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.25, |
| "epoch": 1.1179698216735254, |
| "grad_norm": 11.477377991572434, |
| "kl": 0.4287109375, |
| "learning_rate": 6.3e-07, |
| "loss": 0.0171, |
| "reward": 1.4914062023162842, |
| "reward_std": 0.7070780694484711, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7414062321186066, |
| "step": 407 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.15625, |
| "epoch": 1.120713305898491, |
| "grad_norm": 3.1650276913328623, |
| "kl": 0.1318359375, |
| "learning_rate": 6.290909090909091e-07, |
| "loss": 0.0053, |
| "reward": 1.6742187142372131, |
| "reward_std": 0.44198162853717804, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8304687440395355, |
| "step": 408 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.40625, |
| "epoch": 1.123456790123457, |
| "grad_norm": 3.244376837236869, |
| "kl": 0.097412109375, |
| "learning_rate": 6.281818181818181e-07, |
| "loss": 0.0039, |
| "reward": 1.163281261920929, |
| "reward_std": 0.21971211954951286, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.600781261920929, |
| "step": 409 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.21875, |
| "epoch": 1.1262002743484225, |
| "grad_norm": 119.55222008861053, |
| "kl": 0.14453125, |
| "learning_rate": 6.272727272727273e-07, |
| "loss": 0.0058, |
| "reward": 1.415624976158142, |
| "reward_std": 0.516777977347374, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7124999761581421, |
| "step": 410 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.390625, |
| "epoch": 1.1289437585733881, |
| "grad_norm": 3.217253515690669, |
| "kl": 0.12890625, |
| "learning_rate": 6.263636363636363e-07, |
| "loss": 0.0051, |
| "reward": 1.728906273841858, |
| "reward_std": 0.40882396697998047, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8695312738418579, |
| "step": 411 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 60.390625, |
| "epoch": 1.131687242798354, |
| "grad_norm": 4.822658004657836, |
| "kl": 0.15625, |
| "learning_rate": 6.254545454545455e-07, |
| "loss": 0.0062, |
| "reward": 1.4898437559604645, |
| "reward_std": 0.09943688940256834, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7554687261581421, |
| "step": 412 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 64.484375, |
| "epoch": 1.1344307270233196, |
| "grad_norm": 20.237362408713906, |
| "kl": 0.17138671875, |
| "learning_rate": 6.245454545454545e-07, |
| "loss": 0.0068, |
| "reward": 1.46875, |
| "reward_std": 0.6092181205749512, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.734375, |
| "step": 413 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.515625, |
| "epoch": 1.1371742112482854, |
| "grad_norm": 3.455645129224603, |
| "kl": 0.115478515625, |
| "learning_rate": 6.236363636363636e-07, |
| "loss": 0.0046, |
| "reward": 1.634374976158142, |
| "reward_std": 0.4435206800699234, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8062499761581421, |
| "step": 414 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 65.453125, |
| "epoch": 1.139917695473251, |
| "grad_norm": 6.928489906125386, |
| "kl": 0.14599609375, |
| "learning_rate": 6.227272727272727e-07, |
| "loss": 0.0059, |
| "reward": 1.607812523841858, |
| "reward_std": 0.3787750005722046, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8109374940395355, |
| "step": 415 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.53125, |
| "epoch": 1.1426611796982167, |
| "grad_norm": 4.335808496303622, |
| "kl": 0.16162109375, |
| "learning_rate": 6.218181818181817e-07, |
| "loss": 0.0065, |
| "reward": 1.370312511920929, |
| "reward_std": 0.30234459042549133, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6984374821186066, |
| "step": 416 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.359375, |
| "epoch": 1.1454046639231825, |
| "grad_norm": 21.102134572561926, |
| "kl": 0.1298828125, |
| "learning_rate": 6.209090909090908e-07, |
| "loss": 0.0052, |
| "reward": 1.3117187321186066, |
| "reward_std": 0.25837691873311996, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6710937470197678, |
| "step": 417 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.25, |
| "epoch": 1.1481481481481481, |
| "grad_norm": 4.015658771046256, |
| "kl": 0.14990234375, |
| "learning_rate": 6.2e-07, |
| "loss": 0.006, |
| "reward": 1.609375, |
| "reward_std": 0.46780863404273987, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8125, |
| "step": 418 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.640625, |
| "epoch": 1.1508916323731138, |
| "grad_norm": 5.415690577625519, |
| "kl": 0.1494140625, |
| "learning_rate": 6.190909090909091e-07, |
| "loss": 0.006, |
| "reward": 1.5187499523162842, |
| "reward_std": 0.45385438203811646, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7687499821186066, |
| "step": 419 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.328125, |
| "epoch": 1.1536351165980796, |
| "grad_norm": 3.3292903149711597, |
| "kl": 0.1240234375, |
| "learning_rate": 6.181818181818181e-07, |
| "loss": 0.005, |
| "reward": 1.4609375, |
| "reward_std": 0.6869581341743469, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7421874701976776, |
| "step": 420 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.109375, |
| "epoch": 1.1563786008230452, |
| "grad_norm": 5.649453075128176, |
| "kl": 0.119873046875, |
| "learning_rate": 6.172727272727273e-07, |
| "loss": 0.0048, |
| "reward": 1.642187476158142, |
| "reward_std": 0.28229840844869614, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8296875059604645, |
| "step": 421 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.3125, |
| "epoch": 1.159122085048011, |
| "grad_norm": 4.931504755535921, |
| "kl": 0.12451171875, |
| "learning_rate": 6.163636363636363e-07, |
| "loss": 0.005, |
| "reward": 1.3453125953674316, |
| "reward_std": 0.5185706317424774, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6734375059604645, |
| "step": 422 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.21875, |
| "epoch": 1.1618655692729767, |
| "grad_norm": 4.230390624232728, |
| "kl": 0.14111328125, |
| "learning_rate": 6.154545454545455e-07, |
| "loss": 0.0056, |
| "reward": 1.5476562976837158, |
| "reward_std": 0.6381469368934631, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7820312082767487, |
| "step": 423 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.546875, |
| "epoch": 1.1646090534979423, |
| "grad_norm": 1.9034830148607589, |
| "kl": 0.1240234375, |
| "learning_rate": 6.145454545454545e-07, |
| "loss": 0.005, |
| "reward": 1.682812511920929, |
| "reward_std": 0.15528614819049835, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.823437511920929, |
| "step": 424 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.859375, |
| "epoch": 1.1673525377229081, |
| "grad_norm": 8.461247119068744, |
| "kl": 0.36181640625, |
| "learning_rate": 6.136363636363636e-07, |
| "loss": 0.0144, |
| "reward": 1.6953125, |
| "reward_std": 0.5045858174562454, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8515625, |
| "step": 425 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.65625, |
| "epoch": 1.1700960219478738, |
| "grad_norm": 3.09962122399241, |
| "kl": 0.1318359375, |
| "learning_rate": 6.127272727272727e-07, |
| "loss": 0.0053, |
| "reward": 1.043749988079071, |
| "reward_std": 0.39528100192546844, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.543749988079071, |
| "step": 426 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.171875, |
| "epoch": 1.1728395061728394, |
| "grad_norm": 4.196491201372685, |
| "kl": 0.24853515625, |
| "learning_rate": 6.118181818181818e-07, |
| "loss": 0.01, |
| "reward": 1.4593749642372131, |
| "reward_std": 0.43471772968769073, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7093749642372131, |
| "step": 427 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.6875, |
| "epoch": 1.1755829903978052, |
| "grad_norm": 1.439836627773731, |
| "kl": 0.080810546875, |
| "learning_rate": 6.109090909090909e-07, |
| "loss": 0.0032, |
| "reward": 1.565625011920929, |
| "reward_std": 0.18724938482046127, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.784375011920929, |
| "step": 428 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.296875, |
| "epoch": 1.1783264746227708, |
| "grad_norm": 3.6172862056669377, |
| "kl": 0.11083984375, |
| "learning_rate": 6.1e-07, |
| "loss": 0.0044, |
| "reward": 1.3890624642372131, |
| "reward_std": 0.6481033265590668, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7015625238418579, |
| "step": 429 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.390625, |
| "epoch": 1.1810699588477367, |
| "grad_norm": 5.37902502650951, |
| "kl": 0.096923828125, |
| "learning_rate": 6.090909090909091e-07, |
| "loss": 0.0039, |
| "reward": 1.775781273841858, |
| "reward_std": 0.3665124475955963, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8851562440395355, |
| "step": 430 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.703125, |
| "epoch": 1.1838134430727023, |
| "grad_norm": 4.021069933817606, |
| "kl": 0.122802734375, |
| "learning_rate": 6.081818181818181e-07, |
| "loss": 0.0049, |
| "reward": 1.399218738079071, |
| "reward_std": 0.43452630192041397, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.711718738079071, |
| "step": 431 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.625, |
| "epoch": 1.186556927297668, |
| "grad_norm": 2.371316302774302, |
| "kl": 0.0908203125, |
| "learning_rate": 6.072727272727273e-07, |
| "loss": 0.0036, |
| "reward": 1.251562476158142, |
| "reward_std": 0.22032135352492332, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6265625059604645, |
| "step": 432 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.6875, |
| "epoch": 1.1893004115226338, |
| "grad_norm": 3.562586641125276, |
| "kl": 0.088134765625, |
| "learning_rate": 6.063636363636363e-07, |
| "loss": 0.0035, |
| "reward": 1.3554686903953552, |
| "reward_std": 0.6390049308538437, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.68359375, |
| "step": 433 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.40625, |
| "epoch": 1.1920438957475994, |
| "grad_norm": 3.2369462814293923, |
| "kl": 0.10302734375, |
| "learning_rate": 6.054545454545454e-07, |
| "loss": 0.0041, |
| "reward": 1.5859375, |
| "reward_std": 0.4189769923686981, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7578124403953552, |
| "step": 434 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.71875, |
| "epoch": 1.1947873799725652, |
| "grad_norm": 4.340266580844012, |
| "kl": 0.087646484375, |
| "learning_rate": 6.045454545454545e-07, |
| "loss": 0.0035, |
| "reward": 1.7515625357627869, |
| "reward_std": 0.2581353858113289, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8609375059604645, |
| "step": 435 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.8125, |
| "epoch": 1.1975308641975309, |
| "grad_norm": 4.637561852398335, |
| "kl": 0.080078125, |
| "learning_rate": 6.036363636363636e-07, |
| "loss": 0.0032, |
| "reward": 1.895312488079071, |
| "reward_std": 0.2395406812429428, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.926562488079071, |
| "step": 436 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.84375, |
| "epoch": 1.2002743484224965, |
| "grad_norm": 5.829391306788839, |
| "kl": 0.084228515625, |
| "learning_rate": 6.027272727272728e-07, |
| "loss": 0.0034, |
| "reward": 1.4124999642372131, |
| "reward_std": 0.6100139319896698, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6937500089406967, |
| "step": 437 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.34375, |
| "epoch": 1.2030178326474623, |
| "grad_norm": 2.929133446760945, |
| "kl": 0.07958984375, |
| "learning_rate": 6.018181818181818e-07, |
| "loss": 0.0032, |
| "reward": 1.404687523841858, |
| "reward_std": 0.5406377464532852, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6859374940395355, |
| "step": 438 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.703125, |
| "epoch": 1.205761316872428, |
| "grad_norm": 6.924132587020331, |
| "kl": 0.081787109375, |
| "learning_rate": 6.009090909090909e-07, |
| "loss": 0.0033, |
| "reward": 1.2460936903953552, |
| "reward_std": 0.47603175044059753, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.62109375, |
| "step": 439 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.546875, |
| "epoch": 1.2085048010973938, |
| "grad_norm": 52.78660154107059, |
| "kl": 0.075927734375, |
| "learning_rate": 6e-07, |
| "loss": 0.003, |
| "reward": 1.6765625476837158, |
| "reward_std": 0.3512795567512512, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.817187488079071, |
| "step": 440 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.578125, |
| "epoch": 1.2112482853223594, |
| "grad_norm": 11.455020404299585, |
| "kl": 0.0633544921875, |
| "learning_rate": 5.99090909090909e-07, |
| "loss": 0.0025, |
| "reward": 1.649999976158142, |
| "reward_std": 0.4676620364189148, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.7906250357627869, |
| "step": 441 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.046875, |
| "epoch": 1.213991769547325, |
| "grad_norm": 6.88451765941755, |
| "kl": 0.0638427734375, |
| "learning_rate": 5.981818181818181e-07, |
| "loss": 0.0026, |
| "reward": 1.5304688215255737, |
| "reward_std": 0.43148937821388245, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7492187321186066, |
| "step": 442 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.3125, |
| "epoch": 1.2167352537722909, |
| "grad_norm": 4.809497740022853, |
| "kl": 0.1015625, |
| "learning_rate": 5.972727272727273e-07, |
| "loss": 0.0041, |
| "reward": 1.459375023841858, |
| "reward_std": 0.3682703413069248, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7249999940395355, |
| "step": 443 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.75, |
| "epoch": 1.2194787379972565, |
| "grad_norm": 5.3235611348346, |
| "kl": 0.070068359375, |
| "learning_rate": 5.963636363636363e-07, |
| "loss": 0.0028, |
| "reward": 1.458593726158142, |
| "reward_std": 0.3838815838098526, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7085937559604645, |
| "step": 444 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.09375, |
| "epoch": 1.2222222222222223, |
| "grad_norm": 5.054732585213759, |
| "kl": 0.071044921875, |
| "learning_rate": 5.954545454545454e-07, |
| "loss": 0.0028, |
| "reward": 1.4960938096046448, |
| "reward_std": 0.38986389338970184, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7460937201976776, |
| "step": 445 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.375, |
| "epoch": 1.224965706447188, |
| "grad_norm": 3.7264747718150946, |
| "kl": 0.08642578125, |
| "learning_rate": 5.945454545454546e-07, |
| "loss": 0.0035, |
| "reward": 1.1820312142372131, |
| "reward_std": 0.5168150663375854, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.5882812142372131, |
| "step": 446 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.28125, |
| "epoch": 1.2277091906721536, |
| "grad_norm": 6.107198492961743, |
| "kl": 0.173095703125, |
| "learning_rate": 5.936363636363636e-07, |
| "loss": 0.0069, |
| "reward": 1.6039061546325684, |
| "reward_std": 0.5601920038461685, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7914062440395355, |
| "step": 447 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.125, |
| "epoch": 1.2304526748971194, |
| "grad_norm": 4.458635095195637, |
| "kl": 0.1337890625, |
| "learning_rate": 5.927272727272728e-07, |
| "loss": 0.0053, |
| "reward": 1.262499988079071, |
| "reward_std": 0.4394213408231735, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6375000178813934, |
| "step": 448 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.78125, |
| "epoch": 1.233196159122085, |
| "grad_norm": 4.552607093157277, |
| "kl": 0.08056640625, |
| "learning_rate": 5.918181818181818e-07, |
| "loss": 0.0032, |
| "reward": 1.5703125, |
| "reward_std": 0.4914304316043854, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7734375, |
| "step": 449 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.296875, |
| "epoch": 1.2359396433470509, |
| "grad_norm": 5.07754883381758, |
| "kl": 0.1396484375, |
| "learning_rate": 5.909090909090909e-07, |
| "loss": 0.0056, |
| "reward": 1.2890625, |
| "reward_std": 0.43803659081459045, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6484375, |
| "step": 450 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.578125, |
| "epoch": 1.2386831275720165, |
| "grad_norm": 2.578924195516793, |
| "kl": 0.087890625, |
| "learning_rate": 5.9e-07, |
| "loss": 0.0035, |
| "reward": 1.557812511920929, |
| "reward_std": 0.4418707937002182, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7765624523162842, |
| "step": 451 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.359375, |
| "epoch": 1.241426611796982, |
| "grad_norm": 3.4245702428563156, |
| "kl": 0.10595703125, |
| "learning_rate": 5.89090909090909e-07, |
| "loss": 0.0042, |
| "reward": 1.6875, |
| "reward_std": 0.4009390026330948, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8437500298023224, |
| "step": 452 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.703125, |
| "epoch": 1.244170096021948, |
| "grad_norm": 5.119261486750348, |
| "kl": 0.088623046875, |
| "learning_rate": 5.881818181818181e-07, |
| "loss": 0.0035, |
| "reward": 1.3804687857627869, |
| "reward_std": 0.5968214869499207, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6617187261581421, |
| "step": 453 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.078125, |
| "epoch": 1.2469135802469136, |
| "grad_norm": 6.440970252082927, |
| "kl": 0.11181640625, |
| "learning_rate": 5.872727272727273e-07, |
| "loss": 0.0045, |
| "reward": 1.8210937976837158, |
| "reward_std": 0.4471282362937927, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.899218738079071, |
| "step": 454 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.84375, |
| "epoch": 1.2496570644718794, |
| "grad_norm": 3.8773183626162235, |
| "kl": 0.140625, |
| "learning_rate": 5.863636363636362e-07, |
| "loss": 0.0056, |
| "reward": 1.223437488079071, |
| "reward_std": 0.48718254268169403, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.614062488079071, |
| "step": 455 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.859375, |
| "epoch": 1.252400548696845, |
| "grad_norm": 2.9298753340037322, |
| "kl": 0.097412109375, |
| "learning_rate": 5.854545454545454e-07, |
| "loss": 0.0039, |
| "reward": 1.760937511920929, |
| "reward_std": 0.5081861466169357, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8703124821186066, |
| "step": 456 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.734375, |
| "epoch": 1.2551440329218106, |
| "grad_norm": 3.8792742707571604, |
| "kl": 0.09326171875, |
| "learning_rate": 5.845454545454546e-07, |
| "loss": 0.0037, |
| "reward": 1.75, |
| "reward_std": 0.42434723675251007, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8750000298023224, |
| "step": 457 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.46875, |
| "epoch": 1.2578875171467763, |
| "grad_norm": 3.23678170692346, |
| "kl": 0.09716796875, |
| "learning_rate": 5.836363636363636e-07, |
| "loss": 0.0039, |
| "reward": 1.4023438096046448, |
| "reward_std": 0.38366977870464325, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7148437201976776, |
| "step": 458 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.421875, |
| "epoch": 1.260631001371742, |
| "grad_norm": 6.6456792173455375, |
| "kl": 0.110595703125, |
| "learning_rate": 5.827272727272727e-07, |
| "loss": 0.0044, |
| "reward": 0.8304687440395355, |
| "reward_std": 0.649567037820816, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.43984371423721313, |
| "step": 459 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.0, |
| "epoch": 1.263374485596708, |
| "grad_norm": 2.9585309457636804, |
| "kl": 0.107666015625, |
| "learning_rate": 5.818181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.603906273841858, |
| "reward_std": 0.5155107527971268, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7914062142372131, |
| "step": 460 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.328125, |
| "epoch": 1.2661179698216736, |
| "grad_norm": 6.193455070609818, |
| "kl": 0.1220703125, |
| "learning_rate": 5.809090909090909e-07, |
| "loss": 0.0049, |
| "reward": 1.3578124642372131, |
| "reward_std": 0.6153180003166199, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 461 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.046875, |
| "epoch": 1.2688614540466392, |
| "grad_norm": 6.112679464169271, |
| "kl": 0.14990234375, |
| "learning_rate": 5.8e-07, |
| "loss": 0.006, |
| "reward": 1.241406261920929, |
| "reward_std": 0.4134686142206192, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6320312321186066, |
| "step": 462 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.53125, |
| "epoch": 1.2716049382716048, |
| "grad_norm": 2.9616624839056933, |
| "kl": 0.1162109375, |
| "learning_rate": 5.79090909090909e-07, |
| "loss": 0.0047, |
| "reward": 1.4453125, |
| "reward_std": 0.15817427728325129, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7265624701976776, |
| "step": 463 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.078125, |
| "epoch": 1.2743484224965707, |
| "grad_norm": 7.09015056585456, |
| "kl": 0.115234375, |
| "learning_rate": 5.781818181818181e-07, |
| "loss": 0.0046, |
| "reward": 1.2546875476837158, |
| "reward_std": 0.2974911257624626, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.645312488079071, |
| "step": 464 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.78125, |
| "epoch": 1.2770919067215363, |
| "grad_norm": 2.6284994699392006, |
| "kl": 0.13232421875, |
| "learning_rate": 5.772727272727273e-07, |
| "loss": 0.0053, |
| "reward": 1.811718761920929, |
| "reward_std": 0.3243738263845444, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9054687917232513, |
| "step": 465 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.484375, |
| "epoch": 1.2798353909465021, |
| "grad_norm": 4.86271779024084, |
| "kl": 0.12646484375, |
| "learning_rate": 5.763636363636363e-07, |
| "loss": 0.0051, |
| "reward": 1.1578125059604645, |
| "reward_std": 0.38512764871120453, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5953124612569809, |
| "step": 466 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.0625, |
| "epoch": 1.2825788751714677, |
| "grad_norm": 12.418372543140359, |
| "kl": 0.115478515625, |
| "learning_rate": 5.754545454545454e-07, |
| "loss": 0.0046, |
| "reward": 0.895312488079071, |
| "reward_std": 0.3500541225075722, |
| "rewards/choice_reward": 0.421875, |
| "rewards/roam_reward": 0.47343748807907104, |
| "step": 467 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.078125, |
| "epoch": 1.2853223593964334, |
| "grad_norm": 15.054577189255165, |
| "kl": 0.10546875, |
| "learning_rate": 5.745454545454546e-07, |
| "loss": 0.0042, |
| "reward": 1.7906250357627869, |
| "reward_std": 0.4052163362503052, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8843750059604645, |
| "step": 468 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.109375, |
| "epoch": 1.2880658436213992, |
| "grad_norm": 0.23607626191653258, |
| "kl": 0.09375, |
| "learning_rate": 5.736363636363636e-07, |
| "loss": 0.0037, |
| "reward": 2.0, |
| "reward_std": 0.0, |
| "rewards/choice_reward": 1.0, |
| "rewards/roam_reward": 1.0, |
| "step": 469 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.640625, |
| "epoch": 1.2908093278463648, |
| "grad_norm": 5.63969040120091, |
| "kl": 0.10107421875, |
| "learning_rate": 5.727272727272727e-07, |
| "loss": 0.004, |
| "reward": 1.323437511920929, |
| "reward_std": 0.5278631448745728, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.651562511920929, |
| "step": 470 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.796875, |
| "epoch": 1.2935528120713307, |
| "grad_norm": 4.383849861820226, |
| "kl": 0.13671875, |
| "learning_rate": 5.718181818181818e-07, |
| "loss": 0.0055, |
| "reward": 1.465624988079071, |
| "reward_std": 0.42359398305416107, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.731249988079071, |
| "step": 471 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.65625, |
| "epoch": 1.2962962962962963, |
| "grad_norm": 3.288610200800226, |
| "kl": 0.111083984375, |
| "learning_rate": 5.709090909090909e-07, |
| "loss": 0.0044, |
| "reward": 1.7421875, |
| "reward_std": 0.44370877742767334, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8671875298023224, |
| "step": 472 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.703125, |
| "epoch": 1.299039780521262, |
| "grad_norm": 9.827746624272217, |
| "kl": 0.101318359375, |
| "learning_rate": 5.699999999999999e-07, |
| "loss": 0.0041, |
| "reward": 1.074999988079071, |
| "reward_std": 0.5073258280754089, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.559374988079071, |
| "step": 473 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.015625, |
| "epoch": 1.3017832647462277, |
| "grad_norm": 3.348360712200408, |
| "kl": 0.13037109375, |
| "learning_rate": 5.69090909090909e-07, |
| "loss": 0.0052, |
| "reward": 1.087499976158142, |
| "reward_std": 0.6843429207801819, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5562499761581421, |
| "step": 474 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.609375, |
| "epoch": 1.3045267489711934, |
| "grad_norm": 14.093744111040772, |
| "kl": 0.12451171875, |
| "learning_rate": 5.681818181818182e-07, |
| "loss": 0.005, |
| "reward": 1.4609375, |
| "reward_std": 0.5192954540252686, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7421875, |
| "step": 475 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.65625, |
| "epoch": 1.3072702331961592, |
| "grad_norm": 9.855896100482207, |
| "kl": 0.13916015625, |
| "learning_rate": 5.672727272727273e-07, |
| "loss": 0.0056, |
| "reward": 1.60546875, |
| "reward_std": 0.3087161513976753, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.80859375, |
| "step": 476 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.953125, |
| "epoch": 1.3100137174211248, |
| "grad_norm": 3.842627802662204, |
| "kl": 0.106689453125, |
| "learning_rate": 5.663636363636363e-07, |
| "loss": 0.0043, |
| "reward": 1.668749988079071, |
| "reward_std": 0.5410042703151703, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.840624988079071, |
| "step": 477 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.625, |
| "epoch": 1.3127572016460904, |
| "grad_norm": 3.1006416439220086, |
| "kl": 0.111328125, |
| "learning_rate": 5.654545454545454e-07, |
| "loss": 0.0045, |
| "reward": 1.7156250476837158, |
| "reward_std": 0.13258251920342445, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.856249988079071, |
| "step": 478 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.25, |
| "epoch": 1.3155006858710563, |
| "grad_norm": 1.897058956293695, |
| "kl": 0.110107421875, |
| "learning_rate": 5.645454545454546e-07, |
| "loss": 0.0044, |
| "reward": 1.996874988079071, |
| "reward_std": 0.008838835172355175, |
| "rewards/choice_reward": 1.0, |
| "rewards/roam_reward": 0.996874988079071, |
| "step": 479 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.75, |
| "epoch": 1.318244170096022, |
| "grad_norm": 4.35448032695745, |
| "kl": 0.11328125, |
| "learning_rate": 5.636363636363635e-07, |
| "loss": 0.0045, |
| "reward": 1.485937476158142, |
| "reward_std": 0.5335795134305954, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7515624463558197, |
| "step": 480 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.484375, |
| "epoch": 1.3209876543209877, |
| "grad_norm": 7.093619038036287, |
| "kl": 0.124755859375, |
| "learning_rate": 5.627272727272727e-07, |
| "loss": 0.005, |
| "reward": 1.188281238079071, |
| "reward_std": 0.4797893762588501, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.610156238079071, |
| "step": 481 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.0, |
| "epoch": 1.3237311385459534, |
| "grad_norm": 3.5193018775527736, |
| "kl": 0.110595703125, |
| "learning_rate": 5.618181818181818e-07, |
| "loss": 0.0044, |
| "reward": 1.8015625476837158, |
| "reward_std": 0.4114743173122406, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.879687488079071, |
| "step": 482 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.140625, |
| "epoch": 1.326474622770919, |
| "grad_norm": 4.831405820614919, |
| "kl": 0.11328125, |
| "learning_rate": 5.609090909090909e-07, |
| "loss": 0.0045, |
| "reward": 1.818750023841858, |
| "reward_std": 0.36465759575366974, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.8968750238418579, |
| "step": 483 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.890625, |
| "epoch": 1.3292181069958848, |
| "grad_norm": 7.379749718313667, |
| "kl": 0.12548828125, |
| "learning_rate": 5.6e-07, |
| "loss": 0.005, |
| "reward": 1.2062499523162842, |
| "reward_std": 0.5794805586338043, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6124999523162842, |
| "step": 484 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.359375, |
| "epoch": 1.3319615912208504, |
| "grad_norm": 2.487766952563411, |
| "kl": 0.116455078125, |
| "learning_rate": 5.590909090909091e-07, |
| "loss": 0.0047, |
| "reward": 1.7109375, |
| "reward_std": 0.2527294456958771, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8515624701976776, |
| "step": 485 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.15625, |
| "epoch": 1.3347050754458163, |
| "grad_norm": 9.399922702963035, |
| "kl": 0.103515625, |
| "learning_rate": 5.581818181818182e-07, |
| "loss": 0.0041, |
| "reward": 1.6734375357627869, |
| "reward_std": 0.3038512021303177, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8453125357627869, |
| "step": 486 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.046875, |
| "epoch": 1.337448559670782, |
| "grad_norm": 8.353208322666267, |
| "kl": 0.130126953125, |
| "learning_rate": 5.572727272727273e-07, |
| "loss": 0.0052, |
| "reward": 1.251562476158142, |
| "reward_std": 0.5889841318130493, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6421875059604645, |
| "step": 487 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.0, |
| "epoch": 1.3401920438957475, |
| "grad_norm": 4.434212844778169, |
| "kl": 0.120849609375, |
| "learning_rate": 5.563636363636363e-07, |
| "loss": 0.0048, |
| "reward": 1.403124988079071, |
| "reward_std": 0.5678224265575409, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.715624988079071, |
| "step": 488 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.84375, |
| "epoch": 1.3429355281207134, |
| "grad_norm": 5.101295400485607, |
| "kl": 0.11669921875, |
| "learning_rate": 5.554545454545454e-07, |
| "loss": 0.0047, |
| "reward": 1.584375023841858, |
| "reward_std": 0.23689022660255432, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.8031249940395355, |
| "step": 489 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.078125, |
| "epoch": 1.345679012345679, |
| "grad_norm": 10.778408611138774, |
| "kl": 0.16162109375, |
| "learning_rate": 5.545454545454546e-07, |
| "loss": 0.0065, |
| "reward": 1.393750011920929, |
| "reward_std": 0.4552782475948334, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7062499821186066, |
| "step": 490 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.140625, |
| "epoch": 1.3484224965706448, |
| "grad_norm": 2.9983226699679957, |
| "kl": 0.093017578125, |
| "learning_rate": 5.536363636363635e-07, |
| "loss": 0.0037, |
| "reward": 1.3453125357627869, |
| "reward_std": 0.44656118750572205, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6890625059604645, |
| "step": 491 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.453125, |
| "epoch": 1.3511659807956105, |
| "grad_norm": 3.350677952118022, |
| "kl": 0.104736328125, |
| "learning_rate": 5.527272727272727e-07, |
| "loss": 0.0042, |
| "reward": 1.581250011920929, |
| "reward_std": 0.4546556994318962, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7999999523162842, |
| "step": 492 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.453125, |
| "epoch": 1.353909465020576, |
| "grad_norm": 4.15019426566186, |
| "kl": 0.103515625, |
| "learning_rate": 5.518181818181818e-07, |
| "loss": 0.0042, |
| "reward": 1.495312511920929, |
| "reward_std": 0.45879365503787994, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7609374821186066, |
| "step": 493 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.328125, |
| "epoch": 1.356652949245542, |
| "grad_norm": 9.8280328163177, |
| "kl": 0.117431640625, |
| "learning_rate": 5.509090909090909e-07, |
| "loss": 0.0047, |
| "reward": 1.5812499523162842, |
| "reward_std": 0.5516014248132706, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7999999821186066, |
| "step": 494 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.71875, |
| "epoch": 1.3593964334705075, |
| "grad_norm": 8.289229537545548, |
| "kl": 0.1904296875, |
| "learning_rate": 5.5e-07, |
| "loss": 0.0076, |
| "reward": 1.3085936903953552, |
| "reward_std": 0.4198668450117111, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.66796875, |
| "step": 495 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.34375, |
| "epoch": 1.3621399176954734, |
| "grad_norm": 5.25299152931547, |
| "kl": 0.123779296875, |
| "learning_rate": 5.490909090909091e-07, |
| "loss": 0.005, |
| "reward": 1.590624988079071, |
| "reward_std": 0.4451471269130707, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.793749988079071, |
| "step": 496 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.75, |
| "epoch": 1.364883401920439, |
| "grad_norm": 5.404401168995441, |
| "kl": 0.18359375, |
| "learning_rate": 5.481818181818182e-07, |
| "loss": 0.0073, |
| "reward": 1.2242187857627869, |
| "reward_std": 0.17593394219875336, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6304687559604645, |
| "step": 497 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.796875, |
| "epoch": 1.3676268861454046, |
| "grad_norm": 8.571801986242038, |
| "kl": 0.101806640625, |
| "learning_rate": 5.472727272727272e-07, |
| "loss": 0.0041, |
| "reward": 1.612500011920929, |
| "reward_std": 0.3662673383951187, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.815625011920929, |
| "step": 498 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.9375, |
| "epoch": 1.3703703703703702, |
| "grad_norm": 5.118638981200302, |
| "kl": 0.125732421875, |
| "learning_rate": 5.463636363636363e-07, |
| "loss": 0.005, |
| "reward": 1.323437511920929, |
| "reward_std": 0.5765286982059479, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6671874821186066, |
| "step": 499 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.75, |
| "epoch": 1.373113854595336, |
| "grad_norm": 2.610331549333793, |
| "kl": 0.100341796875, |
| "learning_rate": 5.454545454545454e-07, |
| "loss": 0.004, |
| "reward": 1.524999976158142, |
| "reward_std": 0.2198822721838951, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7750000059604645, |
| "step": 500 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.875, |
| "epoch": 1.3758573388203017, |
| "grad_norm": 3.028713970445138, |
| "kl": 0.09423828125, |
| "learning_rate": 5.445454545454546e-07, |
| "loss": 0.0038, |
| "reward": 1.2359374463558197, |
| "reward_std": 0.47837182879447937, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6265625059604645, |
| "step": 501 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.234375, |
| "epoch": 1.3786008230452675, |
| "grad_norm": 5.823742179238196, |
| "kl": 0.119140625, |
| "learning_rate": 5.436363636363635e-07, |
| "loss": 0.0048, |
| "reward": 1.948437511920929, |
| "reward_std": 0.13620062451809645, |
| "rewards/choice_reward": 0.984375, |
| "rewards/roam_reward": 0.964062511920929, |
| "step": 502 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.65625, |
| "epoch": 1.3813443072702332, |
| "grad_norm": 3.6877422509468816, |
| "kl": 0.0947265625, |
| "learning_rate": 5.427272727272727e-07, |
| "loss": 0.0038, |
| "reward": 1.8132812976837158, |
| "reward_std": 0.40527141094207764, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.8914062678813934, |
| "step": 503 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.5625, |
| "epoch": 1.3840877914951988, |
| "grad_norm": 3.7952095790573237, |
| "kl": 0.097900390625, |
| "learning_rate": 5.418181818181819e-07, |
| "loss": 0.0039, |
| "reward": 1.39453125, |
| "reward_std": 0.6221335232257843, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7070312201976776, |
| "step": 504 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.796875, |
| "epoch": 1.3868312757201646, |
| "grad_norm": 2.987830267870743, |
| "kl": 0.100341796875, |
| "learning_rate": 5.409090909090908e-07, |
| "loss": 0.004, |
| "reward": 1.517187476158142, |
| "reward_std": 0.38894159346818924, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7671874761581421, |
| "step": 505 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.390625, |
| "epoch": 1.3895747599451302, |
| "grad_norm": 2.7113804376330464, |
| "kl": 0.116455078125, |
| "learning_rate": 5.4e-07, |
| "loss": 0.0047, |
| "reward": 1.649999976158142, |
| "reward_std": 0.2602013275027275, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8218749761581421, |
| "step": 506 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.765625, |
| "epoch": 1.392318244170096, |
| "grad_norm": 3.3082946283729946, |
| "kl": 0.123046875, |
| "learning_rate": 5.390909090909091e-07, |
| "loss": 0.0049, |
| "reward": 1.7109375, |
| "reward_std": 0.4046720564365387, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8515625, |
| "step": 507 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.859375, |
| "epoch": 1.3950617283950617, |
| "grad_norm": 2.647995577478945, |
| "kl": 0.09521484375, |
| "learning_rate": 5.381818181818182e-07, |
| "loss": 0.0038, |
| "reward": 1.832812488079071, |
| "reward_std": 0.4148809313774109, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9109375178813934, |
| "step": 508 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.375, |
| "epoch": 1.3978052126200273, |
| "grad_norm": 4.815418563626276, |
| "kl": 0.16552734375, |
| "learning_rate": 5.372727272727272e-07, |
| "loss": 0.0066, |
| "reward": 1.53125, |
| "reward_std": 0.1755305677652359, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.765625, |
| "step": 509 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.640625, |
| "epoch": 1.4005486968449932, |
| "grad_norm": 3.8831795024860827, |
| "kl": 0.0830078125, |
| "learning_rate": 5.363636363636363e-07, |
| "loss": 0.0033, |
| "reward": 1.4921874403953552, |
| "reward_std": 0.37759263068437576, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7578124701976776, |
| "step": 510 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.59375, |
| "epoch": 1.4032921810699588, |
| "grad_norm": 2.928606079305618, |
| "kl": 0.09619140625, |
| "learning_rate": 5.354545454545454e-07, |
| "loss": 0.0039, |
| "reward": 1.669531226158142, |
| "reward_std": 0.2422011522576213, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8414062261581421, |
| "step": 511 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.625, |
| "epoch": 1.4060356652949246, |
| "grad_norm": 13.053683686361019, |
| "kl": 0.11669921875, |
| "learning_rate": 5.345454545454545e-07, |
| "loss": 0.0047, |
| "reward": 1.6304687857627869, |
| "reward_std": 0.5575891137123108, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8179687559604645, |
| "step": 512 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.671875, |
| "epoch": 1.4087791495198903, |
| "grad_norm": 5.3727465109473975, |
| "kl": 0.10791015625, |
| "learning_rate": 5.336363636363636e-07, |
| "loss": 0.0043, |
| "reward": 1.6382812857627869, |
| "reward_std": 0.43155744671821594, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8101562559604645, |
| "step": 513 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.859375, |
| "epoch": 1.4115226337448559, |
| "grad_norm": 3.641029486434777, |
| "kl": 0.10205078125, |
| "learning_rate": 5.327272727272727e-07, |
| "loss": 0.0041, |
| "reward": 1.76953125, |
| "reward_std": 0.36117296665906906, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.87890625, |
| "step": 514 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.21875, |
| "epoch": 1.4142661179698217, |
| "grad_norm": 6.256028372574312, |
| "kl": 0.09130859375, |
| "learning_rate": 5.318181818181819e-07, |
| "loss": 0.0037, |
| "reward": 1.603124976158142, |
| "reward_std": 0.4715536832809448, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7906250059604645, |
| "step": 515 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.34375, |
| "epoch": 1.4170096021947873, |
| "grad_norm": 3.5855068495143723, |
| "kl": 0.111328125, |
| "learning_rate": 5.309090909090908e-07, |
| "loss": 0.0045, |
| "reward": 1.475000023841858, |
| "reward_std": 0.37462590634822845, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7406249940395355, |
| "step": 516 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.328125, |
| "epoch": 1.4197530864197532, |
| "grad_norm": 13.345238848509045, |
| "kl": 0.09375, |
| "learning_rate": 5.3e-07, |
| "loss": 0.0038, |
| "reward": 1.3984374403953552, |
| "reward_std": 0.5894862413406372, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7109375, |
| "step": 517 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.609375, |
| "epoch": 1.4224965706447188, |
| "grad_norm": 3.487109567518966, |
| "kl": 0.110107421875, |
| "learning_rate": 5.290909090909091e-07, |
| "loss": 0.0044, |
| "reward": 1.216406226158142, |
| "reward_std": 0.5684505552053452, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.5914062559604645, |
| "step": 518 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.78125, |
| "epoch": 1.4252400548696844, |
| "grad_norm": 3.4179403170394216, |
| "kl": 0.110595703125, |
| "learning_rate": 5.281818181818181e-07, |
| "loss": 0.0044, |
| "reward": 1.489062488079071, |
| "reward_std": 0.4661572128534317, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.739062488079071, |
| "step": 519 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.328125, |
| "epoch": 1.4279835390946503, |
| "grad_norm": 4.304257394593773, |
| "kl": 0.09521484375, |
| "learning_rate": 5.272727272727272e-07, |
| "loss": 0.0038, |
| "reward": 1.431249976158142, |
| "reward_std": 0.5100004374980927, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7281250059604645, |
| "step": 520 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.328125, |
| "epoch": 1.4307270233196159, |
| "grad_norm": 7.649430043436629, |
| "kl": 0.098388671875, |
| "learning_rate": 5.263636363636363e-07, |
| "loss": 0.0039, |
| "reward": 1.7992187142372131, |
| "reward_std": 0.2976684421300888, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8929687440395355, |
| "step": 521 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.5625, |
| "epoch": 1.4334705075445817, |
| "grad_norm": 5.650031888030248, |
| "kl": 0.1015625, |
| "learning_rate": 5.254545454545455e-07, |
| "loss": 0.0041, |
| "reward": 1.44140625, |
| "reward_std": 0.4561328738927841, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7226562201976776, |
| "step": 522 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.671875, |
| "epoch": 1.4362139917695473, |
| "grad_norm": 4.660635426861998, |
| "kl": 0.0869140625, |
| "learning_rate": 5.245454545454545e-07, |
| "loss": 0.0035, |
| "reward": 1.7624999284744263, |
| "reward_std": 0.3338359519839287, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.887499988079071, |
| "step": 523 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.046875, |
| "epoch": 1.438957475994513, |
| "grad_norm": 2.6979251782646902, |
| "kl": 0.104248046875, |
| "learning_rate": 5.236363636363636e-07, |
| "loss": 0.0042, |
| "reward": 1.9203124642372131, |
| "reward_std": 0.22539027780294418, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.9515624940395355, |
| "step": 524 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.5, |
| "epoch": 1.4417009602194788, |
| "grad_norm": 18.90388830504221, |
| "kl": 0.096923828125, |
| "learning_rate": 5.227272727272727e-07, |
| "loss": 0.0039, |
| "reward": 1.3968749642372131, |
| "reward_std": 0.4366264343261719, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7093750238418579, |
| "step": 525 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.328125, |
| "epoch": 1.4444444444444444, |
| "grad_norm": 3.489414046490887, |
| "kl": 0.088134765625, |
| "learning_rate": 5.218181818181819e-07, |
| "loss": 0.0035, |
| "reward": 1.8656249642372131, |
| "reward_std": 0.25108102709054947, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9281250238418579, |
| "step": 526 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.28125, |
| "epoch": 1.4471879286694103, |
| "grad_norm": 5.09523654572199, |
| "kl": 0.15087890625, |
| "learning_rate": 5.209090909090908e-07, |
| "loss": 0.006, |
| "reward": 1.4765625, |
| "reward_std": 0.4231499135494232, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7421875298023224, |
| "step": 527 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.578125, |
| "epoch": 1.4499314128943759, |
| "grad_norm": 2.7909099615882034, |
| "kl": 0.1279296875, |
| "learning_rate": 5.2e-07, |
| "loss": 0.0051, |
| "reward": 1.658593773841858, |
| "reward_std": 0.272448793053627, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8304687440395355, |
| "step": 528 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.875, |
| "epoch": 1.4526748971193415, |
| "grad_norm": 4.183851213619129, |
| "kl": 0.096923828125, |
| "learning_rate": 5.190909090909091e-07, |
| "loss": 0.0039, |
| "reward": 0.7804687321186066, |
| "reward_std": 0.5706401765346527, |
| "rewards/choice_reward": 0.359375, |
| "rewards/roam_reward": 0.42109374701976776, |
| "step": 529 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.203125, |
| "epoch": 1.4554183813443073, |
| "grad_norm": 5.202662608016372, |
| "kl": 0.11962890625, |
| "learning_rate": 5.181818181818181e-07, |
| "loss": 0.0048, |
| "reward": 1.42578125, |
| "reward_std": 0.5004200339317322, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.72265625, |
| "step": 530 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.875, |
| "epoch": 1.458161865569273, |
| "grad_norm": 2.784816392977025, |
| "kl": 0.11474609375, |
| "learning_rate": 5.172727272727272e-07, |
| "loss": 0.0046, |
| "reward": 1.532812476158142, |
| "reward_std": 0.24656263273209333, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7671875059604645, |
| "step": 531 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.796875, |
| "epoch": 1.4609053497942388, |
| "grad_norm": 2.9087030973929977, |
| "kl": 0.088134765625, |
| "learning_rate": 5.163636363636364e-07, |
| "loss": 0.0035, |
| "reward": 1.133593738079071, |
| "reward_std": 0.463393434882164, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5867187082767487, |
| "step": 532 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.4375, |
| "epoch": 1.4636488340192044, |
| "grad_norm": 3.1964784565555124, |
| "kl": 0.107177734375, |
| "learning_rate": 5.154545454545455e-07, |
| "loss": 0.0043, |
| "reward": 1.903906226158142, |
| "reward_std": 0.13558390457183123, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9507812559604645, |
| "step": 533 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.171875, |
| "epoch": 1.46639231824417, |
| "grad_norm": 3.9945355817322614, |
| "kl": 0.113037109375, |
| "learning_rate": 5.145454545454545e-07, |
| "loss": 0.0045, |
| "reward": 1.04296875, |
| "reward_std": 0.40418025851249695, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5429687201976776, |
| "step": 534 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.25, |
| "epoch": 1.4691358024691357, |
| "grad_norm": 8.348785816730594, |
| "kl": 0.09619140625, |
| "learning_rate": 5.136363636363636e-07, |
| "loss": 0.0039, |
| "reward": 1.46875, |
| "reward_std": 0.5412850975990295, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7343749701976776, |
| "step": 535 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.609375, |
| "epoch": 1.4718792866941015, |
| "grad_norm": 7.640760240916528, |
| "kl": 0.115478515625, |
| "learning_rate": 5.127272727272727e-07, |
| "loss": 0.0046, |
| "reward": 1.2687499523162842, |
| "reward_std": 0.7544959485530853, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6437499970197678, |
| "step": 536 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.84375, |
| "epoch": 1.4746227709190673, |
| "grad_norm": 4.266240020097782, |
| "kl": 0.1005859375, |
| "learning_rate": 5.118181818181818e-07, |
| "loss": 0.004, |
| "reward": 1.295312523841858, |
| "reward_std": 0.5077711343765259, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6546874940395355, |
| "step": 537 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.359375, |
| "epoch": 1.477366255144033, |
| "grad_norm": 3.923833439442307, |
| "kl": 0.11181640625, |
| "learning_rate": 5.109090909090908e-07, |
| "loss": 0.0045, |
| "reward": 1.323437511920929, |
| "reward_std": 0.4369395971298218, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6359374821186066, |
| "step": 538 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.0625, |
| "epoch": 1.4801097393689986, |
| "grad_norm": 2.857980806131285, |
| "kl": 0.1064453125, |
| "learning_rate": 5.1e-07, |
| "loss": 0.0042, |
| "reward": 1.561718761920929, |
| "reward_std": 0.4069434851408005, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.780468761920929, |
| "step": 539 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.140625, |
| "epoch": 1.4828532235939642, |
| "grad_norm": 12.797512293581908, |
| "kl": 0.1123046875, |
| "learning_rate": 5.090909090909091e-07, |
| "loss": 0.0045, |
| "reward": 1.63671875, |
| "reward_std": 0.3925085961818695, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.82421875, |
| "step": 540 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.734375, |
| "epoch": 1.48559670781893, |
| "grad_norm": 2.833671268147599, |
| "kl": 0.10400390625, |
| "learning_rate": 5.081818181818181e-07, |
| "loss": 0.0042, |
| "reward": 1.376562476158142, |
| "reward_std": 0.40079691261053085, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.7046875059604645, |
| "step": 541 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.03125, |
| "epoch": 1.4883401920438957, |
| "grad_norm": 3.82926451926805, |
| "kl": 0.089599609375, |
| "learning_rate": 5.072727272727273e-07, |
| "loss": 0.0036, |
| "reward": 1.091406226158142, |
| "reward_std": 0.6246378421783447, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5601562261581421, |
| "step": 542 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.453125, |
| "epoch": 1.4910836762688615, |
| "grad_norm": 3.7490085338944876, |
| "kl": 0.143798828125, |
| "learning_rate": 5.063636363636364e-07, |
| "loss": 0.0058, |
| "reward": 1.0445312559604645, |
| "reward_std": 0.3862235099077225, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5445312261581421, |
| "step": 543 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.0625, |
| "epoch": 1.4938271604938271, |
| "grad_norm": 9.781717240616556, |
| "kl": 0.10400390625, |
| "learning_rate": 5.054545454545454e-07, |
| "loss": 0.0042, |
| "reward": 1.1070312559604645, |
| "reward_std": 0.45780712366104126, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5757812559604645, |
| "step": 544 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.578125, |
| "epoch": 1.4965706447187928, |
| "grad_norm": 2.6181307764218715, |
| "kl": 0.123291015625, |
| "learning_rate": 5.045454545454545e-07, |
| "loss": 0.0049, |
| "reward": 1.890625, |
| "reward_std": 0.30935921519994736, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9375000298023224, |
| "step": 545 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.953125, |
| "epoch": 1.4993141289437586, |
| "grad_norm": 6.164237942689379, |
| "kl": 0.101318359375, |
| "learning_rate": 5.036363636363636e-07, |
| "loss": 0.0041, |
| "reward": 0.960156261920929, |
| "reward_std": 0.2131277248263359, |
| "rewards/choice_reward": 0.453125, |
| "rewards/roam_reward": 0.5070312321186066, |
| "step": 546 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.765625, |
| "epoch": 1.5020576131687244, |
| "grad_norm": 8.03304964733052, |
| "kl": 0.12548828125, |
| "learning_rate": 5.027272727272727e-07, |
| "loss": 0.005, |
| "reward": 1.434374988079071, |
| "reward_std": 0.6121031939983368, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.731249988079071, |
| "step": 547 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.03125, |
| "epoch": 1.50480109739369, |
| "grad_norm": 17.883401277948902, |
| "kl": 0.115234375, |
| "learning_rate": 5.018181818181818e-07, |
| "loss": 0.0046, |
| "reward": 1.8890624642372131, |
| "reward_std": 0.18478976469486952, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9359374940395355, |
| "step": 548 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.625, |
| "epoch": 1.5075445816186557, |
| "grad_norm": 4.871786541222827, |
| "kl": 0.11083984375, |
| "learning_rate": 5.009090909090908e-07, |
| "loss": 0.0044, |
| "reward": 1.75390625, |
| "reward_std": 0.3689107820391655, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8789062201976776, |
| "step": 549 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.421875, |
| "epoch": 1.5102880658436213, |
| "grad_norm": 11.945953118265406, |
| "kl": 0.09521484375, |
| "learning_rate": 5e-07, |
| "loss": 0.0038, |
| "reward": 1.5843749642372131, |
| "reward_std": 0.41377370804548264, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.8031250238418579, |
| "step": 550 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.578125, |
| "epoch": 1.5130315500685871, |
| "grad_norm": 3.2427799757731357, |
| "kl": 0.0966796875, |
| "learning_rate": 4.99090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.5499999523162842, |
| "reward_std": 0.09355210792273283, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7843749523162842, |
| "step": 551 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.765625, |
| "epoch": 1.5157750342935528, |
| "grad_norm": 27.58114087642646, |
| "kl": 0.114501953125, |
| "learning_rate": 4.981818181818181e-07, |
| "loss": 0.0046, |
| "reward": 1.536718726158142, |
| "reward_std": 0.6080199331045151, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7554687261581421, |
| "step": 552 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.921875, |
| "epoch": 1.5185185185185186, |
| "grad_norm": 3.117876526704857, |
| "kl": 0.115966796875, |
| "learning_rate": 4.972727272727272e-07, |
| "loss": 0.0046, |
| "reward": 1.732812523841858, |
| "reward_std": 0.25190678983926773, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8734375238418579, |
| "step": 553 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.5, |
| "epoch": 1.5212620027434842, |
| "grad_norm": 4.933936379602721, |
| "kl": 0.1103515625, |
| "learning_rate": 4.963636363636364e-07, |
| "loss": 0.0044, |
| "reward": 1.703125, |
| "reward_std": 0.3168279826641083, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.859375, |
| "step": 554 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.6875, |
| "epoch": 1.5240054869684498, |
| "grad_norm": 4.628806200489313, |
| "kl": 0.16357421875, |
| "learning_rate": 4.954545454545454e-07, |
| "loss": 0.0065, |
| "reward": 1.431249976158142, |
| "reward_std": 0.47491100430488586, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.6968749910593033, |
| "step": 555 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.984375, |
| "epoch": 1.5267489711934157, |
| "grad_norm": 10.569874783958957, |
| "kl": 0.11279296875, |
| "learning_rate": 4.945454545454545e-07, |
| "loss": 0.0045, |
| "reward": 1.254687488079071, |
| "reward_std": 0.4676324874162674, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6453124582767487, |
| "step": 556 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.96875, |
| "epoch": 1.5294924554183813, |
| "grad_norm": 23.02839701452903, |
| "kl": 0.15673828125, |
| "learning_rate": 4.936363636363636e-07, |
| "loss": 0.0063, |
| "reward": 1.5046875476837158, |
| "reward_std": 0.650595635175705, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7546875178813934, |
| "step": 557 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.796875, |
| "epoch": 1.5322359396433471, |
| "grad_norm": 3.2318566693408313, |
| "kl": 0.14208984375, |
| "learning_rate": 4.927272727272728e-07, |
| "loss": 0.0057, |
| "reward": 1.3976562023162842, |
| "reward_std": 0.2317863437347114, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7101562321186066, |
| "step": 558 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.484375, |
| "epoch": 1.5349794238683128, |
| "grad_norm": 6.730100046084902, |
| "kl": 0.13916015625, |
| "learning_rate": 4.918181818181818e-07, |
| "loss": 0.0056, |
| "reward": 1.1320313215255737, |
| "reward_std": 0.539662778377533, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5851562321186066, |
| "step": 559 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.71875, |
| "epoch": 1.5377229080932784, |
| "grad_norm": 4.391655190501676, |
| "kl": 0.10498046875, |
| "learning_rate": 4.909090909090909e-07, |
| "loss": 0.0042, |
| "reward": 1.609375, |
| "reward_std": 0.4833652973175049, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7968749701976776, |
| "step": 560 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.40625, |
| "epoch": 1.540466392318244, |
| "grad_norm": 3.4647848378452, |
| "kl": 0.1181640625, |
| "learning_rate": 4.9e-07, |
| "loss": 0.0047, |
| "reward": 1.43359375, |
| "reward_std": 0.44188737869262695, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7304687201976776, |
| "step": 561 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.078125, |
| "epoch": 1.5432098765432098, |
| "grad_norm": 4.4882395443760155, |
| "kl": 0.138427734375, |
| "learning_rate": 4.89090909090909e-07, |
| "loss": 0.0055, |
| "reward": 1.56640625, |
| "reward_std": 0.3243209421634674, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.76953125, |
| "step": 562 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.109375, |
| "epoch": 1.5459533607681757, |
| "grad_norm": 3.5210911842576804, |
| "kl": 0.135498046875, |
| "learning_rate": 4.881818181818181e-07, |
| "loss": 0.0054, |
| "reward": 1.532812476158142, |
| "reward_std": 0.4216124564409256, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7671875059604645, |
| "step": 563 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.546875, |
| "epoch": 1.5486968449931413, |
| "grad_norm": 1.6962515713747988, |
| "kl": 0.106689453125, |
| "learning_rate": 4.872727272727272e-07, |
| "loss": 0.0043, |
| "reward": 1.732812523841858, |
| "reward_std": 0.2328590601682663, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8734375238418579, |
| "step": 564 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.328125, |
| "epoch": 1.551440329218107, |
| "grad_norm": 3.930797977742594, |
| "kl": 0.11767578125, |
| "learning_rate": 4.863636363636364e-07, |
| "loss": 0.0047, |
| "reward": 1.615625023841858, |
| "reward_std": 0.22946061939001083, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7875000238418579, |
| "step": 565 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.265625, |
| "epoch": 1.5541838134430725, |
| "grad_norm": 7.4504102904463965, |
| "kl": 0.114013671875, |
| "learning_rate": 4.854545454545454e-07, |
| "loss": 0.0046, |
| "reward": 1.524218738079071, |
| "reward_std": 0.5162283182144165, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.774218738079071, |
| "step": 566 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.0, |
| "epoch": 1.5569272976680384, |
| "grad_norm": 2.417056436812491, |
| "kl": 0.08837890625, |
| "learning_rate": 4.845454545454545e-07, |
| "loss": 0.0035, |
| "reward": 1.80078125, |
| "reward_std": 0.40284576267004013, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.87890625, |
| "step": 567 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.09375, |
| "epoch": 1.5596707818930042, |
| "grad_norm": 3.7816972340293797, |
| "kl": 0.121337890625, |
| "learning_rate": 4.836363636363636e-07, |
| "loss": 0.0049, |
| "reward": 1.399999976158142, |
| "reward_std": 0.32486552000045776, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7124999761581421, |
| "step": 568 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.703125, |
| "epoch": 1.5624142661179699, |
| "grad_norm": 4.999981285474218, |
| "kl": 0.1396484375, |
| "learning_rate": 4.827272727272727e-07, |
| "loss": 0.0056, |
| "reward": 1.757031261920929, |
| "reward_std": 0.42812520265579224, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8820312321186066, |
| "step": 569 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.953125, |
| "epoch": 1.5651577503429355, |
| "grad_norm": 5.426287768038498, |
| "kl": 0.1103515625, |
| "learning_rate": 4.818181818181818e-07, |
| "loss": 0.0044, |
| "reward": 1.6609374284744263, |
| "reward_std": 0.4797618091106415, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.832812488079071, |
| "step": 570 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.046875, |
| "epoch": 1.567901234567901, |
| "grad_norm": 5.988679095790291, |
| "kl": 0.12060546875, |
| "learning_rate": 4.809090909090909e-07, |
| "loss": 0.0048, |
| "reward": 1.38671875, |
| "reward_std": 0.5065616965293884, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6992187201976776, |
| "step": 571 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.296875, |
| "epoch": 1.570644718792867, |
| "grad_norm": 4.7176649262724615, |
| "kl": 0.108642578125, |
| "learning_rate": 4.8e-07, |
| "loss": 0.0043, |
| "reward": 1.5374999642372131, |
| "reward_std": 0.49469953775405884, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7718749940395355, |
| "step": 572 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.4375, |
| "epoch": 1.5733882030178328, |
| "grad_norm": 4.621444143827856, |
| "kl": 0.140380859375, |
| "learning_rate": 4.79090909090909e-07, |
| "loss": 0.0056, |
| "reward": 1.6992186903953552, |
| "reward_std": 0.4136374071240425, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8554687201976776, |
| "step": 573 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.515625, |
| "epoch": 1.5761316872427984, |
| "grad_norm": 2.88230948081132, |
| "kl": 0.12646484375, |
| "learning_rate": 4.781818181818182e-07, |
| "loss": 0.005, |
| "reward": 1.69921875, |
| "reward_std": 0.25533968955278397, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8554687798023224, |
| "step": 574 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.984375, |
| "epoch": 1.578875171467764, |
| "grad_norm": 4.184751540073032, |
| "kl": 0.12255859375, |
| "learning_rate": 4.772727272727273e-07, |
| "loss": 0.0049, |
| "reward": 1.626562476158142, |
| "reward_std": 0.36238789558410645, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8140625059604645, |
| "step": 575 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.859375, |
| "epoch": 1.5816186556927296, |
| "grad_norm": 1.4444185806466716, |
| "kl": 0.113037109375, |
| "learning_rate": 4.7636363636363635e-07, |
| "loss": 0.0045, |
| "reward": 1.435937523841858, |
| "reward_std": 0.19391006976366043, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7328125238418579, |
| "step": 576 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.234375, |
| "epoch": 1.5843621399176955, |
| "grad_norm": 5.755466833556867, |
| "kl": 0.14013671875, |
| "learning_rate": 4.754545454545454e-07, |
| "loss": 0.0056, |
| "reward": 1.638281226158142, |
| "reward_std": 0.5257053971290588, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8257812857627869, |
| "step": 577 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.921875, |
| "epoch": 1.5871056241426613, |
| "grad_norm": 13.267074126831844, |
| "kl": 0.14453125, |
| "learning_rate": 4.7454545454545454e-07, |
| "loss": 0.0058, |
| "reward": 1.232812523841858, |
| "reward_std": 0.4578182101249695, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6234374642372131, |
| "step": 578 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 64.546875, |
| "epoch": 1.589849108367627, |
| "grad_norm": 3.8694886430696425, |
| "kl": 0.09716796875, |
| "learning_rate": 4.736363636363636e-07, |
| "loss": 0.0039, |
| "reward": 1.4031250476837158, |
| "reward_std": 0.3256668224930763, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7156249582767487, |
| "step": 579 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.1875, |
| "epoch": 1.5925925925925926, |
| "grad_norm": 15.953835080678518, |
| "kl": 0.09765625, |
| "learning_rate": 4.727272727272727e-07, |
| "loss": 0.0039, |
| "reward": 1.5390625596046448, |
| "reward_std": 0.2961009591817856, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7734375, |
| "step": 580 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.796875, |
| "epoch": 1.5953360768175582, |
| "grad_norm": 3.4278361707564002, |
| "kl": 0.132568359375, |
| "learning_rate": 4.718181818181818e-07, |
| "loss": 0.0053, |
| "reward": 1.28125, |
| "reward_std": 0.33917755633592606, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.65625, |
| "step": 581 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.3125, |
| "epoch": 1.598079561042524, |
| "grad_norm": 3.9189807225679303, |
| "kl": 0.14111328125, |
| "learning_rate": 4.709090909090909e-07, |
| "loss": 0.0057, |
| "reward": 1.384374976158142, |
| "reward_std": 0.4875795245170593, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6968750059604645, |
| "step": 582 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.875, |
| "epoch": 1.6008230452674899, |
| "grad_norm": 9.724241563545263, |
| "kl": 0.12158203125, |
| "learning_rate": 4.6999999999999995e-07, |
| "loss": 0.0049, |
| "reward": 1.649999976158142, |
| "reward_std": 0.5250689834356308, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8218749761581421, |
| "step": 583 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.65625, |
| "epoch": 1.6035665294924555, |
| "grad_norm": 6.05474610978915, |
| "kl": 0.142333984375, |
| "learning_rate": 4.6909090909090907e-07, |
| "loss": 0.0057, |
| "reward": 1.389062523841858, |
| "reward_std": 0.5110916793346405, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 584 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.15625, |
| "epoch": 1.606310013717421, |
| "grad_norm": 6.233889163012752, |
| "kl": 0.1552734375, |
| "learning_rate": 4.681818181818182e-07, |
| "loss": 0.0062, |
| "reward": 1.703125, |
| "reward_std": 0.42360812425613403, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.84375, |
| "step": 585 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.125, |
| "epoch": 1.6090534979423867, |
| "grad_norm": 3.2489686821377086, |
| "kl": 0.116455078125, |
| "learning_rate": 4.6727272727272727e-07, |
| "loss": 0.0047, |
| "reward": 1.13671875, |
| "reward_std": 0.5294448733329773, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5898436903953552, |
| "step": 586 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.46875, |
| "epoch": 1.6117969821673526, |
| "grad_norm": 2.3855608825870394, |
| "kl": 0.134765625, |
| "learning_rate": 4.6636363636363634e-07, |
| "loss": 0.0054, |
| "reward": 1.890625, |
| "reward_std": 0.24214322119951248, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9375, |
| "step": 587 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.515625, |
| "epoch": 1.6145404663923184, |
| "grad_norm": 4.168536384395145, |
| "kl": 0.12841796875, |
| "learning_rate": 4.6545454545454546e-07, |
| "loss": 0.0052, |
| "reward": 1.3265624642372131, |
| "reward_std": 0.510853722691536, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6703124642372131, |
| "step": 588 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.78125, |
| "epoch": 1.617283950617284, |
| "grad_norm": 4.689041205415565, |
| "kl": 0.1416015625, |
| "learning_rate": 4.6454545454545453e-07, |
| "loss": 0.0057, |
| "reward": 1.1421875357627869, |
| "reward_std": 0.5705660581588745, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5796874761581421, |
| "step": 589 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.234375, |
| "epoch": 1.6200274348422496, |
| "grad_norm": 3.0173678990110187, |
| "kl": 0.15771484375, |
| "learning_rate": 4.636363636363636e-07, |
| "loss": 0.0063, |
| "reward": 1.4898437857627869, |
| "reward_std": 0.33587054908275604, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7398437261581421, |
| "step": 590 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.65625, |
| "epoch": 1.6227709190672153, |
| "grad_norm": 3.6987374890571068, |
| "kl": 0.094482421875, |
| "learning_rate": 4.6272727272727267e-07, |
| "loss": 0.0038, |
| "reward": 1.805468738079071, |
| "reward_std": 0.25329074263572693, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.899218738079071, |
| "step": 591 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.53125, |
| "epoch": 1.625514403292181, |
| "grad_norm": 4.2905130258744615, |
| "kl": 0.127685546875, |
| "learning_rate": 4.618181818181818e-07, |
| "loss": 0.0051, |
| "reward": 1.4921875, |
| "reward_std": 0.09280776232481003, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7578125, |
| "step": 592 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.09375, |
| "epoch": 1.6282578875171467, |
| "grad_norm": 5.484523251838174, |
| "kl": 0.15087890625, |
| "learning_rate": 4.609090909090909e-07, |
| "loss": 0.0061, |
| "reward": 1.178125023841858, |
| "reward_std": 0.6947203576564789, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.5999999940395355, |
| "step": 593 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.6875, |
| "epoch": 1.6310013717421126, |
| "grad_norm": 10.441892796687878, |
| "kl": 0.1103515625, |
| "learning_rate": 4.6e-07, |
| "loss": 0.0044, |
| "reward": 1.2367187142372131, |
| "reward_std": 0.5257429778575897, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6273437142372131, |
| "step": 594 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.34375, |
| "epoch": 1.6337448559670782, |
| "grad_norm": 4.828817797430915, |
| "kl": 0.13134765625, |
| "learning_rate": 4.5909090909090906e-07, |
| "loss": 0.0052, |
| "reward": 1.578906238079071, |
| "reward_std": 0.2987357899546623, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.797656238079071, |
| "step": 595 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.203125, |
| "epoch": 1.6364883401920438, |
| "grad_norm": 9.194137482032842, |
| "kl": 0.12890625, |
| "learning_rate": 4.581818181818182e-07, |
| "loss": 0.0052, |
| "reward": 1.161718726158142, |
| "reward_std": 0.3223042516037822, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5992186963558197, |
| "step": 596 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.765625, |
| "epoch": 1.6392318244170097, |
| "grad_norm": 4.86268093816095, |
| "kl": 0.1103515625, |
| "learning_rate": 4.5727272727272725e-07, |
| "loss": 0.0044, |
| "reward": 1.665624976158142, |
| "reward_std": 0.4220944941043854, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8375000059604645, |
| "step": 597 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.625, |
| "epoch": 1.6419753086419753, |
| "grad_norm": 2.8619324737582934, |
| "kl": 0.116943359375, |
| "learning_rate": 4.563636363636363e-07, |
| "loss": 0.0047, |
| "reward": 1.7578125, |
| "reward_std": 0.28776606172323227, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8828125, |
| "step": 598 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.46875, |
| "epoch": 1.6447187928669411, |
| "grad_norm": 3.2934620958997036, |
| "kl": 0.1123046875, |
| "learning_rate": 4.5545454545454545e-07, |
| "loss": 0.0045, |
| "reward": 1.8453125357627869, |
| "reward_std": 0.3014649823307991, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9234375059604645, |
| "step": 599 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 61.8125, |
| "epoch": 1.6474622770919067, |
| "grad_norm": 3.7353753031206534, |
| "kl": 0.14208984375, |
| "learning_rate": 4.545454545454545e-07, |
| "loss": 0.0057, |
| "reward": 1.544531226158142, |
| "reward_std": 0.30668093264102936, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7789062261581421, |
| "step": 600 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.359375, |
| "epoch": 1.6502057613168724, |
| "grad_norm": 2.4930601686369034, |
| "kl": 0.106201171875, |
| "learning_rate": 4.536363636363636e-07, |
| "loss": 0.0042, |
| "reward": 1.576562523841858, |
| "reward_std": 0.2080155168659985, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7953125238418579, |
| "step": 601 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.765625, |
| "epoch": 1.652949245541838, |
| "grad_norm": 44.7657764016526, |
| "kl": 0.107177734375, |
| "learning_rate": 4.5272727272727266e-07, |
| "loss": 0.0043, |
| "reward": 1.547656238079071, |
| "reward_std": 0.6531318426132202, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.782031238079071, |
| "step": 602 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.765625, |
| "epoch": 1.6556927297668038, |
| "grad_norm": 3.435565573409563, |
| "kl": 0.134765625, |
| "learning_rate": 4.5181818181818184e-07, |
| "loss": 0.0054, |
| "reward": 1.4265624284744263, |
| "reward_std": 0.4840279370546341, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.723437488079071, |
| "step": 603 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.15625, |
| "epoch": 1.6584362139917697, |
| "grad_norm": 4.287230409751256, |
| "kl": 0.11669921875, |
| "learning_rate": 4.509090909090909e-07, |
| "loss": 0.0047, |
| "reward": 1.306249976158142, |
| "reward_std": 0.32932937145233154, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6656249761581421, |
| "step": 604 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.953125, |
| "epoch": 1.6611796982167353, |
| "grad_norm": 6.264101524320277, |
| "kl": 0.130859375, |
| "learning_rate": 4.5e-07, |
| "loss": 0.0052, |
| "reward": 1.182812511920929, |
| "reward_std": 0.6257371604442596, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.604687511920929, |
| "step": 605 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.390625, |
| "epoch": 1.663923182441701, |
| "grad_norm": 7.173487886744967, |
| "kl": 0.28466796875, |
| "learning_rate": 4.4909090909090905e-07, |
| "loss": 0.0114, |
| "reward": 1.397656261920929, |
| "reward_std": 0.6923157572746277, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.710156261920929, |
| "step": 606 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.671875, |
| "epoch": 1.6666666666666665, |
| "grad_norm": 3.2794289247863477, |
| "kl": 0.089111328125, |
| "learning_rate": 4.481818181818182e-07, |
| "loss": 0.0036, |
| "reward": 1.670312523841858, |
| "reward_std": 0.29969462007284164, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8421875238418579, |
| "step": 607 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.34375, |
| "epoch": 1.6694101508916324, |
| "grad_norm": 5.6576754017177295, |
| "kl": 0.0986328125, |
| "learning_rate": 4.4727272727272724e-07, |
| "loss": 0.0039, |
| "reward": 1.7078125476837158, |
| "reward_std": 0.4768216609954834, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.832812488079071, |
| "step": 608 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.46875, |
| "epoch": 1.6721536351165982, |
| "grad_norm": 10.137304702290834, |
| "kl": 0.142578125, |
| "learning_rate": 4.463636363636363e-07, |
| "loss": 0.0057, |
| "reward": 1.7421875, |
| "reward_std": 0.5135543495416641, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8671875, |
| "step": 609 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.359375, |
| "epoch": 1.6748971193415638, |
| "grad_norm": 3.438968403143286, |
| "kl": 0.124755859375, |
| "learning_rate": 4.4545454545454544e-07, |
| "loss": 0.005, |
| "reward": 1.3140625357627869, |
| "reward_std": 0.4177750498056412, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6578125059604645, |
| "step": 610 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.671875, |
| "epoch": 1.6776406035665294, |
| "grad_norm": 4.865075486656035, |
| "kl": 0.088623046875, |
| "learning_rate": 4.445454545454545e-07, |
| "loss": 0.0035, |
| "reward": 1.217187523841858, |
| "reward_std": 0.6815090179443359, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6078124940395355, |
| "step": 611 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.890625, |
| "epoch": 1.680384087791495, |
| "grad_norm": 5.572468848996713, |
| "kl": 0.10595703125, |
| "learning_rate": 4.4363636363636363e-07, |
| "loss": 0.0042, |
| "reward": 1.2195312976837158, |
| "reward_std": 0.40889422595500946, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.625781238079071, |
| "step": 612 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.890625, |
| "epoch": 1.683127572016461, |
| "grad_norm": 2.5481794991899283, |
| "kl": 0.105224609375, |
| "learning_rate": 4.427272727272727e-07, |
| "loss": 0.0042, |
| "reward": 1.5843749642372131, |
| "reward_std": 0.3832883834838867, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7875000238418579, |
| "step": 613 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.140625, |
| "epoch": 1.6858710562414267, |
| "grad_norm": 4.31918666722551, |
| "kl": 0.10107421875, |
| "learning_rate": 4.4181818181818183e-07, |
| "loss": 0.0041, |
| "reward": 1.40625, |
| "reward_std": 0.4046207368373871, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7031249701976776, |
| "step": 614 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.46875, |
| "epoch": 1.6886145404663924, |
| "grad_norm": 3.681758432983122, |
| "kl": 0.130859375, |
| "learning_rate": 4.409090909090909e-07, |
| "loss": 0.0052, |
| "reward": 0.811718761920929, |
| "reward_std": 0.5958449244499207, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.4210937023162842, |
| "step": 615 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.203125, |
| "epoch": 1.691358024691358, |
| "grad_norm": 5.594612819372044, |
| "kl": 0.091796875, |
| "learning_rate": 4.3999999999999997e-07, |
| "loss": 0.0037, |
| "reward": 1.2671875357627869, |
| "reward_std": 0.4626980125904083, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6421874761581421, |
| "step": 616 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.6875, |
| "epoch": 1.6941015089163236, |
| "grad_norm": 3.7730089340092716, |
| "kl": 0.1142578125, |
| "learning_rate": 4.390909090909091e-07, |
| "loss": 0.0046, |
| "reward": 1.449999988079071, |
| "reward_std": 0.4162885248661041, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.731249988079071, |
| "step": 617 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.078125, |
| "epoch": 1.6968449931412894, |
| "grad_norm": 4.643329708435787, |
| "kl": 0.0986328125, |
| "learning_rate": 4.3818181818181816e-07, |
| "loss": 0.0039, |
| "reward": 1.4625000357627869, |
| "reward_std": 0.5247026681900024, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7281249761581421, |
| "step": 618 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.0625, |
| "epoch": 1.6995884773662553, |
| "grad_norm": 3.6048763111718376, |
| "kl": 0.08935546875, |
| "learning_rate": 4.3727272727272723e-07, |
| "loss": 0.0036, |
| "reward": 1.125781238079071, |
| "reward_std": 0.44691573083400726, |
| "rewards/choice_reward": 0.5625, |
| "rewards/roam_reward": 0.5632812231779099, |
| "step": 619 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.453125, |
| "epoch": 1.702331961591221, |
| "grad_norm": 4.4723464168826075, |
| "kl": 0.147705078125, |
| "learning_rate": 4.363636363636363e-07, |
| "loss": 0.0059, |
| "reward": 1.6921875476837158, |
| "reward_std": 0.4920467585325241, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8328125476837158, |
| "step": 620 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.21875, |
| "epoch": 1.7050754458161865, |
| "grad_norm": 3.3386140274358804, |
| "kl": 0.114990234375, |
| "learning_rate": 4.3545454545454543e-07, |
| "loss": 0.0046, |
| "reward": 1.299218773841858, |
| "reward_std": 0.5712471902370453, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6585937440395355, |
| "step": 621 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.71875, |
| "epoch": 1.7078189300411522, |
| "grad_norm": 2.621699357498961, |
| "kl": 0.10546875, |
| "learning_rate": 4.3454545454545455e-07, |
| "loss": 0.0042, |
| "reward": 1.7265625, |
| "reward_std": 0.3452789708971977, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8671874701976776, |
| "step": 622 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 67.75, |
| "epoch": 1.710562414266118, |
| "grad_norm": 5.4962516246306, |
| "kl": 0.12109375, |
| "learning_rate": 4.336363636363636e-07, |
| "loss": 0.0049, |
| "reward": 1.54296875, |
| "reward_std": 0.5655232667922974, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7773437201976776, |
| "step": 623 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.03125, |
| "epoch": 1.7133058984910838, |
| "grad_norm": 3.6643652460769727, |
| "kl": 0.081298828125, |
| "learning_rate": 4.327272727272727e-07, |
| "loss": 0.0033, |
| "reward": 1.635937511920929, |
| "reward_std": 0.305075004696846, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.807812511920929, |
| "step": 624 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.828125, |
| "epoch": 1.7160493827160495, |
| "grad_norm": 4.032061968412479, |
| "kl": 0.11181640625, |
| "learning_rate": 4.318181818181818e-07, |
| "loss": 0.0045, |
| "reward": 1.167187511920929, |
| "reward_std": 0.7204632759094238, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.5890624821186066, |
| "step": 625 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.953125, |
| "epoch": 1.718792866941015, |
| "grad_norm": 5.402039980385757, |
| "kl": 0.11572265625, |
| "learning_rate": 4.309090909090909e-07, |
| "loss": 0.0046, |
| "reward": 1.650781273841858, |
| "reward_std": 0.6595319211483002, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8226562440395355, |
| "step": 626 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.984375, |
| "epoch": 1.7215363511659807, |
| "grad_norm": 5.258114174397278, |
| "kl": 0.099365234375, |
| "learning_rate": 4.2999999999999996e-07, |
| "loss": 0.004, |
| "reward": 1.697656273841858, |
| "reward_std": 0.4678082764148712, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8539062440395355, |
| "step": 627 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.5, |
| "epoch": 1.7242798353909465, |
| "grad_norm": 4.930678158123674, |
| "kl": 0.11865234375, |
| "learning_rate": 4.290909090909091e-07, |
| "loss": 0.0047, |
| "reward": 1.543749988079071, |
| "reward_std": 0.5408697128295898, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.778124988079071, |
| "step": 628 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.484375, |
| "epoch": 1.7270233196159122, |
| "grad_norm": 4.254385209226658, |
| "kl": 0.11083984375, |
| "learning_rate": 4.2818181818181815e-07, |
| "loss": 0.0044, |
| "reward": 1.7835937142372131, |
| "reward_std": 0.30397772043943405, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8773437738418579, |
| "step": 629 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.328125, |
| "epoch": 1.729766803840878, |
| "grad_norm": 3.393291956631288, |
| "kl": 0.133544921875, |
| "learning_rate": 4.272727272727272e-07, |
| "loss": 0.0054, |
| "reward": 1.2789062857627869, |
| "reward_std": 0.184274067170918, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6539062559604645, |
| "step": 630 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.90625, |
| "epoch": 1.7325102880658436, |
| "grad_norm": 3.301216041669399, |
| "kl": 0.10302734375, |
| "learning_rate": 4.2636363636363635e-07, |
| "loss": 0.0041, |
| "reward": 1.4796875715255737, |
| "reward_std": 0.4603932499885559, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7453124821186066, |
| "step": 631 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.78125, |
| "epoch": 1.7352537722908092, |
| "grad_norm": 5.67118213131503, |
| "kl": 0.1005859375, |
| "learning_rate": 4.2545454545454547e-07, |
| "loss": 0.004, |
| "reward": 1.578125, |
| "reward_std": 0.33960459381341934, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7968749701976776, |
| "step": 632 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.5625, |
| "epoch": 1.737997256515775, |
| "grad_norm": 16.737984619382317, |
| "kl": 0.125244140625, |
| "learning_rate": 4.2454545454545454e-07, |
| "loss": 0.005, |
| "reward": 1.3218750357627869, |
| "reward_std": 0.6554046273231506, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6656249761581421, |
| "step": 633 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.484375, |
| "epoch": 1.7407407407407407, |
| "grad_norm": 23.548761490488786, |
| "kl": 0.105224609375, |
| "learning_rate": 4.236363636363636e-07, |
| "loss": 0.0042, |
| "reward": 1.5374999642372131, |
| "reward_std": 0.4010937958955765, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7718749940395355, |
| "step": 634 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.390625, |
| "epoch": 1.7434842249657065, |
| "grad_norm": 3.9195780766238997, |
| "kl": 0.111083984375, |
| "learning_rate": 4.227272727272727e-07, |
| "loss": 0.0044, |
| "reward": 1.430468738079071, |
| "reward_std": 0.13726941123604774, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.727343738079071, |
| "step": 635 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.84375, |
| "epoch": 1.7462277091906722, |
| "grad_norm": 4.673777791186161, |
| "kl": 0.128662109375, |
| "learning_rate": 4.218181818181818e-07, |
| "loss": 0.0052, |
| "reward": 1.7734375, |
| "reward_std": 0.32488279044628143, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8828124701976776, |
| "step": 636 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.90625, |
| "epoch": 1.7489711934156378, |
| "grad_norm": 4.151451727323081, |
| "kl": 0.11572265625, |
| "learning_rate": 4.209090909090909e-07, |
| "loss": 0.0046, |
| "reward": 1.5406250357627869, |
| "reward_std": 0.6340201795101166, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7749999761581421, |
| "step": 637 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.65625, |
| "epoch": 1.7517146776406034, |
| "grad_norm": 4.804736060113988, |
| "kl": 0.120849609375, |
| "learning_rate": 4.1999999999999995e-07, |
| "loss": 0.0048, |
| "reward": 1.5140624642372131, |
| "reward_std": 0.675272136926651, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7328124940395355, |
| "step": 638 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.59375, |
| "epoch": 1.7544581618655692, |
| "grad_norm": 4.015887733422738, |
| "kl": 0.10302734375, |
| "learning_rate": 4.1909090909090907e-07, |
| "loss": 0.0041, |
| "reward": 1.204687476158142, |
| "reward_std": 0.45923222601413727, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6109374761581421, |
| "step": 639 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.453125, |
| "epoch": 1.757201646090535, |
| "grad_norm": 1.7349024210804238, |
| "kl": 0.115234375, |
| "learning_rate": 4.1818181818181814e-07, |
| "loss": 0.0046, |
| "reward": 1.6375000476837158, |
| "reward_std": 0.13271774351596832, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8249999582767487, |
| "step": 640 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.15625, |
| "epoch": 1.7599451303155007, |
| "grad_norm": 2.9720485278284987, |
| "kl": 0.130859375, |
| "learning_rate": 4.1727272727272727e-07, |
| "loss": 0.0052, |
| "reward": 1.864062488079071, |
| "reward_std": 0.22904610633850098, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9109375178813934, |
| "step": 641 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.0625, |
| "epoch": 1.7626886145404663, |
| "grad_norm": 2.9198017047150593, |
| "kl": 0.15087890625, |
| "learning_rate": 4.1636363636363634e-07, |
| "loss": 0.006, |
| "reward": 1.598437488079071, |
| "reward_std": 0.48394007980823517, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8015624582767487, |
| "step": 642 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.84375, |
| "epoch": 1.765432098765432, |
| "grad_norm": 4.5819328944239714, |
| "kl": 0.13330078125, |
| "learning_rate": 4.1545454545454546e-07, |
| "loss": 0.0053, |
| "reward": 1.57421875, |
| "reward_std": 0.5152434408664703, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7773437201976776, |
| "step": 643 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.8125, |
| "epoch": 1.7681755829903978, |
| "grad_norm": 2.9261270753295068, |
| "kl": 0.10205078125, |
| "learning_rate": 4.1454545454545453e-07, |
| "loss": 0.0041, |
| "reward": 1.646875023841858, |
| "reward_std": 0.49189814925193787, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8187499940395355, |
| "step": 644 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.46875, |
| "epoch": 1.7709190672153636, |
| "grad_norm": 14.343148169313086, |
| "kl": 0.105712890625, |
| "learning_rate": 4.136363636363636e-07, |
| "loss": 0.0042, |
| "reward": 1.561718761920929, |
| "reward_std": 0.2597835436463356, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.780468761920929, |
| "step": 645 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.5, |
| "epoch": 1.7736625514403292, |
| "grad_norm": 7.952157247384164, |
| "kl": 0.18896484375, |
| "learning_rate": 4.1272727272727273e-07, |
| "loss": 0.0076, |
| "reward": 1.48046875, |
| "reward_std": 0.5070348381996155, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.74609375, |
| "step": 646 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.015625, |
| "epoch": 1.7764060356652949, |
| "grad_norm": 4.110872634394268, |
| "kl": 0.102783203125, |
| "learning_rate": 4.118181818181818e-07, |
| "loss": 0.0041, |
| "reward": 1.28515625, |
| "reward_std": 0.5382444858551025, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.66015625, |
| "step": 647 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.25, |
| "epoch": 1.7791495198902605, |
| "grad_norm": 4.348289505877314, |
| "kl": 0.111083984375, |
| "learning_rate": 4.1090909090909087e-07, |
| "loss": 0.0044, |
| "reward": 0.8304687738418579, |
| "reward_std": 0.8299466967582703, |
| "rewards/choice_reward": 0.390625, |
| "rewards/roam_reward": 0.4398437440395355, |
| "step": 648 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.09375, |
| "epoch": 1.7818930041152263, |
| "grad_norm": 3.642364932683399, |
| "kl": 0.1640625, |
| "learning_rate": 4.0999999999999994e-07, |
| "loss": 0.0066, |
| "reward": 1.5789062976837158, |
| "reward_std": 0.2790534794330597, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7820312678813934, |
| "step": 649 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.78125, |
| "epoch": 1.7846364883401922, |
| "grad_norm": 12.979111094428676, |
| "kl": 0.104736328125, |
| "learning_rate": 4.090909090909091e-07, |
| "loss": 0.0042, |
| "reward": 1.66796875, |
| "reward_std": 0.45473532378673553, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.83984375, |
| "step": 650 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.765625, |
| "epoch": 1.7873799725651578, |
| "grad_norm": 2.1645407688475804, |
| "kl": 0.115478515625, |
| "learning_rate": 4.081818181818182e-07, |
| "loss": 0.0046, |
| "reward": 1.671875, |
| "reward_std": 0.21265346556901932, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8437500298023224, |
| "step": 651 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.53125, |
| "epoch": 1.7901234567901234, |
| "grad_norm": 12.934790238319271, |
| "kl": 0.115478515625, |
| "learning_rate": 4.0727272727272726e-07, |
| "loss": 0.0046, |
| "reward": 1.686718761920929, |
| "reward_std": 0.15634502097964287, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.842968761920929, |
| "step": 652 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.578125, |
| "epoch": 1.792866941015089, |
| "grad_norm": 2.165748636888658, |
| "kl": 0.122314453125, |
| "learning_rate": 4.0636363636363633e-07, |
| "loss": 0.0049, |
| "reward": 1.965624988079071, |
| "reward_std": 0.09722718130797148, |
| "rewards/choice_reward": 0.984375, |
| "rewards/roam_reward": 0.981249988079071, |
| "step": 653 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.953125, |
| "epoch": 1.7956104252400549, |
| "grad_norm": 6.944281402277241, |
| "kl": 0.15185546875, |
| "learning_rate": 4.0545454545454545e-07, |
| "loss": 0.0061, |
| "reward": 1.46484375, |
| "reward_std": 0.5038956105709076, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7304687201976776, |
| "step": 654 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.796875, |
| "epoch": 1.7983539094650207, |
| "grad_norm": 4.714143244008144, |
| "kl": 0.133544921875, |
| "learning_rate": 4.045454545454545e-07, |
| "loss": 0.0053, |
| "reward": 1.044531226158142, |
| "reward_std": 0.2878062203526497, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5445312559604645, |
| "step": 655 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.453125, |
| "epoch": 1.8010973936899863, |
| "grad_norm": 3.510647708440207, |
| "kl": 0.16796875, |
| "learning_rate": 4.036363636363636e-07, |
| "loss": 0.0067, |
| "reward": 1.1023436784744263, |
| "reward_std": 0.5571488589048386, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5710937082767487, |
| "step": 656 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.59375, |
| "epoch": 1.803840877914952, |
| "grad_norm": 2.2097975631180664, |
| "kl": 0.111572265625, |
| "learning_rate": 4.027272727272727e-07, |
| "loss": 0.0045, |
| "reward": 1.845312476158142, |
| "reward_std": 0.14059560280293226, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9234374761581421, |
| "step": 657 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.625, |
| "epoch": 1.8065843621399176, |
| "grad_norm": 16.711973588366792, |
| "kl": 0.15478515625, |
| "learning_rate": 4.018181818181818e-07, |
| "loss": 0.0062, |
| "reward": 1.635156273841858, |
| "reward_std": 0.2930929362773895, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8226562440395355, |
| "step": 658 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.8125, |
| "epoch": 1.8093278463648834, |
| "grad_norm": 5.499431166564593, |
| "kl": 0.12841796875, |
| "learning_rate": 4.0090909090909086e-07, |
| "loss": 0.0051, |
| "reward": 1.571093738079071, |
| "reward_std": 0.4677266329526901, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7898437976837158, |
| "step": 659 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.03125, |
| "epoch": 1.8120713305898493, |
| "grad_norm": 3.3422892166875124, |
| "kl": 0.125244140625, |
| "learning_rate": 4e-07, |
| "loss": 0.005, |
| "reward": 1.5195311903953552, |
| "reward_std": 0.5444206297397614, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7539062201976776, |
| "step": 660 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.8125, |
| "epoch": 1.8148148148148149, |
| "grad_norm": 3.4788880821145636, |
| "kl": 0.0966796875, |
| "learning_rate": 3.990909090909091e-07, |
| "loss": 0.0039, |
| "reward": 1.357812523841858, |
| "reward_std": 0.5057743340730667, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 661 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.53125, |
| "epoch": 1.8175582990397805, |
| "grad_norm": 5.254697267894281, |
| "kl": 0.11572265625, |
| "learning_rate": 3.981818181818182e-07, |
| "loss": 0.0046, |
| "reward": 1.267187476158142, |
| "reward_std": 0.5789059698581696, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6421874761581421, |
| "step": 662 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.109375, |
| "epoch": 1.8203017832647461, |
| "grad_norm": 8.348894651119013, |
| "kl": 0.105712890625, |
| "learning_rate": 3.9727272727272725e-07, |
| "loss": 0.0042, |
| "reward": 1.6343750357627869, |
| "reward_std": 0.46311406791210175, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8062499761581421, |
| "step": 663 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.59375, |
| "epoch": 1.823045267489712, |
| "grad_norm": 22.507933396073454, |
| "kl": 0.104736328125, |
| "learning_rate": 3.9636363636363637e-07, |
| "loss": 0.0042, |
| "reward": 1.87109375, |
| "reward_std": 0.22839166969060898, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.93359375, |
| "step": 664 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.96875, |
| "epoch": 1.8257887517146778, |
| "grad_norm": 5.566050816433911, |
| "kl": 0.15380859375, |
| "learning_rate": 3.9545454545454544e-07, |
| "loss": 0.0062, |
| "reward": 1.389062523841858, |
| "reward_std": 0.5681444704532623, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6859374940395355, |
| "step": 665 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.515625, |
| "epoch": 1.8285322359396434, |
| "grad_norm": 4.828734408740684, |
| "kl": 0.125732421875, |
| "learning_rate": 3.945454545454545e-07, |
| "loss": 0.005, |
| "reward": 1.395312488079071, |
| "reward_std": 0.5245100259780884, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7078125178813934, |
| "step": 666 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.34375, |
| "epoch": 1.831275720164609, |
| "grad_norm": 7.921463334118804, |
| "kl": 0.10888671875, |
| "learning_rate": 3.936363636363636e-07, |
| "loss": 0.0043, |
| "reward": 1.787500023841858, |
| "reward_std": 0.3248981386423111, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8812500238418579, |
| "step": 667 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 65.8125, |
| "epoch": 1.8340192043895747, |
| "grad_norm": 10.30012458597402, |
| "kl": 0.146484375, |
| "learning_rate": 3.927272727272727e-07, |
| "loss": 0.0059, |
| "reward": 1.3664062321186066, |
| "reward_std": 0.3465329110622406, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.694531261920929, |
| "step": 668 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.671875, |
| "epoch": 1.8367626886145405, |
| "grad_norm": 3.219222098247594, |
| "kl": 0.095703125, |
| "learning_rate": 3.9181818181818183e-07, |
| "loss": 0.0038, |
| "reward": 1.5828125476837158, |
| "reward_std": 0.20820768177509308, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.8015625476837158, |
| "step": 669 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.703125, |
| "epoch": 1.8395061728395061, |
| "grad_norm": 4.128373118044727, |
| "kl": 0.13623046875, |
| "learning_rate": 3.909090909090909e-07, |
| "loss": 0.0055, |
| "reward": 1.517187476158142, |
| "reward_std": 0.45177534222602844, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7671874761581421, |
| "step": 670 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.484375, |
| "epoch": 1.842249657064472, |
| "grad_norm": 6.845986594822972, |
| "kl": 0.1064453125, |
| "learning_rate": 3.8999999999999997e-07, |
| "loss": 0.0043, |
| "reward": 1.460156261920929, |
| "reward_std": 0.6346226334571838, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7414062321186066, |
| "step": 671 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.125, |
| "epoch": 1.8449931412894376, |
| "grad_norm": 3.1518606793665565, |
| "kl": 0.1015625, |
| "learning_rate": 3.890909090909091e-07, |
| "loss": 0.0041, |
| "reward": 1.546875, |
| "reward_std": 0.429079607129097, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.78125, |
| "step": 672 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.90625, |
| "epoch": 1.8477366255144032, |
| "grad_norm": 2.8245002884513646, |
| "kl": 0.103271484375, |
| "learning_rate": 3.8818181818181817e-07, |
| "loss": 0.0041, |
| "reward": 1.564843773841858, |
| "reward_std": 0.2965794578194618, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7835937440395355, |
| "step": 673 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.015625, |
| "epoch": 1.850480109739369, |
| "grad_norm": 6.586851850905839, |
| "kl": 0.12744140625, |
| "learning_rate": 3.8727272727272724e-07, |
| "loss": 0.0051, |
| "reward": 1.552343726158142, |
| "reward_std": 0.6085359454154968, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7710937559604645, |
| "step": 674 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.78125, |
| "epoch": 1.8532235939643347, |
| "grad_norm": 5.987304118798936, |
| "kl": 0.116943359375, |
| "learning_rate": 3.8636363636363636e-07, |
| "loss": 0.0047, |
| "reward": 1.310156226158142, |
| "reward_std": 0.49685972929000854, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6695312261581421, |
| "step": 675 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.8125, |
| "epoch": 1.8559670781893005, |
| "grad_norm": 3.561802904173307, |
| "kl": 0.109375, |
| "learning_rate": 3.8545454545454543e-07, |
| "loss": 0.0044, |
| "reward": 1.3343749046325684, |
| "reward_std": 0.5377295911312103, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6781249940395355, |
| "step": 676 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.15625, |
| "epoch": 1.8587105624142661, |
| "grad_norm": 4.178869011124296, |
| "kl": 0.107177734375, |
| "learning_rate": 3.845454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.5750000476837158, |
| "reward_std": 0.5704424381256104, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7937499582767487, |
| "step": 677 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.890625, |
| "epoch": 1.8614540466392318, |
| "grad_norm": 6.402272643220298, |
| "kl": 0.19482421875, |
| "learning_rate": 3.8363636363636357e-07, |
| "loss": 0.0078, |
| "reward": 1.8406250476837158, |
| "reward_std": 0.2866463288664818, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.903124988079071, |
| "step": 678 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.140625, |
| "epoch": 1.8641975308641974, |
| "grad_norm": 2.8322674489238135, |
| "kl": 0.1171875, |
| "learning_rate": 3.8272727272727275e-07, |
| "loss": 0.0047, |
| "reward": 1.7257812023162842, |
| "reward_std": 0.33524563908576965, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8664062321186066, |
| "step": 679 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.125, |
| "epoch": 1.8669410150891632, |
| "grad_norm": 2.868827204779528, |
| "kl": 0.137451171875, |
| "learning_rate": 3.818181818181818e-07, |
| "loss": 0.0055, |
| "reward": 1.86328125, |
| "reward_std": 0.24384143203496933, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9101562798023224, |
| "step": 680 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.6875, |
| "epoch": 1.869684499314129, |
| "grad_norm": 3.46762257318447, |
| "kl": 0.1103515625, |
| "learning_rate": 3.809090909090909e-07, |
| "loss": 0.0044, |
| "reward": 1.6320312023162842, |
| "reward_std": 0.29449596256017685, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8039062023162842, |
| "step": 681 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.59375, |
| "epoch": 1.8724279835390947, |
| "grad_norm": 8.428313963135231, |
| "kl": 0.111572265625, |
| "learning_rate": 3.7999999999999996e-07, |
| "loss": 0.0045, |
| "reward": 0.9843749403953552, |
| "reward_std": 0.7652322351932526, |
| "rewards/choice_reward": 0.46875, |
| "rewards/roam_reward": 0.5156249701976776, |
| "step": 682 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.234375, |
| "epoch": 1.8751714677640603, |
| "grad_norm": 3.6179704590270587, |
| "kl": 0.1181640625, |
| "learning_rate": 3.790909090909091e-07, |
| "loss": 0.0047, |
| "reward": 1.6671875715255737, |
| "reward_std": 0.46436551213264465, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8234374821186066, |
| "step": 683 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.484375, |
| "epoch": 1.877914951989026, |
| "grad_norm": 3.0428398037737105, |
| "kl": 0.117919921875, |
| "learning_rate": 3.7818181818181816e-07, |
| "loss": 0.0047, |
| "reward": 0.98046875, |
| "reward_std": 0.5315862596035004, |
| "rewards/choice_reward": 0.484375, |
| "rewards/roam_reward": 0.49609375, |
| "step": 684 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.75, |
| "epoch": 1.8806584362139918, |
| "grad_norm": 3.110597703409818, |
| "kl": 0.09521484375, |
| "learning_rate": 3.7727272727272723e-07, |
| "loss": 0.0038, |
| "reward": 1.0960937142372131, |
| "reward_std": 0.4587474763393402, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5648437440395355, |
| "step": 685 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.21875, |
| "epoch": 1.8834019204389576, |
| "grad_norm": 6.979388930776721, |
| "kl": 0.1064453125, |
| "learning_rate": 3.7636363636363635e-07, |
| "loss": 0.0043, |
| "reward": 1.598437488079071, |
| "reward_std": 0.481655552983284, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8015625178813934, |
| "step": 686 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 64.40625, |
| "epoch": 1.8861454046639232, |
| "grad_norm": 2.6032786356111064, |
| "kl": 0.104248046875, |
| "learning_rate": 3.754545454545454e-07, |
| "loss": 0.0042, |
| "reward": 1.583593726158142, |
| "reward_std": 0.1682818979024887, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.8023437261581421, |
| "step": 687 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.65625, |
| "epoch": 1.8888888888888888, |
| "grad_norm": 6.423702414090454, |
| "kl": 0.11865234375, |
| "learning_rate": 3.7454545454545454e-07, |
| "loss": 0.0048, |
| "reward": 1.9015624523162842, |
| "reward_std": 0.23329924046993256, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.9328124523162842, |
| "step": 688 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.390625, |
| "epoch": 1.8916323731138545, |
| "grad_norm": 45.432742969077395, |
| "kl": 0.109375, |
| "learning_rate": 3.736363636363636e-07, |
| "loss": 0.0044, |
| "reward": 1.560156226158142, |
| "reward_std": 0.2677382007241249, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7789062559604645, |
| "step": 689 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.234375, |
| "epoch": 1.8943758573388203, |
| "grad_norm": 15.396758274487498, |
| "kl": 0.1298828125, |
| "learning_rate": 3.7272727272727274e-07, |
| "loss": 0.0052, |
| "reward": 1.870312511920929, |
| "reward_std": 0.23891766369342804, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.932812511920929, |
| "step": 690 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.765625, |
| "epoch": 1.8971193415637861, |
| "grad_norm": 3.9482502637428425, |
| "kl": 0.113037109375, |
| "learning_rate": 3.718181818181818e-07, |
| "loss": 0.0045, |
| "reward": 1.549218773841858, |
| "reward_std": 0.3217321038246155, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7835937440395355, |
| "step": 691 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.140625, |
| "epoch": 1.8998628257887518, |
| "grad_norm": 4.5071017302960295, |
| "kl": 0.10009765625, |
| "learning_rate": 3.709090909090909e-07, |
| "loss": 0.004, |
| "reward": 1.2515625357627869, |
| "reward_std": 0.4278821647167206, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6421874463558197, |
| "step": 692 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.75, |
| "epoch": 1.9026063100137174, |
| "grad_norm": 2.244419627785846, |
| "kl": 0.110595703125, |
| "learning_rate": 3.7e-07, |
| "loss": 0.0044, |
| "reward": 1.643750011920929, |
| "reward_std": 0.2198822796344757, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.831250011920929, |
| "step": 693 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.796875, |
| "epoch": 1.905349794238683, |
| "grad_norm": 4.384703844771046, |
| "kl": 0.092041015625, |
| "learning_rate": 3.690909090909091e-07, |
| "loss": 0.0037, |
| "reward": 1.189062476158142, |
| "reward_std": 0.25614288449287415, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.6109374761581421, |
| "step": 694 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.484375, |
| "epoch": 1.9080932784636488, |
| "grad_norm": 5.563534909388885, |
| "kl": 0.099365234375, |
| "learning_rate": 3.6818181818181815e-07, |
| "loss": 0.004, |
| "reward": 1.778124988079071, |
| "reward_std": 0.23368481919169426, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.887499988079071, |
| "step": 695 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.03125, |
| "epoch": 1.9108367626886147, |
| "grad_norm": 4.027865113179509, |
| "kl": 0.12548828125, |
| "learning_rate": 3.672727272727272e-07, |
| "loss": 0.005, |
| "reward": 1.228906273841858, |
| "reward_std": 0.6479907333850861, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6195312142372131, |
| "step": 696 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.671875, |
| "epoch": 1.9135802469135803, |
| "grad_norm": 4.882822249350005, |
| "kl": 0.105224609375, |
| "learning_rate": 3.663636363636364e-07, |
| "loss": 0.0042, |
| "reward": 1.271093726158142, |
| "reward_std": 0.40971219539642334, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6148436963558197, |
| "step": 697 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.984375, |
| "epoch": 1.916323731138546, |
| "grad_norm": 3.900258569974571, |
| "kl": 0.102783203125, |
| "learning_rate": 3.6545454545454546e-07, |
| "loss": 0.0041, |
| "reward": 1.635156273841858, |
| "reward_std": 0.46128255128860474, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8226562738418579, |
| "step": 698 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.546875, |
| "epoch": 1.9190672153635115, |
| "grad_norm": 2.981850065788348, |
| "kl": 0.098876953125, |
| "learning_rate": 3.6454545454545453e-07, |
| "loss": 0.004, |
| "reward": 0.946093738079071, |
| "reward_std": 0.39172008633613586, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.44609374552965164, |
| "step": 699 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.71875, |
| "epoch": 1.9218106995884774, |
| "grad_norm": 3.098333194742997, |
| "kl": 0.119384765625, |
| "learning_rate": 3.636363636363636e-07, |
| "loss": 0.0048, |
| "reward": 1.6484374403953552, |
| "reward_std": 0.39533631503582, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8203124701976776, |
| "step": 700 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.046875, |
| "epoch": 1.9245541838134432, |
| "grad_norm": 5.62601109975019, |
| "kl": 0.108154296875, |
| "learning_rate": 3.6272727272727273e-07, |
| "loss": 0.0043, |
| "reward": 1.2968750596046448, |
| "reward_std": 0.5830048173666, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.65625, |
| "step": 701 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.234375, |
| "epoch": 1.9272976680384089, |
| "grad_norm": 6.641903451864133, |
| "kl": 0.093505859375, |
| "learning_rate": 3.618181818181818e-07, |
| "loss": 0.0037, |
| "reward": 1.2617188096046448, |
| "reward_std": 0.637585312128067, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.63671875, |
| "step": 702 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.390625, |
| "epoch": 1.9300411522633745, |
| "grad_norm": 3.2872816549573898, |
| "kl": 0.0869140625, |
| "learning_rate": 3.6090909090909087e-07, |
| "loss": 0.0035, |
| "reward": 1.5468749403953552, |
| "reward_std": 0.49051383882761, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.78125, |
| "step": 703 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.46875, |
| "epoch": 1.93278463648834, |
| "grad_norm": 3.6181831610348354, |
| "kl": 0.11181640625, |
| "learning_rate": 3.6e-07, |
| "loss": 0.0045, |
| "reward": 1.3617187142372131, |
| "reward_std": 0.24989521596580744, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6898437440395355, |
| "step": 704 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.40625, |
| "epoch": 1.935528120713306, |
| "grad_norm": 3.109006593704769, |
| "kl": 0.10595703125, |
| "learning_rate": 3.5909090909090907e-07, |
| "loss": 0.0042, |
| "reward": 1.7625000476837158, |
| "reward_std": 0.48489242792129517, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.871874988079071, |
| "step": 705 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.734375, |
| "epoch": 1.9382716049382716, |
| "grad_norm": 5.473098192483056, |
| "kl": 0.123779296875, |
| "learning_rate": 3.5818181818181814e-07, |
| "loss": 0.005, |
| "reward": 1.4718750715255737, |
| "reward_std": 0.25257018208503723, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7374999523162842, |
| "step": 706 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.734375, |
| "epoch": 1.9410150891632374, |
| "grad_norm": 4.445413875832428, |
| "kl": 0.11083984375, |
| "learning_rate": 3.5727272727272726e-07, |
| "loss": 0.0044, |
| "reward": 1.393750011920929, |
| "reward_std": 0.5073824375867844, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7062499821186066, |
| "step": 707 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.578125, |
| "epoch": 1.943758573388203, |
| "grad_norm": 4.560508526779024, |
| "kl": 0.197998046875, |
| "learning_rate": 3.563636363636364e-07, |
| "loss": 0.0079, |
| "reward": 1.428125023841858, |
| "reward_std": 0.39360281080007553, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7249999642372131, |
| "step": 708 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.40625, |
| "epoch": 1.9465020576131686, |
| "grad_norm": 2.17085628132125, |
| "kl": 0.09814453125, |
| "learning_rate": 3.5545454545454545e-07, |
| "loss": 0.0039, |
| "reward": 1.800000011920929, |
| "reward_std": 0.3996886909008026, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.893750011920929, |
| "step": 709 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.28125, |
| "epoch": 1.9492455418381345, |
| "grad_norm": 3.57937268248927, |
| "kl": 0.126220703125, |
| "learning_rate": 3.545454545454545e-07, |
| "loss": 0.005, |
| "reward": 1.3960937857627869, |
| "reward_std": 0.6148386299610138, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.6929687559604645, |
| "step": 710 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.375, |
| "epoch": 1.9519890260631, |
| "grad_norm": 3.908202040629821, |
| "kl": 0.08984375, |
| "learning_rate": 3.536363636363636e-07, |
| "loss": 0.0036, |
| "reward": 1.6328125596046448, |
| "reward_std": 0.4680548459291458, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8203125, |
| "step": 711 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.125, |
| "epoch": 1.954732510288066, |
| "grad_norm": 2.184472228624725, |
| "kl": 0.1005859375, |
| "learning_rate": 3.527272727272727e-07, |
| "loss": 0.004, |
| "reward": 1.965624988079071, |
| "reward_std": 0.09722718130797148, |
| "rewards/choice_reward": 1.0, |
| "rewards/roam_reward": 0.965624988079071, |
| "step": 712 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.203125, |
| "epoch": 1.9574759945130316, |
| "grad_norm": 7.86769835988023, |
| "kl": 0.100830078125, |
| "learning_rate": 3.518181818181818e-07, |
| "loss": 0.004, |
| "reward": 1.56640625, |
| "reward_std": 0.366900697350502, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.78515625, |
| "step": 713 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.578125, |
| "epoch": 1.9602194787379972, |
| "grad_norm": 4.349028025159725, |
| "kl": 0.10107421875, |
| "learning_rate": 3.5090909090909086e-07, |
| "loss": 0.004, |
| "reward": 1.495312511920929, |
| "reward_std": 0.4397459104657173, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7609374523162842, |
| "step": 714 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.078125, |
| "epoch": 1.9629629629629628, |
| "grad_norm": 3.6619919266466, |
| "kl": 0.106689453125, |
| "learning_rate": 3.5e-07, |
| "loss": 0.0043, |
| "reward": 1.5539062023162842, |
| "reward_std": 0.3623511716723442, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7882812321186066, |
| "step": 715 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.453125, |
| "epoch": 1.9657064471879286, |
| "grad_norm": 8.744295572219698, |
| "kl": 0.09619140625, |
| "learning_rate": 3.490909090909091e-07, |
| "loss": 0.0039, |
| "reward": 1.357812523841858, |
| "reward_std": 0.450033038854599, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859374940395355, |
| "step": 716 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.5625, |
| "epoch": 1.9684499314128945, |
| "grad_norm": 5.8340616106872885, |
| "kl": 0.103515625, |
| "learning_rate": 3.481818181818182e-07, |
| "loss": 0.0041, |
| "reward": 1.456250011920929, |
| "reward_std": 0.42738912999629974, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7374999523162842, |
| "step": 717 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.734375, |
| "epoch": 1.97119341563786, |
| "grad_norm": 6.850305195091276, |
| "kl": 0.173828125, |
| "learning_rate": 3.4727272727272725e-07, |
| "loss": 0.007, |
| "reward": 1.5437500476837158, |
| "reward_std": 0.4188317656517029, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.778124988079071, |
| "step": 718 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.15625, |
| "epoch": 1.9739368998628257, |
| "grad_norm": 3.5350921655226535, |
| "kl": 0.098388671875, |
| "learning_rate": 3.4636363636363637e-07, |
| "loss": 0.0039, |
| "reward": 1.642968773841858, |
| "reward_std": 0.42786234617233276, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8304687738418579, |
| "step": 719 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.4375, |
| "epoch": 1.9766803840877913, |
| "grad_norm": 4.031256948107078, |
| "kl": 0.139892578125, |
| "learning_rate": 3.4545454545454544e-07, |
| "loss": 0.0056, |
| "reward": 1.5546875596046448, |
| "reward_std": 0.47294948995113373, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7265625, |
| "step": 720 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.71875, |
| "epoch": 1.9794238683127572, |
| "grad_norm": 5.45796597753534, |
| "kl": 0.108642578125, |
| "learning_rate": 3.445454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.260156273841858, |
| "reward_std": 0.2761820927262306, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6195312440395355, |
| "step": 721 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.546875, |
| "epoch": 1.982167352537723, |
| "grad_norm": 4.478599438907719, |
| "kl": 0.112060546875, |
| "learning_rate": 3.4363636363636364e-07, |
| "loss": 0.0045, |
| "reward": 1.6734375357627869, |
| "reward_std": 0.3208591490983963, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8453124761581421, |
| "step": 722 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.0, |
| "epoch": 1.9849108367626886, |
| "grad_norm": 6.9127924619208025, |
| "kl": 0.1376953125, |
| "learning_rate": 3.427272727272727e-07, |
| "loss": 0.0055, |
| "reward": 1.4648436903953552, |
| "reward_std": 0.4159834384918213, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.74609375, |
| "step": 723 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.515625, |
| "epoch": 1.9876543209876543, |
| "grad_norm": 12.186255810002473, |
| "kl": 0.10693359375, |
| "learning_rate": 3.418181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.634374976158142, |
| "reward_std": 0.24806804955005646, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8062500059604645, |
| "step": 724 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.6875, |
| "epoch": 1.9903978052126199, |
| "grad_norm": 4.467905979983616, |
| "kl": 0.1025390625, |
| "learning_rate": 3.4090909090909085e-07, |
| "loss": 0.0041, |
| "reward": 1.5320312976837158, |
| "reward_std": 0.6255079209804535, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.766406238079071, |
| "step": 725 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.09375, |
| "epoch": 1.9931412894375857, |
| "grad_norm": 2.9699493149022604, |
| "kl": 0.103759765625, |
| "learning_rate": 3.4000000000000003e-07, |
| "loss": 0.0042, |
| "reward": 1.504687488079071, |
| "reward_std": 0.46675867587327957, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.754687488079071, |
| "step": 726 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.375, |
| "epoch": 1.9958847736625516, |
| "grad_norm": 7.305345924991261, |
| "kl": 0.12255859375, |
| "learning_rate": 3.390909090909091e-07, |
| "loss": 0.0049, |
| "reward": 1.297656238079071, |
| "reward_std": 0.4874522387981415, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.641406238079071, |
| "step": 727 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.0, |
| "epoch": 1.9986282578875172, |
| "grad_norm": 2.604818082870772, |
| "kl": 0.092041015625, |
| "learning_rate": 3.3818181818181817e-07, |
| "loss": 0.0045, |
| "reward": 1.050000000745058, |
| "reward_std": 0.3358757197856903, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5500000007450581, |
| "step": 728 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.828125, |
| "epoch": 2.0027434842249656, |
| "grad_norm": 2.707911215722067, |
| "kl": 0.135986328125, |
| "learning_rate": 3.3727272727272724e-07, |
| "loss": 0.0054, |
| "reward": 1.59375, |
| "reward_std": 0.40535733103752136, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.796875, |
| "step": 729 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.203125, |
| "epoch": 2.0054869684499312, |
| "grad_norm": 8.722483521923781, |
| "kl": 0.123291015625, |
| "learning_rate": 3.3636363636363636e-07, |
| "loss": 0.0049, |
| "reward": 1.4906249642372131, |
| "reward_std": 0.4499505013227463, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7562499940395355, |
| "step": 730 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.765625, |
| "epoch": 2.0082304526748973, |
| "grad_norm": 2.664329462311894, |
| "kl": 0.12646484375, |
| "learning_rate": 3.3545454545454543e-07, |
| "loss": 0.0051, |
| "reward": 1.475000023841858, |
| "reward_std": 0.4457009881734848, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7406249940395355, |
| "step": 731 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.6875, |
| "epoch": 2.010973936899863, |
| "grad_norm": 3.5090566136071533, |
| "kl": 0.095458984375, |
| "learning_rate": 3.345454545454545e-07, |
| "loss": 0.0038, |
| "reward": 1.551562488079071, |
| "reward_std": 0.286793552339077, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.785937488079071, |
| "step": 732 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.484375, |
| "epoch": 2.0137174211248285, |
| "grad_norm": 2.7358123022416807, |
| "kl": 0.12890625, |
| "learning_rate": 3.3363636363636363e-07, |
| "loss": 0.0051, |
| "reward": 1.731249988079071, |
| "reward_std": 0.1946544125676155, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8718750178813934, |
| "step": 733 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.5625, |
| "epoch": 2.016460905349794, |
| "grad_norm": 1.964439423485233, |
| "kl": 0.104248046875, |
| "learning_rate": 3.327272727272727e-07, |
| "loss": 0.0042, |
| "reward": 1.9375, |
| "reward_std": 0.1767766922712326, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.96875, |
| "step": 734 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.609375, |
| "epoch": 2.01920438957476, |
| "grad_norm": 4.757411261475242, |
| "kl": 0.1103515625, |
| "learning_rate": 3.318181818181818e-07, |
| "loss": 0.0044, |
| "reward": 1.8453125357627869, |
| "reward_std": 0.3698854446411133, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9234375059604645, |
| "step": 735 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.828125, |
| "epoch": 2.021947873799726, |
| "grad_norm": 4.546067844658221, |
| "kl": 0.1025390625, |
| "learning_rate": 3.309090909090909e-07, |
| "loss": 0.0041, |
| "reward": 1.6734375357627869, |
| "reward_std": 0.25190678983926773, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8453124761581421, |
| "step": 736 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.8125, |
| "epoch": 2.0246913580246915, |
| "grad_norm": 3.9872697883190553, |
| "kl": 0.093994140625, |
| "learning_rate": 3.3e-07, |
| "loss": 0.0038, |
| "reward": 1.607812523841858, |
| "reward_std": 0.29563237726688385, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8109374940395355, |
| "step": 737 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.953125, |
| "epoch": 2.027434842249657, |
| "grad_norm": 2.421994079021264, |
| "kl": 0.103759765625, |
| "learning_rate": 3.290909090909091e-07, |
| "loss": 0.0042, |
| "reward": 1.834375023841858, |
| "reward_std": 0.2595315985381603, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9125000238418579, |
| "step": 738 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.59375, |
| "epoch": 2.0301783264746227, |
| "grad_norm": 2.248903579189386, |
| "kl": 0.10498046875, |
| "learning_rate": 3.2818181818181816e-07, |
| "loss": 0.0042, |
| "reward": 1.9375, |
| "reward_std": 0.1767766922712326, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.9687500298023224, |
| "step": 739 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.0, |
| "epoch": 2.0329218106995883, |
| "grad_norm": 6.04867003732456, |
| "kl": 0.126953125, |
| "learning_rate": 3.272727272727273e-07, |
| "loss": 0.0051, |
| "reward": 1.576562523841858, |
| "reward_std": 0.3976454883813858, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7796874642372131, |
| "step": 740 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.390625, |
| "epoch": 2.0356652949245544, |
| "grad_norm": 4.843446654352449, |
| "kl": 0.131103515625, |
| "learning_rate": 3.2636363636363635e-07, |
| "loss": 0.0052, |
| "reward": 1.447656273841858, |
| "reward_std": 0.3401695191860199, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7289062440395355, |
| "step": 741 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.25, |
| "epoch": 2.03840877914952, |
| "grad_norm": 2.3481660236622033, |
| "kl": 0.102783203125, |
| "learning_rate": 3.254545454545454e-07, |
| "loss": 0.0041, |
| "reward": 1.5093750357627869, |
| "reward_std": 0.212132029235363, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7593750357627869, |
| "step": 742 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.53125, |
| "epoch": 2.0411522633744856, |
| "grad_norm": 2.9260190897672067, |
| "kl": 0.117431640625, |
| "learning_rate": 3.245454545454545e-07, |
| "loss": 0.0047, |
| "reward": 1.643750011920929, |
| "reward_std": 0.31682799756526947, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8312499821186066, |
| "step": 743 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.609375, |
| "epoch": 2.0438957475994513, |
| "grad_norm": 5.932083100738138, |
| "kl": 0.12353515625, |
| "learning_rate": 3.236363636363636e-07, |
| "loss": 0.0049, |
| "reward": 1.5929688215255737, |
| "reward_std": 0.5680141746997833, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.764843761920929, |
| "step": 744 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.109375, |
| "epoch": 2.046639231824417, |
| "grad_norm": 9.699257534549908, |
| "kl": 0.1025390625, |
| "learning_rate": 3.2272727272727274e-07, |
| "loss": 0.0041, |
| "reward": 1.474218726158142, |
| "reward_std": 0.518870085477829, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7242187261581421, |
| "step": 745 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.53125, |
| "epoch": 2.049382716049383, |
| "grad_norm": 1.9887217834761526, |
| "kl": 0.08251953125, |
| "learning_rate": 3.218181818181818e-07, |
| "loss": 0.0033, |
| "reward": 1.5234374403953552, |
| "reward_std": 0.34977400302886963, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7734375, |
| "step": 746 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.75, |
| "epoch": 2.0521262002743486, |
| "grad_norm": 5.05821835905173, |
| "kl": 0.10693359375, |
| "learning_rate": 3.209090909090909e-07, |
| "loss": 0.0043, |
| "reward": 1.550000011920929, |
| "reward_std": 0.46106648445129395, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.784375011920929, |
| "step": 747 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.359375, |
| "epoch": 2.054869684499314, |
| "grad_norm": 4.383821715548243, |
| "kl": 0.126953125, |
| "learning_rate": 3.2e-07, |
| "loss": 0.0051, |
| "reward": 1.8593750596046448, |
| "reward_std": 0.33364592492580414, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9218750298023224, |
| "step": 748 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.203125, |
| "epoch": 2.05761316872428, |
| "grad_norm": 3.593747564892723, |
| "kl": 0.094482421875, |
| "learning_rate": 3.190909090909091e-07, |
| "loss": 0.0038, |
| "reward": 1.0468749403953552, |
| "reward_std": 0.4195588529109955, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.53125, |
| "step": 749 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.90625, |
| "epoch": 2.0603566529492454, |
| "grad_norm": 2.070362597813592, |
| "kl": 0.106689453125, |
| "learning_rate": 3.1818181818181815e-07, |
| "loss": 0.0043, |
| "reward": 1.732812523841858, |
| "reward_std": 0.19391006976366043, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8734375238418579, |
| "step": 750 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.875, |
| "epoch": 2.063100137174211, |
| "grad_norm": 1.4578266059771372, |
| "kl": 0.107666015625, |
| "learning_rate": 3.1727272727272727e-07, |
| "loss": 0.0043, |
| "reward": 1.621874988079071, |
| "reward_std": 0.21226724609732628, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.793749988079071, |
| "step": 751 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.78125, |
| "epoch": 2.065843621399177, |
| "grad_norm": 3.0790482141875115, |
| "kl": 0.08740234375, |
| "learning_rate": 3.1636363636363634e-07, |
| "loss": 0.0035, |
| "reward": 1.6578125357627869, |
| "reward_std": 0.40604209899902344, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8296874761581421, |
| "step": 752 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.71875, |
| "epoch": 2.0685871056241427, |
| "grad_norm": 3.557197238977422, |
| "kl": 0.129150390625, |
| "learning_rate": 3.154545454545454e-07, |
| "loss": 0.0052, |
| "reward": 1.578125, |
| "reward_std": 0.45973244309425354, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.78125, |
| "step": 753 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.53125, |
| "epoch": 2.0713305898491083, |
| "grad_norm": 15.43956507169567, |
| "kl": 0.171142578125, |
| "learning_rate": 3.1454545454545454e-07, |
| "loss": 0.0069, |
| "reward": 1.709375023841858, |
| "reward_std": 0.3877876400947571, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8500000238418579, |
| "step": 754 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.6875, |
| "epoch": 2.074074074074074, |
| "grad_norm": 6.911957444749393, |
| "kl": 0.10498046875, |
| "learning_rate": 3.1363636363636366e-07, |
| "loss": 0.0042, |
| "reward": 1.774999976158142, |
| "reward_std": 0.29579664394259453, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8843749761581421, |
| "step": 755 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.3125, |
| "epoch": 2.0768175582990396, |
| "grad_norm": 3.0039043899115296, |
| "kl": 0.112060546875, |
| "learning_rate": 3.1272727272727273e-07, |
| "loss": 0.0045, |
| "reward": 1.714062511920929, |
| "reward_std": 0.2952996604144573, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.854687511920929, |
| "step": 756 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.65625, |
| "epoch": 2.0795610425240056, |
| "grad_norm": 5.021720651179575, |
| "kl": 0.1083984375, |
| "learning_rate": 3.118181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.7281250357627869, |
| "reward_std": 0.511000894010067, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8687499761581421, |
| "step": 757 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.515625, |
| "epoch": 2.0823045267489713, |
| "grad_norm": 9.233165776712532, |
| "kl": 0.097412109375, |
| "learning_rate": 3.1090909090909087e-07, |
| "loss": 0.0039, |
| "reward": 1.6695312857627869, |
| "reward_std": 0.2206084057688713, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8414062559604645, |
| "step": 758 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.796875, |
| "epoch": 2.085048010973937, |
| "grad_norm": 3.577220183692013, |
| "kl": 0.09716796875, |
| "learning_rate": 3.1e-07, |
| "loss": 0.0039, |
| "reward": 1.903124988079071, |
| "reward_std": 0.26434044539928436, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9500000178813934, |
| "step": 759 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.8125, |
| "epoch": 2.0877914951989025, |
| "grad_norm": 3.5183811498721904, |
| "kl": 0.11962890625, |
| "learning_rate": 3.0909090909090907e-07, |
| "loss": 0.0048, |
| "reward": 1.282031238079071, |
| "reward_std": 0.40158161520957947, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6570312082767487, |
| "step": 760 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.78125, |
| "epoch": 2.090534979423868, |
| "grad_norm": 3.3151064924082942, |
| "kl": 0.13818359375, |
| "learning_rate": 3.0818181818181814e-07, |
| "loss": 0.0055, |
| "reward": 1.715624988079071, |
| "reward_std": 0.298599936068058, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8406250178813934, |
| "step": 761 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.265625, |
| "epoch": 2.093278463648834, |
| "grad_norm": 9.575535716339024, |
| "kl": 0.1171875, |
| "learning_rate": 3.0727272727272726e-07, |
| "loss": 0.0047, |
| "reward": 1.682812511920929, |
| "reward_std": 0.3616396635770798, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8390624821186066, |
| "step": 762 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.90625, |
| "epoch": 2.0960219478738, |
| "grad_norm": 5.799133083162924, |
| "kl": 0.1103515625, |
| "learning_rate": 3.0636363636363633e-07, |
| "loss": 0.0044, |
| "reward": 1.443750023841858, |
| "reward_std": 0.555210679769516, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7249999940395355, |
| "step": 763 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.4375, |
| "epoch": 2.0987654320987654, |
| "grad_norm": 4.9470429795418, |
| "kl": 0.095703125, |
| "learning_rate": 3.0545454545454546e-07, |
| "loss": 0.0038, |
| "reward": 1.6390624642372131, |
| "reward_std": 0.24187414348125458, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8109375238418579, |
| "step": 764 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.015625, |
| "epoch": 2.101508916323731, |
| "grad_norm": 2.9719822175731756, |
| "kl": 0.13818359375, |
| "learning_rate": 3.0454545454545453e-07, |
| "loss": 0.0055, |
| "reward": 1.8593749403953552, |
| "reward_std": 0.18882093112915754, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.921875, |
| "step": 765 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 62.390625, |
| "epoch": 2.1042524005486967, |
| "grad_norm": 2.8071825422509127, |
| "kl": 0.112548828125, |
| "learning_rate": 3.0363636363636365e-07, |
| "loss": 0.0045, |
| "reward": 1.434374988079071, |
| "reward_std": 0.23727848380804062, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7312499582767487, |
| "step": 766 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.140625, |
| "epoch": 2.1069958847736627, |
| "grad_norm": 2.783798353824772, |
| "kl": 0.10107421875, |
| "learning_rate": 3.027272727272727e-07, |
| "loss": 0.004, |
| "reward": 1.1953125, |
| "reward_std": 0.17222431302070618, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6015624850988388, |
| "step": 767 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.125, |
| "epoch": 2.1097393689986284, |
| "grad_norm": 3.7969867351903215, |
| "kl": 0.1171875, |
| "learning_rate": 3.018181818181818e-07, |
| "loss": 0.0047, |
| "reward": 1.505468726158142, |
| "reward_std": 0.5320801958441734, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7554687261581421, |
| "step": 768 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.671875, |
| "epoch": 2.112482853223594, |
| "grad_norm": 2.4280428057760908, |
| "kl": 0.106689453125, |
| "learning_rate": 3.009090909090909e-07, |
| "loss": 0.0043, |
| "reward": 1.03125, |
| "reward_std": 0.22097086161375046, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5312499850988388, |
| "step": 769 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.109375, |
| "epoch": 2.1152263374485596, |
| "grad_norm": 4.325617079844216, |
| "kl": 0.104248046875, |
| "learning_rate": 3e-07, |
| "loss": 0.0042, |
| "reward": 1.3859374523162842, |
| "reward_std": 0.36535846441984177, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6984374523162842, |
| "step": 770 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.921875, |
| "epoch": 2.117969821673525, |
| "grad_norm": 2.602909903575354, |
| "kl": 0.0966796875, |
| "learning_rate": 2.9909090909090906e-07, |
| "loss": 0.0039, |
| "reward": 1.7406249046325684, |
| "reward_std": 0.34644468128681183, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8656249940395355, |
| "step": 771 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.765625, |
| "epoch": 2.1207133058984913, |
| "grad_norm": 3.290830115950048, |
| "kl": 0.142578125, |
| "learning_rate": 2.9818181818181813e-07, |
| "loss": 0.0057, |
| "reward": 1.475000023841858, |
| "reward_std": 0.3219795897603035, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7406249940395355, |
| "step": 772 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.625, |
| "epoch": 2.123456790123457, |
| "grad_norm": 10.577530102589195, |
| "kl": 0.16455078125, |
| "learning_rate": 2.972727272727273e-07, |
| "loss": 0.0066, |
| "reward": 1.470312476158142, |
| "reward_std": 0.5060344040393829, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7359375059604645, |
| "step": 773 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.015625, |
| "epoch": 2.1262002743484225, |
| "grad_norm": 6.183379887663399, |
| "kl": 0.140380859375, |
| "learning_rate": 2.963636363636364e-07, |
| "loss": 0.0056, |
| "reward": 1.6015625, |
| "reward_std": 0.7116466164588928, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8046875298023224, |
| "step": 774 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.359375, |
| "epoch": 2.128943758573388, |
| "grad_norm": 3.3452115484957266, |
| "kl": 0.110595703125, |
| "learning_rate": 2.9545454545454545e-07, |
| "loss": 0.0044, |
| "reward": 1.5499999523162842, |
| "reward_std": 0.3921141056343913, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7843749523162842, |
| "step": 775 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.96875, |
| "epoch": 2.1316872427983538, |
| "grad_norm": 3.8120103516787482, |
| "kl": 0.10693359375, |
| "learning_rate": 2.945454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.4554687142372131, |
| "reward_std": 0.6374655365943909, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7367187440395355, |
| "step": 776 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.046875, |
| "epoch": 2.13443072702332, |
| "grad_norm": 3.7087021187291227, |
| "kl": 0.126953125, |
| "learning_rate": 2.9363636363636364e-07, |
| "loss": 0.0051, |
| "reward": 1.8093749284744263, |
| "reward_std": 0.40407513082027435, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.903124988079071, |
| "step": 777 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.328125, |
| "epoch": 2.1371742112482854, |
| "grad_norm": 3.680864303291531, |
| "kl": 0.118408203125, |
| "learning_rate": 2.927272727272727e-07, |
| "loss": 0.0047, |
| "reward": 1.662500023841858, |
| "reward_std": 0.3835296258330345, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8343750238418579, |
| "step": 778 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.171875, |
| "epoch": 2.139917695473251, |
| "grad_norm": 5.814044481438807, |
| "kl": 0.107666015625, |
| "learning_rate": 2.918181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.0546875, |
| "reward_std": 0.33707825164310634, |
| "rewards/choice_reward": 0.515625, |
| "rewards/roam_reward": 0.5390624403953552, |
| "step": 779 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.046875, |
| "epoch": 2.1426611796982167, |
| "grad_norm": 11.543603320171377, |
| "kl": 0.17236328125, |
| "learning_rate": 2.909090909090909e-07, |
| "loss": 0.0069, |
| "reward": 1.418749988079071, |
| "reward_std": 0.4447284936904907, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.715624988079071, |
| "step": 780 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.796875, |
| "epoch": 2.1454046639231823, |
| "grad_norm": 4.414795879502755, |
| "kl": 0.104248046875, |
| "learning_rate": 2.9e-07, |
| "loss": 0.0042, |
| "reward": 1.6125000715255737, |
| "reward_std": 0.4747316986322403, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8156249821186066, |
| "step": 781 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.484375, |
| "epoch": 2.148148148148148, |
| "grad_norm": 3.508188786631475, |
| "kl": 0.1298828125, |
| "learning_rate": 2.8909090909090905e-07, |
| "loss": 0.0052, |
| "reward": 1.658593773841858, |
| "reward_std": 0.2683626413345337, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8304687738418579, |
| "step": 782 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.0625, |
| "epoch": 2.150891632373114, |
| "grad_norm": 4.240108432429567, |
| "kl": 0.14697265625, |
| "learning_rate": 2.8818181818181817e-07, |
| "loss": 0.0059, |
| "reward": 1.4374999403953552, |
| "reward_std": 0.6624129414558411, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7187499701976776, |
| "step": 783 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.90625, |
| "epoch": 2.1536351165980796, |
| "grad_norm": 5.274311339934734, |
| "kl": 0.1181640625, |
| "learning_rate": 2.872727272727273e-07, |
| "loss": 0.0047, |
| "reward": 1.728124976158142, |
| "reward_std": 0.09722718596458435, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8687499761581421, |
| "step": 784 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.640625, |
| "epoch": 2.156378600823045, |
| "grad_norm": 4.175305160972081, |
| "kl": 0.13818359375, |
| "learning_rate": 2.8636363636363637e-07, |
| "loss": 0.0055, |
| "reward": 1.618749976158142, |
| "reward_std": 0.4070549011230469, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7906250059604645, |
| "step": 785 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.609375, |
| "epoch": 2.159122085048011, |
| "grad_norm": 3.509394846684709, |
| "kl": 0.10498046875, |
| "learning_rate": 2.8545454545454544e-07, |
| "loss": 0.0042, |
| "reward": 1.456250011920929, |
| "reward_std": 0.3542475253343582, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.737500011920929, |
| "step": 786 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.609375, |
| "epoch": 2.161865569272977, |
| "grad_norm": 3.8637174394342217, |
| "kl": 0.112548828125, |
| "learning_rate": 2.845454545454545e-07, |
| "loss": 0.0045, |
| "reward": 1.7562499642372131, |
| "reward_std": 0.36303091794252396, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8812499642372131, |
| "step": 787 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.625, |
| "epoch": 2.1646090534979425, |
| "grad_norm": 3.0933947112166003, |
| "kl": 0.12158203125, |
| "learning_rate": 2.8363636363636363e-07, |
| "loss": 0.0049, |
| "reward": 1.4867187142372131, |
| "reward_std": 0.5142303705215454, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7523437142372131, |
| "step": 788 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.75, |
| "epoch": 2.167352537722908, |
| "grad_norm": 4.061168976745869, |
| "kl": 0.09228515625, |
| "learning_rate": 2.827272727272727e-07, |
| "loss": 0.0037, |
| "reward": 1.6757811903953552, |
| "reward_std": 0.46660368144512177, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8320312201976776, |
| "step": 789 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 55.5625, |
| "epoch": 2.1700960219478738, |
| "grad_norm": 1.2070475469082251, |
| "kl": 0.11083984375, |
| "learning_rate": 2.8181818181818177e-07, |
| "loss": 0.0044, |
| "reward": 1.732812523841858, |
| "reward_std": 0.08396892994642258, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8734374642372131, |
| "step": 790 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.734375, |
| "epoch": 2.1728395061728394, |
| "grad_norm": 4.913584657192109, |
| "kl": 0.0986328125, |
| "learning_rate": 2.809090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.5148437023162842, |
| "reward_std": 0.358761690557003, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7648437321186066, |
| "step": 791 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.40625, |
| "epoch": 2.175582990397805, |
| "grad_norm": 3.2707132647043875, |
| "kl": 0.1279296875, |
| "learning_rate": 2.8e-07, |
| "loss": 0.0051, |
| "reward": 1.603906273841858, |
| "reward_std": 0.5658104568719864, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7914062440395355, |
| "step": 792 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.609375, |
| "epoch": 2.178326474622771, |
| "grad_norm": 4.3355832161580246, |
| "kl": 0.126953125, |
| "learning_rate": 2.790909090909091e-07, |
| "loss": 0.0051, |
| "reward": 1.7234375476837158, |
| "reward_std": 0.47183680534362793, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.864062488079071, |
| "step": 793 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 66.859375, |
| "epoch": 2.1810699588477367, |
| "grad_norm": 4.129358376134468, |
| "kl": 0.108642578125, |
| "learning_rate": 2.7818181818181816e-07, |
| "loss": 0.0043, |
| "reward": 1.830468773841858, |
| "reward_std": 0.4118579924106598, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9085937440395355, |
| "step": 794 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.859375, |
| "epoch": 2.1838134430727023, |
| "grad_norm": 4.373027044183056, |
| "kl": 0.11376953125, |
| "learning_rate": 2.772727272727273e-07, |
| "loss": 0.0046, |
| "reward": 1.7937499284744263, |
| "reward_std": 0.29688415490090847, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.871874988079071, |
| "step": 795 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.015625, |
| "epoch": 2.186556927297668, |
| "grad_norm": 2.5642796130070566, |
| "kl": 0.0908203125, |
| "learning_rate": 2.7636363636363635e-07, |
| "loss": 0.0036, |
| "reward": 1.6804687976837158, |
| "reward_std": 0.2819945365190506, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.836718738079071, |
| "step": 796 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.484375, |
| "epoch": 2.1893004115226335, |
| "grad_norm": 5.357012535761609, |
| "kl": 0.259521484375, |
| "learning_rate": 2.754545454545454e-07, |
| "loss": 0.0104, |
| "reward": 1.837499976158142, |
| "reward_std": 0.2781676575541496, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.8843750059604645, |
| "step": 797 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 69.671875, |
| "epoch": 2.1920438957475996, |
| "grad_norm": 4.22130164678084, |
| "kl": 0.111083984375, |
| "learning_rate": 2.7454545454545455e-07, |
| "loss": 0.0044, |
| "reward": 1.357812523841858, |
| "reward_std": 0.5226652920246124, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859375238418579, |
| "step": 798 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.59375, |
| "epoch": 2.1947873799725652, |
| "grad_norm": 2.6780966138783278, |
| "kl": 0.0872802734375, |
| "learning_rate": 2.736363636363636e-07, |
| "loss": 0.0035, |
| "reward": 1.5828125476837158, |
| "reward_std": 0.3197706267237663, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.801562488079071, |
| "step": 799 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.703125, |
| "epoch": 2.197530864197531, |
| "grad_norm": 3.2315439352160684, |
| "kl": 0.09912109375, |
| "learning_rate": 2.727272727272727e-07, |
| "loss": 0.004, |
| "reward": 1.696874976158142, |
| "reward_std": 0.37345467507839203, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8531249761581421, |
| "step": 800 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.609375, |
| "epoch": 2.2002743484224965, |
| "grad_norm": 6.333539078309606, |
| "kl": 0.098388671875, |
| "learning_rate": 2.7181818181818176e-07, |
| "loss": 0.0039, |
| "reward": 1.623437523841858, |
| "reward_std": 0.5012295097112656, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8109375238418579, |
| "step": 801 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.4375, |
| "epoch": 2.203017832647462, |
| "grad_norm": 2.232547080842151, |
| "kl": 0.095947265625, |
| "learning_rate": 2.7090909090909094e-07, |
| "loss": 0.0038, |
| "reward": 1.8656249642372131, |
| "reward_std": 0.21204502135515213, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9281249940395355, |
| "step": 802 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.4375, |
| "epoch": 2.205761316872428, |
| "grad_norm": 3.8566382494589813, |
| "kl": 0.10107421875, |
| "learning_rate": 2.7e-07, |
| "loss": 0.004, |
| "reward": 1.2835937142372131, |
| "reward_std": 0.2889275439083576, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6585937142372131, |
| "step": 803 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.171875, |
| "epoch": 2.2085048010973938, |
| "grad_norm": 2.921891948516463, |
| "kl": 0.114013671875, |
| "learning_rate": 2.690909090909091e-07, |
| "loss": 0.0046, |
| "reward": 1.580468773841858, |
| "reward_std": 0.3070296049118042, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7835937440395355, |
| "step": 804 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.703125, |
| "epoch": 2.2112482853223594, |
| "grad_norm": 6.092366474924012, |
| "kl": 0.121826171875, |
| "learning_rate": 2.6818181818181815e-07, |
| "loss": 0.0049, |
| "reward": 1.8203125, |
| "reward_std": 0.44890232384204865, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.8828125, |
| "step": 805 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.90625, |
| "epoch": 2.213991769547325, |
| "grad_norm": 0.8292229427625117, |
| "kl": 0.0849609375, |
| "learning_rate": 2.672727272727273e-07, |
| "loss": 0.0034, |
| "reward": 1.731249988079071, |
| "reward_std": 0.0578637570142746, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.856249988079071, |
| "step": 806 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.203125, |
| "epoch": 2.2167352537722906, |
| "grad_norm": 4.975513240528351, |
| "kl": 0.1103515625, |
| "learning_rate": 2.6636363636363634e-07, |
| "loss": 0.0044, |
| "reward": 1.4742187857627869, |
| "reward_std": 0.46045906841754913, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7398437559604645, |
| "step": 807 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.015625, |
| "epoch": 2.2194787379972567, |
| "grad_norm": 4.22241272304683, |
| "kl": 0.1337890625, |
| "learning_rate": 2.654545454545454e-07, |
| "loss": 0.0054, |
| "reward": 1.574999988079071, |
| "reward_std": 0.35842348635196686, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.778124988079071, |
| "step": 808 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.4375, |
| "epoch": 2.2222222222222223, |
| "grad_norm": 4.555498756335306, |
| "kl": 0.10791015625, |
| "learning_rate": 2.6454545454545454e-07, |
| "loss": 0.0043, |
| "reward": 1.1343750357627869, |
| "reward_std": 0.520625114440918, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5874999761581421, |
| "step": 809 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.5625, |
| "epoch": 2.224965706447188, |
| "grad_norm": 3.437950147489919, |
| "kl": 0.09033203125, |
| "learning_rate": 2.636363636363636e-07, |
| "loss": 0.0036, |
| "reward": 1.4007813334465027, |
| "reward_std": 0.5314416140317917, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.6820312440395355, |
| "step": 810 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.765625, |
| "epoch": 2.2277091906721536, |
| "grad_norm": 3.8079860024144585, |
| "kl": 0.10302734375, |
| "learning_rate": 2.6272727272727273e-07, |
| "loss": 0.0041, |
| "reward": 1.717187523841858, |
| "reward_std": 0.1220581904053688, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8578125238418579, |
| "step": 811 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.21875, |
| "epoch": 2.230452674897119, |
| "grad_norm": 3.0291385388945766, |
| "kl": 0.123779296875, |
| "learning_rate": 2.618181818181818e-07, |
| "loss": 0.005, |
| "reward": 1.734375, |
| "reward_std": 0.3770694434642792, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.84375, |
| "step": 812 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.296875, |
| "epoch": 2.2331961591220852, |
| "grad_norm": 5.123582714680745, |
| "kl": 0.104736328125, |
| "learning_rate": 2.6090909090909093e-07, |
| "loss": 0.0042, |
| "reward": 1.821874976158142, |
| "reward_std": 0.294886939227581, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9156250059604645, |
| "step": 813 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.96875, |
| "epoch": 2.235939643347051, |
| "grad_norm": 2.1805429532884464, |
| "kl": 0.099609375, |
| "learning_rate": 2.6e-07, |
| "loss": 0.004, |
| "reward": 1.7578125, |
| "reward_std": 0.2911372510716319, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8828125, |
| "step": 814 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 71.359375, |
| "epoch": 2.2386831275720165, |
| "grad_norm": 14.093464621240487, |
| "kl": 0.13134765625, |
| "learning_rate": 2.5909090909090907e-07, |
| "loss": 0.0052, |
| "reward": 1.603124976158142, |
| "reward_std": 0.37940096855163574, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8062500059604645, |
| "step": 815 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.96875, |
| "epoch": 2.241426611796982, |
| "grad_norm": 11.832522126618683, |
| "kl": 0.10791015625, |
| "learning_rate": 2.581818181818182e-07, |
| "loss": 0.0043, |
| "reward": 1.7578125596046448, |
| "reward_std": 0.5320637822151184, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8671875, |
| "step": 816 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.65625, |
| "epoch": 2.2441700960219477, |
| "grad_norm": 6.779992242783354, |
| "kl": 0.1259765625, |
| "learning_rate": 2.5727272727272726e-07, |
| "loss": 0.005, |
| "reward": 1.647656261920929, |
| "reward_std": 0.5611380934715271, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8195312321186066, |
| "step": 817 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.78125, |
| "epoch": 2.246913580246914, |
| "grad_norm": 5.229875349558636, |
| "kl": 0.114990234375, |
| "learning_rate": 2.5636363636363633e-07, |
| "loss": 0.0046, |
| "reward": 1.3835937976837158, |
| "reward_std": 0.5313563942909241, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.696093738079071, |
| "step": 818 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.046875, |
| "epoch": 2.2496570644718794, |
| "grad_norm": 5.072122967999993, |
| "kl": 0.14697265625, |
| "learning_rate": 2.554545454545454e-07, |
| "loss": 0.0059, |
| "reward": 1.232812523841858, |
| "reward_std": 0.5871487855911255, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6234374940395355, |
| "step": 819 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.703125, |
| "epoch": 2.252400548696845, |
| "grad_norm": 2.985874495956574, |
| "kl": 0.13134765625, |
| "learning_rate": 2.5454545454545453e-07, |
| "loss": 0.0053, |
| "reward": 1.8359375, |
| "reward_std": 0.30598607659339905, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.9140624701976776, |
| "step": 820 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.703125, |
| "epoch": 2.2551440329218106, |
| "grad_norm": 3.556409499813488, |
| "kl": 0.12451171875, |
| "learning_rate": 2.5363636363636365e-07, |
| "loss": 0.005, |
| "reward": 1.55078125, |
| "reward_std": 0.4785647839307785, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.78515625, |
| "step": 821 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.765625, |
| "epoch": 2.2578875171467763, |
| "grad_norm": 2.6368990361460876, |
| "kl": 0.104736328125, |
| "learning_rate": 2.527272727272727e-07, |
| "loss": 0.0042, |
| "reward": 1.315625011920929, |
| "reward_std": 0.34501613676548004, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6749999821186066, |
| "step": 822 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.890625, |
| "epoch": 2.260631001371742, |
| "grad_norm": 4.607358551482156, |
| "kl": 0.117431640625, |
| "learning_rate": 2.518181818181818e-07, |
| "loss": 0.0047, |
| "reward": 1.374218761920929, |
| "reward_std": 0.3684770464897156, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.7023437023162842, |
| "step": 823 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 72.265625, |
| "epoch": 2.263374485596708, |
| "grad_norm": 4.031602805941072, |
| "kl": 0.098388671875, |
| "learning_rate": 2.509090909090909e-07, |
| "loss": 0.0039, |
| "reward": 1.1226562857627869, |
| "reward_std": 0.406386137008667, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5757812559604645, |
| "step": 824 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.953125, |
| "epoch": 2.2661179698216736, |
| "grad_norm": 3.5991674682192696, |
| "kl": 0.126953125, |
| "learning_rate": 2.5e-07, |
| "loss": 0.0051, |
| "reward": 1.3109374642372131, |
| "reward_std": 0.2621586322784424, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6703124344348907, |
| "step": 825 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.46875, |
| "epoch": 2.268861454046639, |
| "grad_norm": 4.67083394246974, |
| "kl": 0.110107421875, |
| "learning_rate": 2.4909090909090906e-07, |
| "loss": 0.0044, |
| "reward": 1.0859375, |
| "reward_std": 0.6535941660404205, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5546875, |
| "step": 826 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.28125, |
| "epoch": 2.271604938271605, |
| "grad_norm": 1.9804141623697706, |
| "kl": 0.09619140625, |
| "learning_rate": 2.481818181818182e-07, |
| "loss": 0.0039, |
| "reward": 1.7890625, |
| "reward_std": 0.09280776232481003, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8984375, |
| "step": 827 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.359375, |
| "epoch": 2.274348422496571, |
| "grad_norm": 9.729038956473259, |
| "kl": 0.09228515625, |
| "learning_rate": 2.4727272727272725e-07, |
| "loss": 0.0037, |
| "reward": 1.692187488079071, |
| "reward_std": 0.3357216417789459, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8328125178813934, |
| "step": 828 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.75, |
| "epoch": 2.2770919067215365, |
| "grad_norm": 3.8855375569193304, |
| "kl": 0.115966796875, |
| "learning_rate": 2.463636363636364e-07, |
| "loss": 0.0046, |
| "reward": 1.4171874523162842, |
| "reward_std": 0.7684605121612549, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7140624821186066, |
| "step": 829 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.875, |
| "epoch": 2.279835390946502, |
| "grad_norm": 3.15608198733615, |
| "kl": 0.123046875, |
| "learning_rate": 2.4545454545454545e-07, |
| "loss": 0.0049, |
| "reward": 1.608593761920929, |
| "reward_std": 0.46275579929351807, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.811718761920929, |
| "step": 830 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.125, |
| "epoch": 2.2825788751714677, |
| "grad_norm": 3.225464493364868, |
| "kl": 0.09423828125, |
| "learning_rate": 2.445454545454545e-07, |
| "loss": 0.0038, |
| "reward": 1.564062476158142, |
| "reward_std": 0.42860206216573715, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7828125059604645, |
| "step": 831 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.8125, |
| "epoch": 2.2853223593964334, |
| "grad_norm": 7.063770706719847, |
| "kl": 0.1162109375, |
| "learning_rate": 2.436363636363636e-07, |
| "loss": 0.0046, |
| "reward": 1.724218726158142, |
| "reward_std": 0.274951308965683, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8492187559604645, |
| "step": 832 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.625, |
| "epoch": 2.288065843621399, |
| "grad_norm": 2.3658771387066144, |
| "kl": 0.14892578125, |
| "learning_rate": 2.427272727272727e-07, |
| "loss": 0.006, |
| "reward": 1.581250011920929, |
| "reward_std": 0.3219865560531616, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.800000011920929, |
| "step": 833 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.609375, |
| "epoch": 2.290809327846365, |
| "grad_norm": 39.96062417184321, |
| "kl": 0.108154296875, |
| "learning_rate": 2.418181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.770312488079071, |
| "reward_std": 0.48108595609664917, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.864062488079071, |
| "step": 834 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.203125, |
| "epoch": 2.2935528120713307, |
| "grad_norm": 1.548739464441859, |
| "kl": 0.102783203125, |
| "learning_rate": 2.409090909090909e-07, |
| "loss": 0.0041, |
| "reward": 1.567187488079071, |
| "reward_std": 0.21341075748205185, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7859375178813934, |
| "step": 835 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.609375, |
| "epoch": 2.2962962962962963, |
| "grad_norm": 2.6372360900746297, |
| "kl": 0.1005859375, |
| "learning_rate": 2.4e-07, |
| "loss": 0.004, |
| "reward": 1.864843726158142, |
| "reward_std": 0.2521742656826973, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9273437559604645, |
| "step": 836 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.40625, |
| "epoch": 2.299039780521262, |
| "grad_norm": 6.055698129525655, |
| "kl": 0.12890625, |
| "learning_rate": 2.390909090909091e-07, |
| "loss": 0.0052, |
| "reward": 1.535937488079071, |
| "reward_std": 0.748941570520401, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7703125178813934, |
| "step": 837 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.625, |
| "epoch": 2.3017832647462275, |
| "grad_norm": 5.718085633049784, |
| "kl": 0.102294921875, |
| "learning_rate": 2.3818181818181817e-07, |
| "loss": 0.0041, |
| "reward": 1.934374988079071, |
| "reward_std": 0.18561553210020065, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.9656250178813934, |
| "step": 838 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.609375, |
| "epoch": 2.3045267489711936, |
| "grad_norm": 3.4220848985214447, |
| "kl": 0.1708984375, |
| "learning_rate": 2.3727272727272727e-07, |
| "loss": 0.0068, |
| "reward": 1.5148438215255737, |
| "reward_std": 0.19026567973196507, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.764843761920929, |
| "step": 839 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.859375, |
| "epoch": 2.307270233196159, |
| "grad_norm": 3.883757282531534, |
| "kl": 0.124267578125, |
| "learning_rate": 2.3636363636363634e-07, |
| "loss": 0.005, |
| "reward": 1.614062488079071, |
| "reward_std": 0.28888290375471115, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.7859374582767487, |
| "step": 840 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.0625, |
| "epoch": 2.310013717421125, |
| "grad_norm": 2.571179323855547, |
| "kl": 0.083251953125, |
| "learning_rate": 2.3545454545454544e-07, |
| "loss": 0.0033, |
| "reward": 1.641406238079071, |
| "reward_std": 0.3707147389650345, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.813281238079071, |
| "step": 841 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.296875, |
| "epoch": 2.3127572016460904, |
| "grad_norm": 7.098184106639789, |
| "kl": 0.117431640625, |
| "learning_rate": 2.3454545454545454e-07, |
| "loss": 0.0047, |
| "reward": 1.6953125, |
| "reward_std": 0.3832200914621353, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8515624701976776, |
| "step": 842 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 75.140625, |
| "epoch": 2.315500685871056, |
| "grad_norm": 4.19099378367587, |
| "kl": 0.11181640625, |
| "learning_rate": 2.3363636363636363e-07, |
| "loss": 0.0045, |
| "reward": 1.786718726158142, |
| "reward_std": 0.33557650446891785, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8960937559604645, |
| "step": 843 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.78125, |
| "epoch": 2.318244170096022, |
| "grad_norm": 4.962168354226179, |
| "kl": 0.117431640625, |
| "learning_rate": 2.3272727272727273e-07, |
| "loss": 0.0047, |
| "reward": 1.205468773841858, |
| "reward_std": 0.5368544310331345, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6117187738418579, |
| "step": 844 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.203125, |
| "epoch": 2.3209876543209877, |
| "grad_norm": 3.709767154383809, |
| "kl": 0.104736328125, |
| "learning_rate": 2.318181818181818e-07, |
| "loss": 0.0042, |
| "reward": 1.483593761920929, |
| "reward_std": 0.32407404482364655, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7492187321186066, |
| "step": 845 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.375, |
| "epoch": 2.3237311385459534, |
| "grad_norm": 3.826413640525221, |
| "kl": 0.08447265625, |
| "learning_rate": 2.309090909090909e-07, |
| "loss": 0.0034, |
| "reward": 1.31640625, |
| "reward_std": 0.5299557000398636, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.67578125, |
| "step": 846 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.578125, |
| "epoch": 2.326474622770919, |
| "grad_norm": 2.651734680987155, |
| "kl": 0.10791015625, |
| "learning_rate": 2.3e-07, |
| "loss": 0.0043, |
| "reward": 1.4578125476837158, |
| "reward_std": 0.3175119236111641, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7390625178813934, |
| "step": 847 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.53125, |
| "epoch": 2.3292181069958846, |
| "grad_norm": 3.1907652990124857, |
| "kl": 0.112060546875, |
| "learning_rate": 2.290909090909091e-07, |
| "loss": 0.0045, |
| "reward": 1.5390625, |
| "reward_std": 0.5349350571632385, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7734374701976776, |
| "step": 848 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.515625, |
| "epoch": 2.3319615912208507, |
| "grad_norm": 4.01003089380689, |
| "kl": 0.111083984375, |
| "learning_rate": 2.2818181818181816e-07, |
| "loss": 0.0044, |
| "reward": 1.785937488079071, |
| "reward_std": 0.39371173083782196, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.8796875178813934, |
| "step": 849 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.15625, |
| "epoch": 2.3347050754458163, |
| "grad_norm": 51.61948966412654, |
| "kl": 0.10498046875, |
| "learning_rate": 2.2727272727272726e-07, |
| "loss": 0.0042, |
| "reward": 1.471875011920929, |
| "reward_std": 0.30076853930950165, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7374999821186066, |
| "step": 850 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.015625, |
| "epoch": 2.337448559670782, |
| "grad_norm": 4.390409835085189, |
| "kl": 0.090087890625, |
| "learning_rate": 2.2636363636363633e-07, |
| "loss": 0.0036, |
| "reward": 1.4320312142372131, |
| "reward_std": 0.22010089457035065, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7289062142372131, |
| "step": 851 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.578125, |
| "epoch": 2.3401920438957475, |
| "grad_norm": 3.194735158147514, |
| "kl": 0.096435546875, |
| "learning_rate": 2.2545454545454545e-07, |
| "loss": 0.0039, |
| "reward": 1.623437523841858, |
| "reward_std": 0.325691282749176, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8109374940395355, |
| "step": 852 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.09375, |
| "epoch": 2.342935528120713, |
| "grad_norm": 3.239870186539037, |
| "kl": 0.117431640625, |
| "learning_rate": 2.2454545454545453e-07, |
| "loss": 0.0047, |
| "reward": 1.38671875, |
| "reward_std": 0.4322517514228821, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.6992187649011612, |
| "step": 853 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.25, |
| "epoch": 2.3456790123456788, |
| "grad_norm": 4.085367729309037, |
| "kl": 0.09765625, |
| "learning_rate": 2.2363636363636362e-07, |
| "loss": 0.0039, |
| "reward": 1.532031238079071, |
| "reward_std": 0.7208892107009888, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.766406238079071, |
| "step": 854 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.1875, |
| "epoch": 2.348422496570645, |
| "grad_norm": 20.88507515916802, |
| "kl": 0.115966796875, |
| "learning_rate": 2.2272727272727272e-07, |
| "loss": 0.0046, |
| "reward": 1.4679687023162842, |
| "reward_std": 0.5433869957923889, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7335937023162842, |
| "step": 855 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.390625, |
| "epoch": 2.3511659807956105, |
| "grad_norm": 5.171802093144948, |
| "kl": 0.131591796875, |
| "learning_rate": 2.2181818181818182e-07, |
| "loss": 0.0053, |
| "reward": 1.754687488079071, |
| "reward_std": 0.3516154810786247, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.879687488079071, |
| "step": 856 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.765625, |
| "epoch": 2.353909465020576, |
| "grad_norm": 2.6822859489888233, |
| "kl": 0.119384765625, |
| "learning_rate": 2.2090909090909091e-07, |
| "loss": 0.0048, |
| "reward": 1.792187511920929, |
| "reward_std": 0.33487100899219513, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.885937511920929, |
| "step": 857 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.203125, |
| "epoch": 2.3566529492455417, |
| "grad_norm": 3.852181269092699, |
| "kl": 0.138427734375, |
| "learning_rate": 2.1999999999999998e-07, |
| "loss": 0.0055, |
| "reward": 1.567187488079071, |
| "reward_std": 0.43235016614198685, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.785937488079071, |
| "step": 858 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.203125, |
| "epoch": 2.3593964334705078, |
| "grad_norm": 9.283208203193293, |
| "kl": 0.11376953125, |
| "learning_rate": 2.1909090909090908e-07, |
| "loss": 0.0046, |
| "reward": 1.4859375357627869, |
| "reward_std": 0.25532449781894684, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7359375059604645, |
| "step": 859 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.546875, |
| "epoch": 2.3621399176954734, |
| "grad_norm": 4.301864973090632, |
| "kl": 0.0986328125, |
| "learning_rate": 2.1818181818181815e-07, |
| "loss": 0.0039, |
| "reward": 1.053125023841858, |
| "reward_std": 0.3161915987730026, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5218749940395355, |
| "step": 860 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.921875, |
| "epoch": 2.364883401920439, |
| "grad_norm": 3.942303526415412, |
| "kl": 0.115234375, |
| "learning_rate": 2.1727272727272728e-07, |
| "loss": 0.0046, |
| "reward": 1.7210937142372131, |
| "reward_std": 0.3669208139181137, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8304687738418579, |
| "step": 861 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.53125, |
| "epoch": 2.3676268861454046, |
| "grad_norm": 4.143736465551136, |
| "kl": 0.099609375, |
| "learning_rate": 2.1636363636363635e-07, |
| "loss": 0.004, |
| "reward": 1.525781273841858, |
| "reward_std": 0.15028471499681473, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7601562440395355, |
| "step": 862 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.5, |
| "epoch": 2.3703703703703702, |
| "grad_norm": 2.643244037430125, |
| "kl": 0.10498046875, |
| "learning_rate": 2.1545454545454544e-07, |
| "loss": 0.0042, |
| "reward": 1.635156273841858, |
| "reward_std": 0.33017241954803467, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8226562738418579, |
| "step": 863 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.484375, |
| "epoch": 2.373113854595336, |
| "grad_norm": 3.12918081111299, |
| "kl": 0.1162109375, |
| "learning_rate": 2.1454545454545454e-07, |
| "loss": 0.0046, |
| "reward": 1.5859375, |
| "reward_std": 0.27471774071455, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7890625, |
| "step": 864 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.328125, |
| "epoch": 2.375857338820302, |
| "grad_norm": 4.868710249582456, |
| "kl": 0.109619140625, |
| "learning_rate": 2.136363636363636e-07, |
| "loss": 0.0044, |
| "reward": 1.428125023841858, |
| "reward_std": 0.5239690691232681, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7249999642372131, |
| "step": 865 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.5625, |
| "epoch": 2.3786008230452675, |
| "grad_norm": 6.729707203115704, |
| "kl": 0.1025390625, |
| "learning_rate": 2.1272727272727274e-07, |
| "loss": 0.0041, |
| "reward": 1.515625, |
| "reward_std": 0.5596156716346741, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.765625, |
| "step": 866 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.953125, |
| "epoch": 2.381344307270233, |
| "grad_norm": 2.627118608764886, |
| "kl": 0.0927734375, |
| "learning_rate": 2.118181818181818e-07, |
| "loss": 0.0037, |
| "reward": 1.766406238079071, |
| "reward_std": 0.14580436330288649, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.875781238079071, |
| "step": 867 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.21875, |
| "epoch": 2.384087791495199, |
| "grad_norm": 40.22424412671131, |
| "kl": 0.1025390625, |
| "learning_rate": 2.109090909090909e-07, |
| "loss": 0.0041, |
| "reward": 1.3320311903953552, |
| "reward_std": 0.2978522703051567, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6757812201976776, |
| "step": 868 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.40625, |
| "epoch": 2.386831275720165, |
| "grad_norm": 2.801683774665442, |
| "kl": 0.112060546875, |
| "learning_rate": 2.0999999999999997e-07, |
| "loss": 0.0045, |
| "reward": 1.665624976158142, |
| "reward_std": 0.37274664640426636, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8375000059604645, |
| "step": 869 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.046875, |
| "epoch": 2.3895747599451305, |
| "grad_norm": 7.799828848279018, |
| "kl": 0.09423828125, |
| "learning_rate": 2.0909090909090907e-07, |
| "loss": 0.0038, |
| "reward": 0.8023437634110451, |
| "reward_std": 0.18869349360466003, |
| "rewards/choice_reward": 0.375, |
| "rewards/roam_reward": 0.4273437634110451, |
| "step": 870 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.0625, |
| "epoch": 2.392318244170096, |
| "grad_norm": 3.5958536385128066, |
| "kl": 0.096923828125, |
| "learning_rate": 2.0818181818181817e-07, |
| "loss": 0.0039, |
| "reward": 1.5695312023162842, |
| "reward_std": 0.5512479394674301, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.7570312321186066, |
| "step": 871 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.234375, |
| "epoch": 2.3950617283950617, |
| "grad_norm": 5.192872918934563, |
| "kl": 0.161376953125, |
| "learning_rate": 2.0727272727272727e-07, |
| "loss": 0.0065, |
| "reward": 1.4531249403953552, |
| "reward_std": 0.4561312794685364, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7343749701976776, |
| "step": 872 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.3125, |
| "epoch": 2.3978052126200273, |
| "grad_norm": 5.315526210318176, |
| "kl": 0.13720703125, |
| "learning_rate": 2.0636363636363636e-07, |
| "loss": 0.0055, |
| "reward": 1.651562511920929, |
| "reward_std": 0.42224298417568207, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.823437511920929, |
| "step": 873 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.015625, |
| "epoch": 2.400548696844993, |
| "grad_norm": 2.0419602089837015, |
| "kl": 0.108154296875, |
| "learning_rate": 2.0545454545454543e-07, |
| "loss": 0.0043, |
| "reward": 1.901562511920929, |
| "reward_std": 0.1383042959496379, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9484374821186066, |
| "step": 874 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.546875, |
| "epoch": 2.403292181069959, |
| "grad_norm": 10.813583170383565, |
| "kl": 0.111083984375, |
| "learning_rate": 2.0454545454545456e-07, |
| "loss": 0.0044, |
| "reward": 1.5453124642372131, |
| "reward_std": 0.4527755454182625, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7796874940395355, |
| "step": 875 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 74.328125, |
| "epoch": 2.4060356652949246, |
| "grad_norm": 4.038706968687633, |
| "kl": 0.10400390625, |
| "learning_rate": 2.0363636363636363e-07, |
| "loss": 0.0042, |
| "reward": 1.754687488079071, |
| "reward_std": 0.02209708746522665, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.879687488079071, |
| "step": 876 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.375, |
| "epoch": 2.4087791495198903, |
| "grad_norm": 1.3484577069751817, |
| "kl": 0.071044921875, |
| "learning_rate": 2.0272727272727273e-07, |
| "loss": 0.0028, |
| "reward": 1.7023437023162842, |
| "reward_std": 0.20723089575767517, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.858593761920929, |
| "step": 877 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.71875, |
| "epoch": 2.411522633744856, |
| "grad_norm": 12.510104389183923, |
| "kl": 0.108154296875, |
| "learning_rate": 2.018181818181818e-07, |
| "loss": 0.0043, |
| "reward": 1.364843726158142, |
| "reward_std": 0.2217685431241989, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6929687261581421, |
| "step": 878 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.171875, |
| "epoch": 2.4142661179698215, |
| "grad_norm": 4.296791480235377, |
| "kl": 0.11865234375, |
| "learning_rate": 2.009090909090909e-07, |
| "loss": 0.0047, |
| "reward": 1.7554687857627869, |
| "reward_std": 0.28783681988716125, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8648437559604645, |
| "step": 879 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.765625, |
| "epoch": 2.4170096021947876, |
| "grad_norm": 2.247367124041163, |
| "kl": 0.09228515625, |
| "learning_rate": 2e-07, |
| "loss": 0.0037, |
| "reward": 1.300000011920929, |
| "reward_std": 0.2808035835623741, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.659375011920929, |
| "step": 880 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.796875, |
| "epoch": 2.419753086419753, |
| "grad_norm": 5.109391138156952, |
| "kl": 0.1064453125, |
| "learning_rate": 1.990909090909091e-07, |
| "loss": 0.0043, |
| "reward": 1.4718749523162842, |
| "reward_std": 0.5331380069255829, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7374999821186066, |
| "step": 881 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.234375, |
| "epoch": 2.422496570644719, |
| "grad_norm": 4.962619557575083, |
| "kl": 0.131103515625, |
| "learning_rate": 1.9818181818181819e-07, |
| "loss": 0.0052, |
| "reward": 0.8562500178813934, |
| "reward_std": 0.5578047335147858, |
| "rewards/choice_reward": 0.40625, |
| "rewards/roam_reward": 0.44999998807907104, |
| "step": 882 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.25, |
| "epoch": 2.4252400548696844, |
| "grad_norm": 7.189866516262367, |
| "kl": 0.114013671875, |
| "learning_rate": 1.9727272727272726e-07, |
| "loss": 0.0046, |
| "reward": 1.546093761920929, |
| "reward_std": 0.35202687978744507, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7804687321186066, |
| "step": 883 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.5625, |
| "epoch": 2.42798353909465, |
| "grad_norm": 7.578308289122683, |
| "kl": 0.102783203125, |
| "learning_rate": 1.9636363636363635e-07, |
| "loss": 0.0041, |
| "reward": 1.589062511920929, |
| "reward_std": 0.3643190935254097, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7921874821186066, |
| "step": 884 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 68.796875, |
| "epoch": 2.430727023319616, |
| "grad_norm": 2.800001423540346, |
| "kl": 0.109619140625, |
| "learning_rate": 1.9545454545454545e-07, |
| "loss": 0.0044, |
| "reward": 1.7890625, |
| "reward_std": 0.20274889469146729, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8984375, |
| "step": 885 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.875, |
| "epoch": 2.4334705075445817, |
| "grad_norm": 3.285631712678603, |
| "kl": 0.095458984375, |
| "learning_rate": 1.9454545454545455e-07, |
| "loss": 0.0038, |
| "reward": 1.2093749642372131, |
| "reward_std": 0.3309120088815689, |
| "rewards/choice_reward": 0.59375, |
| "rewards/roam_reward": 0.6156249493360519, |
| "step": 886 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.15625, |
| "epoch": 2.4362139917695473, |
| "grad_norm": 2.3317520627287553, |
| "kl": 0.105224609375, |
| "learning_rate": 1.9363636363636362e-07, |
| "loss": 0.0042, |
| "reward": 1.7132812142372131, |
| "reward_std": 0.3037515729665756, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8539062142372131, |
| "step": 887 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.34375, |
| "epoch": 2.438957475994513, |
| "grad_norm": 1.8537767822147528, |
| "kl": 0.097900390625, |
| "learning_rate": 1.9272727272727272e-07, |
| "loss": 0.0039, |
| "reward": 1.666406273841858, |
| "reward_std": 0.20769253838807344, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8382812738418579, |
| "step": 888 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.421875, |
| "epoch": 2.4417009602194786, |
| "grad_norm": 4.388051923324324, |
| "kl": 0.10107421875, |
| "learning_rate": 1.9181818181818179e-07, |
| "loss": 0.004, |
| "reward": 1.814062476158142, |
| "reward_std": 0.399493470788002, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.8921874761581421, |
| "step": 889 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.796875, |
| "epoch": 2.4444444444444446, |
| "grad_norm": 6.489982708043214, |
| "kl": 0.13427734375, |
| "learning_rate": 1.909090909090909e-07, |
| "loss": 0.0054, |
| "reward": 1.6664062142372131, |
| "reward_std": 0.21181846410036087, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8382812142372131, |
| "step": 890 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.03125, |
| "epoch": 2.4471879286694103, |
| "grad_norm": 4.975200941536546, |
| "kl": 0.095703125, |
| "learning_rate": 1.8999999999999998e-07, |
| "loss": 0.0038, |
| "reward": 1.666406273841858, |
| "reward_std": 0.35487619042396545, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8382812440395355, |
| "step": 891 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.59375, |
| "epoch": 2.449931412894376, |
| "grad_norm": 1.9188236461190016, |
| "kl": 0.110107421875, |
| "learning_rate": 1.8909090909090908e-07, |
| "loss": 0.0044, |
| "reward": 1.607031226158142, |
| "reward_std": 0.13777265278622508, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8101562559604645, |
| "step": 892 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.1875, |
| "epoch": 2.4526748971193415, |
| "grad_norm": 3.5207228078772452, |
| "kl": 0.16552734375, |
| "learning_rate": 1.8818181818181818e-07, |
| "loss": 0.0066, |
| "reward": 1.6656250357627869, |
| "reward_std": 0.42394691705703735, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8374999761581421, |
| "step": 893 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.015625, |
| "epoch": 2.455418381344307, |
| "grad_norm": 2.2571189778777634, |
| "kl": 0.106201171875, |
| "learning_rate": 1.8727272727272727e-07, |
| "loss": 0.0042, |
| "reward": 1.5390625, |
| "reward_std": 0.27705324813723564, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7734374701976776, |
| "step": 894 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.625, |
| "epoch": 2.4581618655692727, |
| "grad_norm": 5.707502573988361, |
| "kl": 0.1328125, |
| "learning_rate": 1.8636363636363637e-07, |
| "loss": 0.0053, |
| "reward": 1.756250023841858, |
| "reward_std": 0.3345056623220444, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.8812500238418579, |
| "step": 895 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.640625, |
| "epoch": 2.460905349794239, |
| "grad_norm": 5.383223650884088, |
| "kl": 0.16357421875, |
| "learning_rate": 1.8545454545454544e-07, |
| "loss": 0.0065, |
| "reward": 1.634374976158142, |
| "reward_std": 0.3058193624019623, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8218749761581421, |
| "step": 896 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.859375, |
| "epoch": 2.4636488340192044, |
| "grad_norm": 2.264871070360021, |
| "kl": 0.087646484375, |
| "learning_rate": 1.8454545454545454e-07, |
| "loss": 0.0035, |
| "reward": 1.907812476158142, |
| "reward_std": 0.192325159907341, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9546875059604645, |
| "step": 897 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.8125, |
| "epoch": 2.46639231824417, |
| "grad_norm": 6.510398650036107, |
| "kl": 0.1025390625, |
| "learning_rate": 1.836363636363636e-07, |
| "loss": 0.0041, |
| "reward": 1.6257811784744263, |
| "reward_std": 0.3411605954170227, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.813281238079071, |
| "step": 898 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.15625, |
| "epoch": 2.4691358024691357, |
| "grad_norm": 5.050535777404141, |
| "kl": 0.132080078125, |
| "learning_rate": 1.8272727272727273e-07, |
| "loss": 0.0053, |
| "reward": 1.7132812142372131, |
| "reward_std": 0.4274836480617523, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8539062738418579, |
| "step": 899 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.390625, |
| "epoch": 2.4718792866941017, |
| "grad_norm": 2.4150872091575435, |
| "kl": 0.1044921875, |
| "learning_rate": 1.818181818181818e-07, |
| "loss": 0.0042, |
| "reward": 1.7140624523162842, |
| "reward_std": 0.1370019344612956, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8546874821186066, |
| "step": 900 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.984375, |
| "epoch": 2.4746227709190673, |
| "grad_norm": 2.9416235492274403, |
| "kl": 0.087890625, |
| "learning_rate": 1.809090909090909e-07, |
| "loss": 0.0035, |
| "reward": 1.66796875, |
| "reward_std": 0.4173927679657936, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.83984375, |
| "step": 901 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.390625, |
| "epoch": 2.477366255144033, |
| "grad_norm": 3.0301688079902607, |
| "kl": 0.106689453125, |
| "learning_rate": 1.8e-07, |
| "loss": 0.0043, |
| "reward": 1.517187476158142, |
| "reward_std": 0.6114129722118378, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7515624761581421, |
| "step": 902 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.9375, |
| "epoch": 2.4801097393689986, |
| "grad_norm": 2.3823712947325295, |
| "kl": 0.086669921875, |
| "learning_rate": 1.7909090909090907e-07, |
| "loss": 0.0035, |
| "reward": 1.821874976158142, |
| "reward_std": 0.20688685774803162, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9156250059604645, |
| "step": 903 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.25, |
| "epoch": 2.482853223593964, |
| "grad_norm": 5.57538249725218, |
| "kl": 0.137451171875, |
| "learning_rate": 1.781818181818182e-07, |
| "loss": 0.0055, |
| "reward": 1.578125, |
| "reward_std": 0.40212661027908325, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.796875, |
| "step": 904 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.890625, |
| "epoch": 2.48559670781893, |
| "grad_norm": 2.6641129082049244, |
| "kl": 0.115234375, |
| "learning_rate": 1.7727272727272726e-07, |
| "loss": 0.0046, |
| "reward": 1.862500011920929, |
| "reward_std": 0.2599198669195175, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.925000011920929, |
| "step": 905 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.875, |
| "epoch": 2.488340192043896, |
| "grad_norm": 3.0206434726292994, |
| "kl": 0.1044921875, |
| "learning_rate": 1.7636363636363636e-07, |
| "loss": 0.0042, |
| "reward": 1.579687476158142, |
| "reward_std": 0.3286094591021538, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7984374463558197, |
| "step": 906 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.875, |
| "epoch": 2.4910836762688615, |
| "grad_norm": 4.455283714421392, |
| "kl": 0.1044921875, |
| "learning_rate": 1.7545454545454543e-07, |
| "loss": 0.0042, |
| "reward": 1.817187488079071, |
| "reward_std": 0.3769528269767761, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.895312488079071, |
| "step": 907 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.9375, |
| "epoch": 2.493827160493827, |
| "grad_norm": 2.6112679569674717, |
| "kl": 0.093017578125, |
| "learning_rate": 1.7454545454545455e-07, |
| "loss": 0.0037, |
| "reward": 1.1070312559604645, |
| "reward_std": 0.11657025711610913, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5757812559604645, |
| "step": 908 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.265625, |
| "epoch": 2.4965706447187928, |
| "grad_norm": 8.190560736272449, |
| "kl": 0.09326171875, |
| "learning_rate": 1.7363636363636362e-07, |
| "loss": 0.0037, |
| "reward": 1.493749976158142, |
| "reward_std": 0.4932164400815964, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7593750059604645, |
| "step": 909 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.8125, |
| "epoch": 2.499314128943759, |
| "grad_norm": 8.718805678669755, |
| "kl": 0.107177734375, |
| "learning_rate": 1.7272727272727272e-07, |
| "loss": 0.0043, |
| "reward": 1.600781261920929, |
| "reward_std": 0.23684065975248814, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.8039062321186066, |
| "step": 910 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.546875, |
| "epoch": 2.5020576131687244, |
| "grad_norm": 4.198056415866492, |
| "kl": 0.101318359375, |
| "learning_rate": 1.7181818181818182e-07, |
| "loss": 0.004, |
| "reward": 1.310937523841858, |
| "reward_std": 0.3555447608232498, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6703124642372131, |
| "step": 911 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.40625, |
| "epoch": 2.50480109739369, |
| "grad_norm": 4.373002370025674, |
| "kl": 0.133056640625, |
| "learning_rate": 1.709090909090909e-07, |
| "loss": 0.0053, |
| "reward": 1.3937500417232513, |
| "reward_std": 0.5110341310501099, |
| "rewards/choice_reward": 0.6875, |
| "rewards/roam_reward": 0.7062499821186066, |
| "step": 912 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.421875, |
| "epoch": 2.5075445816186557, |
| "grad_norm": 28.22264100129003, |
| "kl": 0.197265625, |
| "learning_rate": 1.7000000000000001e-07, |
| "loss": 0.0079, |
| "reward": 1.8171875476837158, |
| "reward_std": 0.391438253223896, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9109375476837158, |
| "step": 913 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.328125, |
| "epoch": 2.5102880658436213, |
| "grad_norm": 4.10992951027992, |
| "kl": 0.116943359375, |
| "learning_rate": 1.6909090909090908e-07, |
| "loss": 0.0047, |
| "reward": 1.6726561784744263, |
| "reward_std": 0.45760543644428253, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.844531238079071, |
| "step": 914 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.421875, |
| "epoch": 2.513031550068587, |
| "grad_norm": 5.033545976301883, |
| "kl": 0.08740234375, |
| "learning_rate": 1.6818181818181818e-07, |
| "loss": 0.0035, |
| "reward": 1.4656250476837158, |
| "reward_std": 0.5237148702144623, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7468750178813934, |
| "step": 915 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.15625, |
| "epoch": 2.5157750342935525, |
| "grad_norm": 4.4992682723749775, |
| "kl": 0.09912109375, |
| "learning_rate": 1.6727272727272725e-07, |
| "loss": 0.004, |
| "reward": 1.590624988079071, |
| "reward_std": 0.2772092744708061, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.793749988079071, |
| "step": 916 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.953125, |
| "epoch": 2.5185185185185186, |
| "grad_norm": 7.067973308531796, |
| "kl": 0.137939453125, |
| "learning_rate": 1.6636363636363635e-07, |
| "loss": 0.0055, |
| "reward": 1.67578125, |
| "reward_std": 0.46074244379997253, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.83203125, |
| "step": 917 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.78125, |
| "epoch": 2.521262002743484, |
| "grad_norm": 2.956470810154343, |
| "kl": 0.101806640625, |
| "learning_rate": 1.6545454545454545e-07, |
| "loss": 0.0041, |
| "reward": 1.2531249523162842, |
| "reward_std": 0.5866954028606415, |
| "rewards/choice_reward": 0.609375, |
| "rewards/roam_reward": 0.6437499523162842, |
| "step": 918 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.53125, |
| "epoch": 2.52400548696845, |
| "grad_norm": 6.591829552719709, |
| "kl": 0.10205078125, |
| "learning_rate": 1.6454545454545454e-07, |
| "loss": 0.0041, |
| "reward": 1.357812523841858, |
| "reward_std": 0.47224710136651993, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6859374940395355, |
| "step": 919 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.4375, |
| "epoch": 2.526748971193416, |
| "grad_norm": 7.515520905203439, |
| "kl": 0.1015625, |
| "learning_rate": 1.6363636363636364e-07, |
| "loss": 0.0041, |
| "reward": 1.455468773841858, |
| "reward_std": 0.37790268659591675, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7367187738418579, |
| "step": 920 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.90625, |
| "epoch": 2.5294924554183815, |
| "grad_norm": 1.8682810884686853, |
| "kl": 0.1240234375, |
| "learning_rate": 1.627272727272727e-07, |
| "loss": 0.0049, |
| "reward": 1.6382812857627869, |
| "reward_std": 0.28293294459581375, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8257812857627869, |
| "step": 921 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.25, |
| "epoch": 2.532235939643347, |
| "grad_norm": 3.8243588025411652, |
| "kl": 0.0947265625, |
| "learning_rate": 1.618181818181818e-07, |
| "loss": 0.0038, |
| "reward": 1.5343749523162842, |
| "reward_std": 0.5920023918151855, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7374999821186066, |
| "step": 922 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.90625, |
| "epoch": 2.5349794238683128, |
| "grad_norm": 3.694037239049243, |
| "kl": 0.093994140625, |
| "learning_rate": 1.609090909090909e-07, |
| "loss": 0.0038, |
| "reward": 1.2859375178813934, |
| "reward_std": 0.3922395408153534, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6609375178813934, |
| "step": 923 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.265625, |
| "epoch": 2.5377229080932784, |
| "grad_norm": 1.595043885674026, |
| "kl": 0.09033203125, |
| "learning_rate": 1.6e-07, |
| "loss": 0.0036, |
| "reward": 1.7015624642372131, |
| "reward_std": 0.17235727235674858, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8421874940395355, |
| "step": 924 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.34375, |
| "epoch": 2.540466392318244, |
| "grad_norm": 2.959211601757336, |
| "kl": 0.095703125, |
| "learning_rate": 1.5909090909090907e-07, |
| "loss": 0.0038, |
| "reward": 1.5499999523162842, |
| "reward_std": 0.33859558403491974, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7843749821186066, |
| "step": 925 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.203125, |
| "epoch": 2.5432098765432096, |
| "grad_norm": 5.012725913458301, |
| "kl": 0.154296875, |
| "learning_rate": 1.5818181818181817e-07, |
| "loss": 0.0062, |
| "reward": 1.5078125, |
| "reward_std": 0.4259311109781265, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7578125, |
| "step": 926 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.453125, |
| "epoch": 2.5459533607681757, |
| "grad_norm": 3.0997758801642994, |
| "kl": 0.09130859375, |
| "learning_rate": 1.5727272727272727e-07, |
| "loss": 0.0037, |
| "reward": 1.543749988079071, |
| "reward_std": 0.5073673129081726, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.746874988079071, |
| "step": 927 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.078125, |
| "epoch": 2.5486968449931413, |
| "grad_norm": 5.2508460190751975, |
| "kl": 0.1943359375, |
| "learning_rate": 1.5636363636363637e-07, |
| "loss": 0.0078, |
| "reward": 1.306249976158142, |
| "reward_std": 0.694450169801712, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6656250059604645, |
| "step": 928 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.703125, |
| "epoch": 2.551440329218107, |
| "grad_norm": 3.3798454745729902, |
| "kl": 0.09765625, |
| "learning_rate": 1.5545454545454544e-07, |
| "loss": 0.0039, |
| "reward": 1.760937511920929, |
| "reward_std": 0.38189470767974854, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.870312511920929, |
| "step": 929 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.5, |
| "epoch": 2.5541838134430725, |
| "grad_norm": 10.457283078493456, |
| "kl": 0.100830078125, |
| "learning_rate": 1.5454545454545453e-07, |
| "loss": 0.004, |
| "reward": 1.623437523841858, |
| "reward_std": 0.4348487854003906, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8109375238418579, |
| "step": 930 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.296875, |
| "epoch": 2.5569272976680386, |
| "grad_norm": 3.075962228788098, |
| "kl": 0.09765625, |
| "learning_rate": 1.5363636363636363e-07, |
| "loss": 0.0039, |
| "reward": 1.790624976158142, |
| "reward_std": 0.2952752038836479, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.9000000059604645, |
| "step": 931 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 70.703125, |
| "epoch": 2.5596707818930042, |
| "grad_norm": 6.633142433143458, |
| "kl": 0.147705078125, |
| "learning_rate": 1.5272727272727273e-07, |
| "loss": 0.0059, |
| "reward": 1.8921875357627869, |
| "reward_std": 0.2794111743569374, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9390625357627869, |
| "step": 932 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.296875, |
| "epoch": 2.56241426611797, |
| "grad_norm": 2.1414242451237815, |
| "kl": 0.0693359375, |
| "learning_rate": 1.5181818181818183e-07, |
| "loss": 0.0028, |
| "reward": 1.7125000357627869, |
| "reward_std": 0.1414213553071022, |
| "rewards/choice_reward": 0.859375, |
| "rewards/roam_reward": 0.8531249761581421, |
| "step": 933 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.90625, |
| "epoch": 2.5651577503429355, |
| "grad_norm": 41.78678606748375, |
| "kl": 0.092529296875, |
| "learning_rate": 1.509090909090909e-07, |
| "loss": 0.0037, |
| "reward": 1.5468750596046448, |
| "reward_std": 0.377840094268322, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.78125, |
| "step": 934 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.0625, |
| "epoch": 2.567901234567901, |
| "grad_norm": 5.584232435399464, |
| "kl": 0.1123046875, |
| "learning_rate": 1.5e-07, |
| "loss": 0.0045, |
| "reward": 1.489843726158142, |
| "reward_std": 0.44570471346378326, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7398437559604645, |
| "step": 935 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.96875, |
| "epoch": 2.5706447187928667, |
| "grad_norm": 2.791576141853459, |
| "kl": 0.102294921875, |
| "learning_rate": 1.4909090909090906e-07, |
| "loss": 0.0041, |
| "reward": 1.696874976158142, |
| "reward_std": 0.43145138025283813, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8531250059604645, |
| "step": 936 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.578125, |
| "epoch": 2.5733882030178328, |
| "grad_norm": 3.053624480174356, |
| "kl": 0.0888671875, |
| "learning_rate": 1.481818181818182e-07, |
| "loss": 0.0035, |
| "reward": 1.5453124642372131, |
| "reward_std": 0.26508478820323944, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7796875238418579, |
| "step": 937 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.171875, |
| "epoch": 2.5761316872427984, |
| "grad_norm": 2.973330455331203, |
| "kl": 0.09765625, |
| "learning_rate": 1.4727272727272726e-07, |
| "loss": 0.0039, |
| "reward": 1.344531238079071, |
| "reward_std": 0.32123443484306335, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.672656238079071, |
| "step": 938 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.71875, |
| "epoch": 2.578875171467764, |
| "grad_norm": 5.163306118515068, |
| "kl": 0.101318359375, |
| "learning_rate": 1.4636363636363636e-07, |
| "loss": 0.0041, |
| "reward": 1.6968750357627869, |
| "reward_std": 0.3943808674812317, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.8531250357627869, |
| "step": 939 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.515625, |
| "epoch": 2.5816186556927296, |
| "grad_norm": 3.5058106697132816, |
| "kl": 0.106201171875, |
| "learning_rate": 1.4545454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.544531226158142, |
| "reward_std": 0.5131105482578278, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7789062261581421, |
| "step": 940 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 76.921875, |
| "epoch": 2.5843621399176957, |
| "grad_norm": 13.907951590750768, |
| "kl": 0.146484375, |
| "learning_rate": 1.4454545454545452e-07, |
| "loss": 0.0058, |
| "reward": 1.4218749403953552, |
| "reward_std": 0.5525650680065155, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.703125, |
| "step": 941 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.265625, |
| "epoch": 2.5871056241426613, |
| "grad_norm": 14.992872788243073, |
| "kl": 0.098876953125, |
| "learning_rate": 1.4363636363636365e-07, |
| "loss": 0.004, |
| "reward": 1.815625011920929, |
| "reward_std": 0.2820400223135948, |
| "rewards/choice_reward": 0.921875, |
| "rewards/roam_reward": 0.8937499821186066, |
| "step": 942 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.3125, |
| "epoch": 2.589849108367627, |
| "grad_norm": 4.11391804323829, |
| "kl": 0.113037109375, |
| "learning_rate": 1.4272727272727272e-07, |
| "loss": 0.0045, |
| "reward": 1.372656226158142, |
| "reward_std": 0.4981390982866287, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.7007812261581421, |
| "step": 943 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.046875, |
| "epoch": 2.5925925925925926, |
| "grad_norm": 4.316485995076902, |
| "kl": 0.105224609375, |
| "learning_rate": 1.4181818181818182e-07, |
| "loss": 0.0042, |
| "reward": 1.342187523841858, |
| "reward_std": 0.46965810656547546, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6859374642372131, |
| "step": 944 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.375, |
| "epoch": 2.595336076817558, |
| "grad_norm": 1.911352539641548, |
| "kl": 0.089111328125, |
| "learning_rate": 1.4090909090909089e-07, |
| "loss": 0.0036, |
| "reward": 1.547656238079071, |
| "reward_std": 0.0964670218527317, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.782031238079071, |
| "step": 945 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.1875, |
| "epoch": 2.598079561042524, |
| "grad_norm": 2.4993048044470036, |
| "kl": 0.10205078125, |
| "learning_rate": 1.4e-07, |
| "loss": 0.0041, |
| "reward": 1.486718773841858, |
| "reward_std": 0.33659204840660095, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7523437142372131, |
| "step": 946 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.578125, |
| "epoch": 2.60082304526749, |
| "grad_norm": 2.761243514278863, |
| "kl": 0.096435546875, |
| "learning_rate": 1.3909090909090908e-07, |
| "loss": 0.0039, |
| "reward": 1.755468726158142, |
| "reward_std": 0.4127981811761856, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8648437559604645, |
| "step": 947 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.90625, |
| "epoch": 2.6035665294924555, |
| "grad_norm": 3.2114970692303935, |
| "kl": 0.111328125, |
| "learning_rate": 1.3818181818181818e-07, |
| "loss": 0.0044, |
| "reward": 1.672656238079071, |
| "reward_std": 0.6133669316768646, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.828906238079071, |
| "step": 948 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.796875, |
| "epoch": 2.606310013717421, |
| "grad_norm": 3.318991870606774, |
| "kl": 0.0966796875, |
| "learning_rate": 1.3727272727272727e-07, |
| "loss": 0.0039, |
| "reward": 1.489062488079071, |
| "reward_std": 0.38795000687241554, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.739062488079071, |
| "step": 949 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.1875, |
| "epoch": 2.6090534979423867, |
| "grad_norm": 4.713917493991348, |
| "kl": 0.101806640625, |
| "learning_rate": 1.3636363636363635e-07, |
| "loss": 0.0041, |
| "reward": 1.647656261920929, |
| "reward_std": 0.4856095612049103, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8195312321186066, |
| "step": 950 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.796875, |
| "epoch": 2.611796982167353, |
| "grad_norm": 4.250557230931642, |
| "kl": 0.098876953125, |
| "learning_rate": 1.3545454545454547e-07, |
| "loss": 0.004, |
| "reward": 1.0906249582767487, |
| "reward_std": 0.36680732667446136, |
| "rewards/choice_reward": 0.53125, |
| "rewards/roam_reward": 0.5593749582767487, |
| "step": 951 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.796875, |
| "epoch": 2.6145404663923184, |
| "grad_norm": 5.571591310306311, |
| "kl": 0.09423828125, |
| "learning_rate": 1.3454545454545454e-07, |
| "loss": 0.0038, |
| "reward": 1.456250011920929, |
| "reward_std": 0.42433546483516693, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.7374999821186066, |
| "step": 952 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.28125, |
| "epoch": 2.617283950617284, |
| "grad_norm": 8.165042661619838, |
| "kl": 0.121337890625, |
| "learning_rate": 1.3363636363636364e-07, |
| "loss": 0.0048, |
| "reward": 1.5718750357627869, |
| "reward_std": 0.5588430762290955, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7906250357627869, |
| "step": 953 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.765625, |
| "epoch": 2.6200274348422496, |
| "grad_norm": 3.7181708604261883, |
| "kl": 0.093994140625, |
| "learning_rate": 1.327272727272727e-07, |
| "loss": 0.0038, |
| "reward": 1.635937511920929, |
| "reward_std": 0.3050750121474266, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8234374821186066, |
| "step": 954 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.71875, |
| "epoch": 2.6227709190672153, |
| "grad_norm": 4.792225130509411, |
| "kl": 0.10546875, |
| "learning_rate": 1.318181818181818e-07, |
| "loss": 0.0042, |
| "reward": 1.5187499523162842, |
| "reward_std": 0.23608315736055374, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7687499523162842, |
| "step": 955 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.859375, |
| "epoch": 2.625514403292181, |
| "grad_norm": 2.071587920844544, |
| "kl": 0.09326171875, |
| "learning_rate": 1.309090909090909e-07, |
| "loss": 0.0037, |
| "reward": 1.565625011920929, |
| "reward_std": 0.2502797245979309, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7843749821186066, |
| "step": 956 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.0625, |
| "epoch": 2.6282578875171465, |
| "grad_norm": 3.2967853377032807, |
| "kl": 0.10302734375, |
| "learning_rate": 1.3e-07, |
| "loss": 0.0041, |
| "reward": 1.3421874642372131, |
| "reward_std": 0.3264111652970314, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6859374940395355, |
| "step": 957 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.09375, |
| "epoch": 2.6310013717421126, |
| "grad_norm": 4.646329496583205, |
| "kl": 0.094482421875, |
| "learning_rate": 1.290909090909091e-07, |
| "loss": 0.0038, |
| "reward": 1.40625, |
| "reward_std": 0.5289385914802551, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7031249701976776, |
| "step": 958 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.8125, |
| "epoch": 2.633744855967078, |
| "grad_norm": 13.25104189197075, |
| "kl": 0.087158203125, |
| "learning_rate": 1.2818181818181817e-07, |
| "loss": 0.0035, |
| "reward": 1.466406226158142, |
| "reward_std": 0.472397118806839, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7320312261581421, |
| "step": 959 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.40625, |
| "epoch": 2.636488340192044, |
| "grad_norm": 2.5446216685180474, |
| "kl": 0.09423828125, |
| "learning_rate": 1.2727272727272726e-07, |
| "loss": 0.0038, |
| "reward": 1.584375023841858, |
| "reward_std": 0.4048280864953995, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.8031249642372131, |
| "step": 960 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.828125, |
| "epoch": 2.63923182441701, |
| "grad_norm": 3.555437790786787, |
| "kl": 0.109130859375, |
| "learning_rate": 1.2636363636363636e-07, |
| "loss": 0.0044, |
| "reward": 1.470312476158142, |
| "reward_std": 0.35928191244602203, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7359375059604645, |
| "step": 961 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.671875, |
| "epoch": 2.6419753086419755, |
| "grad_norm": 3.6399867879843604, |
| "kl": 0.09619140625, |
| "learning_rate": 1.2545454545454546e-07, |
| "loss": 0.0039, |
| "reward": 1.364062488079071, |
| "reward_std": 0.47104281187057495, |
| "rewards/choice_reward": 0.671875, |
| "rewards/roam_reward": 0.6921875178813934, |
| "step": 962 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.390625, |
| "epoch": 2.644718792866941, |
| "grad_norm": 3.586542860919556, |
| "kl": 0.115478515625, |
| "learning_rate": 1.2454545454545453e-07, |
| "loss": 0.0046, |
| "reward": 1.632031261920929, |
| "reward_std": 0.32563401013612747, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8195312321186066, |
| "step": 963 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.21875, |
| "epoch": 2.6474622770919067, |
| "grad_norm": 9.179120368142398, |
| "kl": 0.07470703125, |
| "learning_rate": 1.2363636363636363e-07, |
| "loss": 0.003, |
| "reward": 0.890625, |
| "reward_std": 0.4841189831495285, |
| "rewards/choice_reward": 0.421875, |
| "rewards/roam_reward": 0.4687499701976776, |
| "step": 964 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.890625, |
| "epoch": 2.6502057613168724, |
| "grad_norm": 6.378830292139383, |
| "kl": 0.12109375, |
| "learning_rate": 1.2272727272727272e-07, |
| "loss": 0.0048, |
| "reward": 1.5859375, |
| "reward_std": 0.4652404338121414, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7890624403953552, |
| "step": 965 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.375, |
| "epoch": 2.652949245541838, |
| "grad_norm": 2.953950711020351, |
| "kl": 0.0966796875, |
| "learning_rate": 1.218181818181818e-07, |
| "loss": 0.0039, |
| "reward": 1.4562499523162842, |
| "reward_std": 0.23340336233377457, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.737500011920929, |
| "step": 966 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 73.28125, |
| "epoch": 2.6556927297668036, |
| "grad_norm": 4.6014001745806015, |
| "kl": 0.134765625, |
| "learning_rate": 1.209090909090909e-07, |
| "loss": 0.0054, |
| "reward": 1.1312499642372131, |
| "reward_std": 0.3415980935096741, |
| "rewards/choice_reward": 0.546875, |
| "rewards/roam_reward": 0.5843749940395355, |
| "step": 967 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.296875, |
| "epoch": 2.6584362139917697, |
| "grad_norm": 3.5615170855204568, |
| "kl": 0.108642578125, |
| "learning_rate": 1.2e-07, |
| "loss": 0.0043, |
| "reward": 1.737500011920929, |
| "reward_std": 0.4942738860845566, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.862500011920929, |
| "step": 968 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.140625, |
| "epoch": 2.6611796982167353, |
| "grad_norm": 3.1516398632960962, |
| "kl": 0.094482421875, |
| "learning_rate": 1.1909090909090909e-07, |
| "loss": 0.0038, |
| "reward": 1.7671875357627869, |
| "reward_std": 0.258515827357769, |
| "rewards/choice_reward": 0.890625, |
| "rewards/roam_reward": 0.8765625059604645, |
| "step": 969 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.4375, |
| "epoch": 2.663923182441701, |
| "grad_norm": 4.6750764993061455, |
| "kl": 0.094482421875, |
| "learning_rate": 1.1818181818181817e-07, |
| "loss": 0.0038, |
| "reward": 1.1898437142372131, |
| "reward_std": 0.45025503635406494, |
| "rewards/choice_reward": 0.578125, |
| "rewards/roam_reward": 0.6117187142372131, |
| "step": 970 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 83.53125, |
| "epoch": 2.6666666666666665, |
| "grad_norm": 2.1423374157776482, |
| "kl": 0.1005859375, |
| "learning_rate": 1.1727272727272727e-07, |
| "loss": 0.004, |
| "reward": 1.879687488079071, |
| "reward_std": 0.21130627393722534, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.9421875178813934, |
| "step": 971 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.859375, |
| "epoch": 2.6694101508916326, |
| "grad_norm": 3.7705135364888744, |
| "kl": 0.11669921875, |
| "learning_rate": 1.1636363636363636e-07, |
| "loss": 0.0047, |
| "reward": 1.311718761920929, |
| "reward_std": 0.17687445972114801, |
| "rewards/choice_reward": 0.65625, |
| "rewards/roam_reward": 0.6554687321186066, |
| "step": 972 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.21875, |
| "epoch": 2.672153635116598, |
| "grad_norm": 2.6694912934157533, |
| "kl": 0.09375, |
| "learning_rate": 1.1545454545454545e-07, |
| "loss": 0.0037, |
| "reward": 1.9046874642372131, |
| "reward_std": 0.23905985802412033, |
| "rewards/choice_reward": 0.96875, |
| "rewards/roam_reward": 0.9359374940395355, |
| "step": 973 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.1875, |
| "epoch": 2.674897119341564, |
| "grad_norm": 4.3604941363825125, |
| "kl": 0.11083984375, |
| "learning_rate": 1.1454545454545455e-07, |
| "loss": 0.0044, |
| "reward": 1.690625011920929, |
| "reward_std": 0.5343562066555023, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.846875011920929, |
| "step": 974 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.265625, |
| "epoch": 2.6776406035665294, |
| "grad_norm": 6.6361785325827185, |
| "kl": 0.100341796875, |
| "learning_rate": 1.1363636363636363e-07, |
| "loss": 0.004, |
| "reward": 1.5109375715255737, |
| "reward_std": 0.49013426899909973, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7453124821186066, |
| "step": 975 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 86.96875, |
| "epoch": 2.680384087791495, |
| "grad_norm": 2.9479493932753082, |
| "kl": 0.09228515625, |
| "learning_rate": 1.1272727272727273e-07, |
| "loss": 0.0037, |
| "reward": 1.653124988079071, |
| "reward_std": 0.39804912731051445, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.824999988079071, |
| "step": 976 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.015625, |
| "epoch": 2.6831275720164607, |
| "grad_norm": 9.367804358161262, |
| "kl": 0.133056640625, |
| "learning_rate": 1.1181818181818181e-07, |
| "loss": 0.0053, |
| "reward": 1.471093773841858, |
| "reward_std": 0.4260319769382477, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7367187142372131, |
| "step": 977 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.453125, |
| "epoch": 2.6858710562414267, |
| "grad_norm": 7.925544902541567, |
| "kl": 0.14599609375, |
| "learning_rate": 1.1090909090909091e-07, |
| "loss": 0.0058, |
| "reward": 1.69921875, |
| "reward_std": 0.4324868619441986, |
| "rewards/choice_reward": 0.84375, |
| "rewards/roam_reward": 0.85546875, |
| "step": 978 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 79.9375, |
| "epoch": 2.6886145404663924, |
| "grad_norm": 6.647960389727826, |
| "kl": 0.103515625, |
| "learning_rate": 1.0999999999999999e-07, |
| "loss": 0.0041, |
| "reward": 1.461718738079071, |
| "reward_std": 0.4704754278063774, |
| "rewards/choice_reward": 0.71875, |
| "rewards/roam_reward": 0.742968738079071, |
| "step": 979 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.421875, |
| "epoch": 2.691358024691358, |
| "grad_norm": 5.327850308136772, |
| "kl": 0.12255859375, |
| "learning_rate": 1.0909090909090908e-07, |
| "loss": 0.0049, |
| "reward": 1.482031226158142, |
| "reward_std": 0.37536171078681946, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7320312261581421, |
| "step": 980 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.859375, |
| "epoch": 2.6941015089163236, |
| "grad_norm": 2.7605446866186165, |
| "kl": 0.083740234375, |
| "learning_rate": 1.0818181818181817e-07, |
| "loss": 0.0034, |
| "reward": 1.420312523841858, |
| "reward_std": 0.4252743124961853, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7171874940395355, |
| "step": 981 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 80.9375, |
| "epoch": 2.6968449931412897, |
| "grad_norm": 3.5504893248826876, |
| "kl": 0.0908203125, |
| "learning_rate": 1.0727272727272727e-07, |
| "loss": 0.0036, |
| "reward": 1.534375011920929, |
| "reward_std": 0.36864815652370453, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7687499821186066, |
| "step": 982 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 85.0625, |
| "epoch": 2.6995884773662553, |
| "grad_norm": 3.1240045755403387, |
| "kl": 0.104736328125, |
| "learning_rate": 1.0636363636363637e-07, |
| "loss": 0.0042, |
| "reward": 1.310937523841858, |
| "reward_std": 0.1016465974971652, |
| "rewards/choice_reward": 0.640625, |
| "rewards/roam_reward": 0.6703124642372131, |
| "step": 983 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 91.609375, |
| "epoch": 2.702331961591221, |
| "grad_norm": 3.7008525745312175, |
| "kl": 0.107421875, |
| "learning_rate": 1.0545454545454545e-07, |
| "loss": 0.0043, |
| "reward": 1.4453125, |
| "reward_std": 0.4314822629094124, |
| "rewards/choice_reward": 0.734375, |
| "rewards/roam_reward": 0.7109374701976776, |
| "step": 984 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 88.28125, |
| "epoch": 2.7050754458161865, |
| "grad_norm": 3.840966428690235, |
| "kl": 0.112060546875, |
| "learning_rate": 1.0454545454545454e-07, |
| "loss": 0.0045, |
| "reward": 1.416406273841858, |
| "reward_std": 0.24614976346492767, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7132812142372131, |
| "step": 985 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.734375, |
| "epoch": 2.707818930041152, |
| "grad_norm": 2.917343856165196, |
| "kl": 0.109375, |
| "learning_rate": 1.0363636363636363e-07, |
| "loss": 0.0044, |
| "reward": 1.73828125, |
| "reward_std": 0.3371606692671776, |
| "rewards/choice_reward": 0.875, |
| "rewards/roam_reward": 0.86328125, |
| "step": 986 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 89.71875, |
| "epoch": 2.7105624142661178, |
| "grad_norm": 3.098225905201064, |
| "kl": 0.087646484375, |
| "learning_rate": 1.0272727272727272e-07, |
| "loss": 0.0035, |
| "reward": 1.2874999642372131, |
| "reward_std": 0.3168279901146889, |
| "rewards/choice_reward": 0.625, |
| "rewards/roam_reward": 0.6624999940395355, |
| "step": 987 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.296875, |
| "epoch": 2.713305898491084, |
| "grad_norm": 2.4222336122305737, |
| "kl": 0.08349609375, |
| "learning_rate": 1.0181818181818181e-07, |
| "loss": 0.0033, |
| "reward": 1.817968726158142, |
| "reward_std": 0.24658063054084778, |
| "rewards/choice_reward": 0.90625, |
| "rewards/roam_reward": 0.9117187559604645, |
| "step": 988 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.78125, |
| "epoch": 2.7160493827160495, |
| "grad_norm": 4.645635153732883, |
| "kl": 0.111083984375, |
| "learning_rate": 1.009090909090909e-07, |
| "loss": 0.0044, |
| "reward": 1.634374976158142, |
| "reward_std": 0.371658131480217, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8218750059604645, |
| "step": 989 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.4375, |
| "epoch": 2.718792866941015, |
| "grad_norm": 3.003399224538686, |
| "kl": 0.09716796875, |
| "learning_rate": 1e-07, |
| "loss": 0.0039, |
| "reward": 1.516406238079071, |
| "reward_std": 0.42474687099456787, |
| "rewards/choice_reward": 0.75, |
| "rewards/roam_reward": 0.7664062678813934, |
| "step": 990 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.46875, |
| "epoch": 2.7215363511659807, |
| "grad_norm": 4.226013969835389, |
| "kl": 0.115966796875, |
| "learning_rate": 9.909090909090909e-08, |
| "loss": 0.0046, |
| "reward": 1.5312500596046448, |
| "reward_std": 0.5685274600982666, |
| "rewards/choice_reward": 0.765625, |
| "rewards/roam_reward": 0.7656250298023224, |
| "step": 991 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 87.390625, |
| "epoch": 2.7242798353909468, |
| "grad_norm": 12.350588784104131, |
| "kl": 0.099365234375, |
| "learning_rate": 9.818181818181818e-08, |
| "loss": 0.004, |
| "reward": 1.4312500357627869, |
| "reward_std": 0.5584160685539246, |
| "rewards/choice_reward": 0.703125, |
| "rewards/roam_reward": 0.7281250059604645, |
| "step": 992 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 82.515625, |
| "epoch": 2.7270233196159124, |
| "grad_norm": 2.46709204077942, |
| "kl": 0.08056640625, |
| "learning_rate": 9.727272727272727e-08, |
| "loss": 0.0032, |
| "reward": 1.591406226158142, |
| "reward_std": 0.4659492149949074, |
| "rewards/choice_reward": 0.796875, |
| "rewards/roam_reward": 0.7945312559604645, |
| "step": 993 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.046875, |
| "epoch": 2.729766803840878, |
| "grad_norm": 2.6581896216315553, |
| "kl": 0.0927734375, |
| "learning_rate": 9.636363636363636e-08, |
| "loss": 0.0037, |
| "reward": 1.028906226158142, |
| "reward_std": 0.48388032615184784, |
| "rewards/choice_reward": 0.5, |
| "rewards/roam_reward": 0.5289062261581421, |
| "step": 994 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 84.8125, |
| "epoch": 2.7325102880658436, |
| "grad_norm": 2.231189670238404, |
| "kl": 0.098876953125, |
| "learning_rate": 9.545454545454546e-08, |
| "loss": 0.0039, |
| "reward": 1.6640625, |
| "reward_std": 0.2100028172135353, |
| "rewards/choice_reward": 0.828125, |
| "rewards/roam_reward": 0.8359375, |
| "step": 995 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.34375, |
| "epoch": 2.7352537722908092, |
| "grad_norm": 7.111804822235356, |
| "kl": 0.1337890625, |
| "learning_rate": 9.454545454545454e-08, |
| "loss": 0.0054, |
| "reward": 1.5531249046325684, |
| "reward_std": 0.49822041392326355, |
| "rewards/choice_reward": 0.78125, |
| "rewards/roam_reward": 0.7718750238418579, |
| "step": 996 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.78125, |
| "epoch": 2.737997256515775, |
| "grad_norm": 2.1757372856376485, |
| "kl": 0.091552734375, |
| "learning_rate": 9.363636363636364e-08, |
| "loss": 0.0037, |
| "reward": 1.833593726158142, |
| "reward_std": 0.22985859215259552, |
| "rewards/choice_reward": 0.9375, |
| "rewards/roam_reward": 0.8960937559604645, |
| "step": 997 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 77.234375, |
| "epoch": 2.7407407407407405, |
| "grad_norm": 6.25212799141905, |
| "kl": 0.137451171875, |
| "learning_rate": 9.272727272727272e-08, |
| "loss": 0.0055, |
| "reward": 1.6390624642372131, |
| "reward_std": 0.2201451137661934, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8265624642372131, |
| "step": 998 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 78.03125, |
| "epoch": 2.7434842249657065, |
| "grad_norm": 3.0969955054568614, |
| "kl": 0.09814453125, |
| "learning_rate": 9.18181818181818e-08, |
| "loss": 0.0039, |
| "reward": 1.6343750357627869, |
| "reward_std": 0.28497179597616196, |
| "rewards/choice_reward": 0.8125, |
| "rewards/roam_reward": 0.8218750357627869, |
| "step": 999 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 81.53125, |
| "epoch": 2.746227709190672, |
| "grad_norm": 2.700965617559501, |
| "kl": 0.122314453125, |
| "learning_rate": 9.09090909090909e-08, |
| "loss": 0.0049, |
| "reward": 1.903124988079071, |
| "reward_std": 0.26436372846364975, |
| "rewards/choice_reward": 0.953125, |
| "rewards/roam_reward": 0.9500000178813934, |
| "step": 1000 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 1100, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 4, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
