| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9978568366909558, |
| "eval_steps": 500, |
| "global_step": 291, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.30859375, |
| "epoch": 0.0034290612944706386, |
| "grad_norm": 4.169865651189505, |
| "kl": 0.0, |
| "learning_rate": 9.965635738831615e-07, |
| "loss": 0.0, |
| "reward": 0.984375, |
| "reward_std": 0.4253819230943918, |
| "rewards/accuracy_reward_action": 0.15625, |
| "rewards/accuracy_reward_coord": 0.01953125, |
| "rewards/format_reward": 0.80859375, |
| "step": 1 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 90.2734375, |
| "epoch": 0.006858122588941277, |
| "grad_norm": 15.354884464504476, |
| "kl": 0.0011739730834960938, |
| "learning_rate": 9.93127147766323e-07, |
| "loss": 0.0, |
| "reward": 1.1484375, |
| "reward_std": 0.47851087525486946, |
| "rewards/accuracy_reward_action": 0.20703125, |
| "rewards/accuracy_reward_coord": 0.03125, |
| "rewards/format_reward": 0.91015625, |
| "step": 2 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.06640625, |
| "epoch": 0.010287183883411917, |
| "grad_norm": 3.579290314608233, |
| "kl": 0.0019245147705078125, |
| "learning_rate": 9.896907216494845e-07, |
| "loss": 0.0001, |
| "reward": 1.27734375, |
| "reward_std": 0.5746662765741348, |
| "rewards/accuracy_reward_action": 0.33203125, |
| "rewards/accuracy_reward_coord": 0.09375, |
| "rewards/format_reward": 0.8515625, |
| "step": 3 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.6875, |
| "epoch": 0.013716245177882554, |
| "grad_norm": 7.574944877248717, |
| "kl": 0.006870269775390625, |
| "learning_rate": 9.86254295532646e-07, |
| "loss": 0.0003, |
| "reward": 1.41796875, |
| "reward_std": 0.5559290274977684, |
| "rewards/accuracy_reward_action": 0.41015625, |
| "rewards/accuracy_reward_coord": 0.109375, |
| "rewards/format_reward": 0.8984375, |
| "step": 4 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.12109375, |
| "epoch": 0.017145306472353194, |
| "grad_norm": 6.512027175960988, |
| "kl": 0.008758544921875, |
| "learning_rate": 9.828178694158075e-07, |
| "loss": 0.0004, |
| "reward": 1.4296875, |
| "reward_std": 0.5759975053369999, |
| "rewards/accuracy_reward_action": 0.49609375, |
| "rewards/accuracy_reward_coord": 0.03125, |
| "rewards/format_reward": 0.90234375, |
| "step": 5 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 116.57421875, |
| "epoch": 0.020574367766823833, |
| "grad_norm": 8.442076588732956, |
| "kl": 0.014019012451171875, |
| "learning_rate": 9.79381443298969e-07, |
| "loss": 0.0006, |
| "reward": 1.6796875, |
| "reward_std": 0.6102640256285667, |
| "rewards/accuracy_reward_action": 0.64453125, |
| "rewards/accuracy_reward_coord": 0.11328125, |
| "rewards/format_reward": 0.921875, |
| "step": 6 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.53515625, |
| "epoch": 0.02400342906129447, |
| "grad_norm": 4.77040444096485, |
| "kl": 0.017120361328125, |
| "learning_rate": 9.759450171821305e-07, |
| "loss": 0.0007, |
| "reward": 1.74609375, |
| "reward_std": 0.4844810925424099, |
| "rewards/accuracy_reward_action": 0.7265625, |
| "rewards/accuracy_reward_coord": 0.0546875, |
| "rewards/format_reward": 0.96484375, |
| "step": 7 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.5390625, |
| "epoch": 0.02743249035576511, |
| "grad_norm": 3.7032651890067525, |
| "kl": 0.01229095458984375, |
| "learning_rate": 9.72508591065292e-07, |
| "loss": 0.0005, |
| "reward": 1.69921875, |
| "reward_std": 0.47686367109417915, |
| "rewards/accuracy_reward_action": 0.69140625, |
| "rewards/accuracy_reward_coord": 0.09375, |
| "rewards/format_reward": 0.9140625, |
| "step": 8 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.12890625, |
| "epoch": 0.030861551650235748, |
| "grad_norm": 5.007916582021746, |
| "kl": 0.01473236083984375, |
| "learning_rate": 9.690721649484535e-07, |
| "loss": 0.0006, |
| "reward": 1.6875, |
| "reward_std": 0.5727768018841743, |
| "rewards/accuracy_reward_action": 0.65625, |
| "rewards/accuracy_reward_coord": 0.08203125, |
| "rewards/format_reward": 0.94921875, |
| "step": 9 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.96875, |
| "epoch": 0.03429061294470639, |
| "grad_norm": 4.167812341811835, |
| "kl": 0.0190887451171875, |
| "learning_rate": 9.65635738831615e-07, |
| "loss": 0.0008, |
| "reward": 1.7109375, |
| "reward_std": 0.45130953192710876, |
| "rewards/accuracy_reward_action": 0.69921875, |
| "rewards/accuracy_reward_coord": 0.06640625, |
| "rewards/format_reward": 0.9453125, |
| "step": 10 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.6875, |
| "epoch": 0.03771967423917703, |
| "grad_norm": 2.6525737109173715, |
| "kl": 0.01401519775390625, |
| "learning_rate": 9.621993127147767e-07, |
| "loss": 0.0006, |
| "reward": 1.6953125, |
| "reward_std": 0.4206106849014759, |
| "rewards/accuracy_reward_action": 0.671875, |
| "rewards/accuracy_reward_coord": 0.08984375, |
| "rewards/format_reward": 0.93359375, |
| "step": 11 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.2734375, |
| "epoch": 0.041148735533647667, |
| "grad_norm": 6.690879246272424, |
| "kl": 0.01448822021484375, |
| "learning_rate": 9.587628865979382e-07, |
| "loss": 0.0006, |
| "reward": 1.78515625, |
| "reward_std": 0.36769258975982666, |
| "rewards/accuracy_reward_action": 0.71484375, |
| "rewards/accuracy_reward_coord": 0.09375, |
| "rewards/format_reward": 0.9765625, |
| "step": 12 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.9375, |
| "epoch": 0.044577796828118306, |
| "grad_norm": 1.8641990568589721, |
| "kl": 0.01544952392578125, |
| "learning_rate": 9.553264604810997e-07, |
| "loss": 0.0006, |
| "reward": 1.9140625, |
| "reward_std": 0.29037280660122633, |
| "rewards/accuracy_reward_action": 0.8046875, |
| "rewards/accuracy_reward_coord": 0.12109375, |
| "rewards/format_reward": 0.98828125, |
| "step": 13 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.015625, |
| "epoch": 0.04800685812258894, |
| "grad_norm": 4.660763921206706, |
| "kl": 0.01983642578125, |
| "learning_rate": 9.518900343642611e-07, |
| "loss": 0.0008, |
| "reward": 1.8515625, |
| "reward_std": 0.37489059288054705, |
| "rewards/accuracy_reward_action": 0.76171875, |
| "rewards/accuracy_reward_coord": 0.1015625, |
| "rewards/format_reward": 0.98828125, |
| "step": 14 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.33203125, |
| "epoch": 0.05143591941705958, |
| "grad_norm": 4.101279763347818, |
| "kl": 0.01727294921875, |
| "learning_rate": 9.484536082474226e-07, |
| "loss": 0.0007, |
| "reward": 1.86328125, |
| "reward_std": 0.3477381020784378, |
| "rewards/accuracy_reward_action": 0.76171875, |
| "rewards/accuracy_reward_coord": 0.11328125, |
| "rewards/format_reward": 0.98828125, |
| "step": 15 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.69140625, |
| "epoch": 0.05486498071153022, |
| "grad_norm": 4.209451477714062, |
| "kl": 0.02355194091796875, |
| "learning_rate": 9.450171821305841e-07, |
| "loss": 0.0009, |
| "reward": 2.06640625, |
| "reward_std": 0.3058856502175331, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.19921875, |
| "rewards/format_reward": 0.98046875, |
| "step": 16 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.98828125, |
| "epoch": 0.05829404200600086, |
| "grad_norm": 3.315964426251405, |
| "kl": 0.023223876953125, |
| "learning_rate": 9.415807560137456e-07, |
| "loss": 0.0009, |
| "reward": 1.8828125, |
| "reward_std": 0.22431878745555878, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.0703125, |
| "rewards/format_reward": 0.984375, |
| "step": 17 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.31640625, |
| "epoch": 0.061723103300471496, |
| "grad_norm": 5.469921639585009, |
| "kl": 0.02007293701171875, |
| "learning_rate": 9.381443298969071e-07, |
| "loss": 0.0008, |
| "reward": 1.9296875, |
| "reward_std": 0.26369384955614805, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.0859375, |
| "rewards/format_reward": 0.9921875, |
| "step": 18 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 114.91796875, |
| "epoch": 0.06515216459494214, |
| "grad_norm": 5.66633473155319, |
| "kl": 0.02239227294921875, |
| "learning_rate": 9.347079037800687e-07, |
| "loss": 0.0009, |
| "reward": 1.86328125, |
| "reward_std": 0.3953590150922537, |
| "rewards/accuracy_reward_action": 0.76171875, |
| "rewards/accuracy_reward_coord": 0.1171875, |
| "rewards/format_reward": 0.984375, |
| "step": 19 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.84375, |
| "epoch": 0.06858122588941278, |
| "grad_norm": 3.371967005233721, |
| "kl": 0.02564239501953125, |
| "learning_rate": 9.312714776632303e-07, |
| "loss": 0.001, |
| "reward": 1.90625, |
| "reward_std": 0.1727309264242649, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.03515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 20 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.23046875, |
| "epoch": 0.07201028718388341, |
| "grad_norm": 9.311589365168965, |
| "kl": 0.021026611328125, |
| "learning_rate": 9.278350515463918e-07, |
| "loss": 0.0008, |
| "reward": 1.94140625, |
| "reward_std": 0.301584979519248, |
| "rewards/accuracy_reward_action": 0.8046875, |
| "rewards/accuracy_reward_coord": 0.140625, |
| "rewards/format_reward": 0.99609375, |
| "step": 21 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.15625, |
| "epoch": 0.07543934847835405, |
| "grad_norm": 2.4741963314612794, |
| "kl": 0.0182037353515625, |
| "learning_rate": 9.243986254295533e-07, |
| "loss": 0.0007, |
| "reward": 2.06640625, |
| "reward_std": 0.20661969017237425, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.1640625, |
| "rewards/format_reward": 0.99609375, |
| "step": 22 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.8046875, |
| "epoch": 0.0788684097728247, |
| "grad_norm": 1.8074831491618741, |
| "kl": 0.02187347412109375, |
| "learning_rate": 9.209621993127147e-07, |
| "loss": 0.0009, |
| "reward": 1.9921875, |
| "reward_std": 0.22818244062364101, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.0859375, |
| "rewards/format_reward": 1.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.1640625, |
| "epoch": 0.08229747106729533, |
| "grad_norm": 2.116378010233233, |
| "kl": 0.0333404541015625, |
| "learning_rate": 9.175257731958762e-07, |
| "loss": 0.0013, |
| "reward": 1.9140625, |
| "reward_std": 0.27086862176656723, |
| "rewards/accuracy_reward_action": 0.78515625, |
| "rewards/accuracy_reward_coord": 0.13671875, |
| "rewards/format_reward": 0.9921875, |
| "step": 24 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.45703125, |
| "epoch": 0.08572653236176597, |
| "grad_norm": 2.017007786819895, |
| "kl": 0.02716064453125, |
| "learning_rate": 9.140893470790378e-07, |
| "loss": 0.0011, |
| "reward": 1.875, |
| "reward_std": 0.38331808149814606, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.0859375, |
| "rewards/format_reward": 0.98046875, |
| "step": 25 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.2734375, |
| "epoch": 0.08915559365623661, |
| "grad_norm": 3.1854233612302796, |
| "kl": 0.025115966796875, |
| "learning_rate": 9.106529209621993e-07, |
| "loss": 0.001, |
| "reward": 1.9140625, |
| "reward_std": 0.23374123964458704, |
| "rewards/accuracy_reward_action": 0.83984375, |
| "rewards/accuracy_reward_coord": 0.078125, |
| "rewards/format_reward": 0.99609375, |
| "step": 26 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.72265625, |
| "epoch": 0.09258465495070724, |
| "grad_norm": 3.25367470492351, |
| "kl": 0.0232696533203125, |
| "learning_rate": 9.072164948453608e-07, |
| "loss": 0.0009, |
| "reward": 1.98828125, |
| "reward_std": 0.15656260214745998, |
| "rewards/accuracy_reward_action": 0.9296875, |
| "rewards/accuracy_reward_coord": 0.06640625, |
| "rewards/format_reward": 0.9921875, |
| "step": 27 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.7890625, |
| "epoch": 0.09601371624517788, |
| "grad_norm": 3.8415142050104167, |
| "kl": 0.0357666015625, |
| "learning_rate": 9.037800687285223e-07, |
| "loss": 0.0014, |
| "reward": 1.8984375, |
| "reward_std": 0.2581741688773036, |
| "rewards/accuracy_reward_action": 0.83984375, |
| "rewards/accuracy_reward_coord": 0.06640625, |
| "rewards/format_reward": 0.9921875, |
| "step": 28 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.82421875, |
| "epoch": 0.09944277753964852, |
| "grad_norm": 2.516551663652597, |
| "kl": 0.0266876220703125, |
| "learning_rate": 9.003436426116838e-07, |
| "loss": 0.0011, |
| "reward": 1.98828125, |
| "reward_std": 0.20948081836104393, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.078125, |
| "rewards/format_reward": 0.99609375, |
| "step": 29 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.3828125, |
| "epoch": 0.10287183883411916, |
| "grad_norm": 2.095156678702029, |
| "kl": 0.03021240234375, |
| "learning_rate": 8.969072164948454e-07, |
| "loss": 0.0012, |
| "reward": 2.0234375, |
| "reward_std": 0.16610248386859894, |
| "rewards/accuracy_reward_action": 0.91015625, |
| "rewards/accuracy_reward_coord": 0.11328125, |
| "rewards/format_reward": 1.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.80859375, |
| "epoch": 0.1063009001285898, |
| "grad_norm": 5.27355668616815, |
| "kl": 0.0283203125, |
| "learning_rate": 8.934707903780069e-07, |
| "loss": 0.0011, |
| "reward": 1.96484375, |
| "reward_std": 0.2747951643541455, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.16015625, |
| "rewards/format_reward": 0.9921875, |
| "step": 31 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.84375, |
| "epoch": 0.10972996142306043, |
| "grad_norm": 6.545819641462799, |
| "kl": 0.0323638916015625, |
| "learning_rate": 8.900343642611683e-07, |
| "loss": 0.0013, |
| "reward": 1.91796875, |
| "reward_std": 0.2592557203024626, |
| "rewards/accuracy_reward_action": 0.79296875, |
| "rewards/accuracy_reward_coord": 0.125, |
| "rewards/format_reward": 1.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.0859375, |
| "epoch": 0.11315902271753107, |
| "grad_norm": 2.345212742782235, |
| "kl": 0.034271240234375, |
| "learning_rate": 8.865979381443298e-07, |
| "loss": 0.0014, |
| "reward": 2.05078125, |
| "reward_std": 0.2973444387316704, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.140625, |
| "rewards/format_reward": 0.99609375, |
| "step": 33 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.234375, |
| "epoch": 0.11658808401200171, |
| "grad_norm": 9.480377541811404, |
| "kl": 0.0315399169921875, |
| "learning_rate": 8.831615120274913e-07, |
| "loss": 0.0013, |
| "reward": 1.91015625, |
| "reward_std": 0.2874891981482506, |
| "rewards/accuracy_reward_action": 0.8046875, |
| "rewards/accuracy_reward_coord": 0.109375, |
| "rewards/format_reward": 0.99609375, |
| "step": 34 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.14453125, |
| "epoch": 0.12001714530647235, |
| "grad_norm": 4.093247592407727, |
| "kl": 0.032470703125, |
| "learning_rate": 8.797250859106528e-07, |
| "loss": 0.0013, |
| "reward": 1.96875, |
| "reward_std": 0.2139760795980692, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.07421875, |
| "rewards/format_reward": 1.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.47265625, |
| "epoch": 0.12344620660094299, |
| "grad_norm": 4.414899743605823, |
| "kl": 0.0321807861328125, |
| "learning_rate": 8.762886597938144e-07, |
| "loss": 0.0013, |
| "reward": 1.99609375, |
| "reward_std": 0.10718034580349922, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.07421875, |
| "rewards/format_reward": 1.0, |
| "step": 36 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.171875, |
| "epoch": 0.12687526789541362, |
| "grad_norm": 1.9240548034063036, |
| "kl": 0.0298919677734375, |
| "learning_rate": 8.728522336769759e-07, |
| "loss": 0.0012, |
| "reward": 1.90625, |
| "reward_std": 0.19055858626961708, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.078125, |
| "rewards/format_reward": 1.0, |
| "step": 37 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.30859375, |
| "epoch": 0.13030432918988427, |
| "grad_norm": 1.7182163505314758, |
| "kl": 0.0316925048828125, |
| "learning_rate": 8.694158075601374e-07, |
| "loss": 0.0013, |
| "reward": 1.9921875, |
| "reward_std": 0.19306486751884222, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.09375, |
| "rewards/format_reward": 0.99609375, |
| "step": 38 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.19921875, |
| "epoch": 0.1337333904843549, |
| "grad_norm": 5.580302117235681, |
| "kl": 0.0377655029296875, |
| "learning_rate": 8.659793814432989e-07, |
| "loss": 0.0015, |
| "reward": 1.96484375, |
| "reward_std": 0.1960927415639162, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.1015625, |
| "rewards/format_reward": 0.99609375, |
| "step": 39 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.640625, |
| "epoch": 0.13716245177882555, |
| "grad_norm": 5.347881422440897, |
| "kl": 0.03704833984375, |
| "learning_rate": 8.625429553264604e-07, |
| "loss": 0.0015, |
| "reward": 1.953125, |
| "reward_std": 0.20940626692026854, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.12890625, |
| "rewards/format_reward": 1.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.08203125, |
| "epoch": 0.14059151307329618, |
| "grad_norm": 1.7711400045751344, |
| "kl": 0.03179931640625, |
| "learning_rate": 8.591065292096219e-07, |
| "loss": 0.0013, |
| "reward": 1.93359375, |
| "reward_std": 0.16333417687565088, |
| "rewards/accuracy_reward_action": 0.85546875, |
| "rewards/accuracy_reward_coord": 0.078125, |
| "rewards/format_reward": 1.0, |
| "step": 41 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.72265625, |
| "epoch": 0.14402057436776683, |
| "grad_norm": 7.596003099794815, |
| "kl": 0.038818359375, |
| "learning_rate": 8.556701030927834e-07, |
| "loss": 0.0016, |
| "reward": 2.015625, |
| "reward_std": 0.2686883732676506, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.1328125, |
| "rewards/format_reward": 1.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.046875, |
| "epoch": 0.14744963566223745, |
| "grad_norm": 2.1589949902852568, |
| "kl": 0.0390167236328125, |
| "learning_rate": 8.52233676975945e-07, |
| "loss": 0.0016, |
| "reward": 1.8984375, |
| "reward_std": 0.19987381342798471, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.08984375, |
| "rewards/format_reward": 0.99609375, |
| "step": 43 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.0859375, |
| "epoch": 0.1508786969567081, |
| "grad_norm": 2.985222575414514, |
| "kl": 0.03912353515625, |
| "learning_rate": 8.487972508591065e-07, |
| "loss": 0.0016, |
| "reward": 1.95703125, |
| "reward_std": 0.26305136550217867, |
| "rewards/accuracy_reward_action": 0.80078125, |
| "rewards/accuracy_reward_coord": 0.16015625, |
| "rewards/format_reward": 0.99609375, |
| "step": 44 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.421875, |
| "epoch": 0.15430775825117873, |
| "grad_norm": 1.3069548395660002, |
| "kl": 0.043212890625, |
| "learning_rate": 8.45360824742268e-07, |
| "loss": 0.0017, |
| "reward": 1.94921875, |
| "reward_std": 0.20725996419787407, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.1328125, |
| "rewards/format_reward": 0.9921875, |
| "step": 45 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.7109375, |
| "epoch": 0.1577368195456494, |
| "grad_norm": 2.351651859555171, |
| "kl": 0.0411376953125, |
| "learning_rate": 8.419243986254296e-07, |
| "loss": 0.0016, |
| "reward": 2.0234375, |
| "reward_std": 0.23960923682898283, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.1328125, |
| "rewards/format_reward": 1.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.859375, |
| "epoch": 0.16116588084012, |
| "grad_norm": 3.6702022844794633, |
| "kl": 0.03863525390625, |
| "learning_rate": 8.384879725085911e-07, |
| "loss": 0.0015, |
| "reward": 1.98828125, |
| "reward_std": 0.3140581510961056, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.1796875, |
| "rewards/format_reward": 1.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.94921875, |
| "epoch": 0.16459494213459067, |
| "grad_norm": 2.2650752826812113, |
| "kl": 0.036224365234375, |
| "learning_rate": 8.350515463917526e-07, |
| "loss": 0.0015, |
| "reward": 2.0, |
| "reward_std": 0.21676901914179325, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.1328125, |
| "rewards/format_reward": 1.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.484375, |
| "epoch": 0.1680240034290613, |
| "grad_norm": 1.6014441912105555, |
| "kl": 0.0355682373046875, |
| "learning_rate": 8.316151202749141e-07, |
| "loss": 0.0014, |
| "reward": 1.96484375, |
| "reward_std": 0.2216318678110838, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.14453125, |
| "rewards/format_reward": 1.0, |
| "step": 49 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.46484375, |
| "epoch": 0.17145306472353194, |
| "grad_norm": 6.83244910904526, |
| "kl": 0.04339599609375, |
| "learning_rate": 8.281786941580756e-07, |
| "loss": 0.0017, |
| "reward": 1.90625, |
| "reward_std": 0.22567125875502825, |
| "rewards/accuracy_reward_action": 0.78515625, |
| "rewards/accuracy_reward_coord": 0.12109375, |
| "rewards/format_reward": 1.0, |
| "step": 50 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.20703125, |
| "epoch": 0.17488212601800257, |
| "grad_norm": 1.3227718672397777, |
| "kl": 0.0413818359375, |
| "learning_rate": 8.24742268041237e-07, |
| "loss": 0.0017, |
| "reward": 2.16796875, |
| "reward_std": 0.15281605254858732, |
| "rewards/accuracy_reward_action": 0.984375, |
| "rewards/accuracy_reward_coord": 0.18359375, |
| "rewards/format_reward": 1.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.94921875, |
| "epoch": 0.17831118731247322, |
| "grad_norm": 2.2204178372642938, |
| "kl": 0.0389251708984375, |
| "learning_rate": 8.213058419243986e-07, |
| "loss": 0.0016, |
| "reward": 2.02734375, |
| "reward_std": 0.18083926383405924, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.17578125, |
| "rewards/format_reward": 1.0, |
| "step": 52 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.4453125, |
| "epoch": 0.18174024860694385, |
| "grad_norm": 3.119541528344858, |
| "kl": 0.0362396240234375, |
| "learning_rate": 8.178694158075601e-07, |
| "loss": 0.0014, |
| "reward": 1.9921875, |
| "reward_std": 0.2792065069079399, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.1796875, |
| "rewards/format_reward": 1.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.23828125, |
| "epoch": 0.18516930990141447, |
| "grad_norm": 6.184155571429652, |
| "kl": 0.0556182861328125, |
| "learning_rate": 8.144329896907216e-07, |
| "loss": 0.0022, |
| "reward": 2.06640625, |
| "reward_std": 0.25091784074902534, |
| "rewards/accuracy_reward_action": 0.93359375, |
| "rewards/accuracy_reward_coord": 0.13671875, |
| "rewards/format_reward": 0.99609375, |
| "step": 54 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.27734375, |
| "epoch": 0.18859837119588513, |
| "grad_norm": 2.2553293815115985, |
| "kl": 0.04754638671875, |
| "learning_rate": 8.109965635738831e-07, |
| "loss": 0.0019, |
| "reward": 2.109375, |
| "reward_std": 0.22331714257597923, |
| "rewards/accuracy_reward_action": 0.9296875, |
| "rewards/accuracy_reward_coord": 0.18359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 55 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.9296875, |
| "epoch": 0.19202743249035575, |
| "grad_norm": 5.306929144656568, |
| "kl": 0.0445556640625, |
| "learning_rate": 8.075601374570446e-07, |
| "loss": 0.0018, |
| "reward": 2.0546875, |
| "reward_std": 0.33029378205537796, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.2265625, |
| "rewards/format_reward": 0.9921875, |
| "step": 56 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.375, |
| "epoch": 0.1954564937848264, |
| "grad_norm": 3.3360995991006313, |
| "kl": 0.037750244140625, |
| "learning_rate": 8.041237113402062e-07, |
| "loss": 0.0015, |
| "reward": 1.9921875, |
| "reward_std": 0.18702251464128494, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.15625, |
| "rewards/format_reward": 1.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.2578125, |
| "epoch": 0.19888555507929703, |
| "grad_norm": 2.5746550275702855, |
| "kl": 0.046600341796875, |
| "learning_rate": 8.006872852233677e-07, |
| "loss": 0.0019, |
| "reward": 2.06640625, |
| "reward_std": 0.27793182618916035, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.23828125, |
| "rewards/format_reward": 1.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.4765625, |
| "epoch": 0.2023146163737677, |
| "grad_norm": 7.922133753917514, |
| "kl": 0.046875, |
| "learning_rate": 7.972508591065292e-07, |
| "loss": 0.0019, |
| "reward": 2.14453125, |
| "reward_std": 0.3949808795005083, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 1.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3046875, |
| "epoch": 0.2057436776682383, |
| "grad_norm": 4.668578467258757, |
| "kl": 0.051544189453125, |
| "learning_rate": 7.938144329896906e-07, |
| "loss": 0.0021, |
| "reward": 2.04296875, |
| "reward_std": 0.24435585644096136, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.1484375, |
| "rewards/format_reward": 1.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.453125, |
| "epoch": 0.20917273896270897, |
| "grad_norm": 2.1768199938478032, |
| "kl": 0.046600341796875, |
| "learning_rate": 7.903780068728521e-07, |
| "loss": 0.0019, |
| "reward": 2.02734375, |
| "reward_std": 0.23643438424915075, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.15625, |
| "rewards/format_reward": 0.99609375, |
| "step": 61 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.3515625, |
| "epoch": 0.2126018002571796, |
| "grad_norm": 2.580776755790538, |
| "kl": 0.06036376953125, |
| "learning_rate": 7.869415807560136e-07, |
| "loss": 0.0024, |
| "reward": 2.2109375, |
| "reward_std": 0.2773527940735221, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.3046875, |
| "rewards/format_reward": 1.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.28515625, |
| "epoch": 0.21603086155165024, |
| "grad_norm": 3.6655554460146433, |
| "kl": 0.0533447265625, |
| "learning_rate": 7.835051546391752e-07, |
| "loss": 0.0021, |
| "reward": 2.046875, |
| "reward_std": 0.21200658101588488, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.1640625, |
| "rewards/format_reward": 0.99609375, |
| "step": 63 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.75390625, |
| "epoch": 0.21945992284612087, |
| "grad_norm": 2.313170128101062, |
| "kl": 0.060028076171875, |
| "learning_rate": 7.800687285223368e-07, |
| "loss": 0.0024, |
| "reward": 2.1953125, |
| "reward_std": 0.14374582655727863, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.2578125, |
| "rewards/format_reward": 1.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.44140625, |
| "epoch": 0.22288898414059152, |
| "grad_norm": 2.3617237358151515, |
| "kl": 0.086151123046875, |
| "learning_rate": 7.766323024054983e-07, |
| "loss": 0.0034, |
| "reward": 2.16796875, |
| "reward_std": 0.23817015439271927, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.2890625, |
| "rewards/format_reward": 1.0, |
| "step": 65 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.0078125, |
| "epoch": 0.22631804543506215, |
| "grad_norm": 3.3699940809288886, |
| "kl": 0.05230712890625, |
| "learning_rate": 7.731958762886598e-07, |
| "loss": 0.0021, |
| "reward": 2.1953125, |
| "reward_std": 0.2761722579598427, |
| "rewards/accuracy_reward_action": 0.91796875, |
| "rewards/accuracy_reward_coord": 0.28125, |
| "rewards/format_reward": 0.99609375, |
| "step": 66 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.72265625, |
| "epoch": 0.2297471067295328, |
| "grad_norm": 45.77607256234026, |
| "kl": 0.054962158203125, |
| "learning_rate": 7.697594501718213e-07, |
| "loss": 0.0022, |
| "reward": 2.125, |
| "reward_std": 0.24921870045363903, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.234375, |
| "rewards/format_reward": 0.99609375, |
| "step": 67 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.109375, |
| "epoch": 0.23317616802400343, |
| "grad_norm": 2.503895542298818, |
| "kl": 0.053070068359375, |
| "learning_rate": 7.663230240549829e-07, |
| "loss": 0.0021, |
| "reward": 2.1953125, |
| "reward_std": 0.18460038863122463, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.2890625, |
| "rewards/format_reward": 1.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.36328125, |
| "epoch": 0.23660522931847408, |
| "grad_norm": 2.9062050779497084, |
| "kl": 0.074798583984375, |
| "learning_rate": 7.628865979381443e-07, |
| "loss": 0.003, |
| "reward": 2.22265625, |
| "reward_std": 0.2973867813125253, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 1.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.3671875, |
| "epoch": 0.2400342906129447, |
| "grad_norm": 5.380735077015675, |
| "kl": 0.056243896484375, |
| "learning_rate": 7.594501718213058e-07, |
| "loss": 0.0023, |
| "reward": 2.12890625, |
| "reward_std": 0.4472039174288511, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.32421875, |
| "rewards/format_reward": 0.99609375, |
| "step": 70 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.67578125, |
| "epoch": 0.24346335190741533, |
| "grad_norm": 3.9243843409226655, |
| "kl": 0.046630859375, |
| "learning_rate": 7.560137457044673e-07, |
| "loss": 0.0019, |
| "reward": 1.95703125, |
| "reward_std": 0.27867609076201916, |
| "rewards/accuracy_reward_action": 0.7734375, |
| "rewards/accuracy_reward_coord": 0.18359375, |
| "rewards/format_reward": 1.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.04296875, |
| "epoch": 0.24689241320188599, |
| "grad_norm": 3.4045750384697007, |
| "kl": 0.0482177734375, |
| "learning_rate": 7.525773195876288e-07, |
| "loss": 0.0019, |
| "reward": 2.16796875, |
| "reward_std": 0.3632250167429447, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 1.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.5078125, |
| "epoch": 0.2503214744963566, |
| "grad_norm": 4.797188528974318, |
| "kl": 0.0504150390625, |
| "learning_rate": 7.491408934707904e-07, |
| "loss": 0.002, |
| "reward": 2.265625, |
| "reward_std": 0.3736470714211464, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.390625, |
| "rewards/format_reward": 0.99609375, |
| "step": 73 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.95703125, |
| "epoch": 0.25375053579082724, |
| "grad_norm": 2.0498266773727294, |
| "kl": 0.044097900390625, |
| "learning_rate": 7.457044673539519e-07, |
| "loss": 0.0018, |
| "reward": 2.09765625, |
| "reward_std": 0.32716665603220463, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.25390625, |
| "rewards/format_reward": 1.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2734375, |
| "epoch": 0.2571795970852979, |
| "grad_norm": 5.395875774985416, |
| "kl": 0.052581787109375, |
| "learning_rate": 7.422680412371134e-07, |
| "loss": 0.0021, |
| "reward": 2.00390625, |
| "reward_std": 0.34148280695080757, |
| "rewards/accuracy_reward_action": 0.8046875, |
| "rewards/accuracy_reward_coord": 0.21875, |
| "rewards/format_reward": 0.98046875, |
| "step": 75 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.4921875, |
| "epoch": 0.26060865837976854, |
| "grad_norm": 2.5491567686917365, |
| "kl": 0.05126953125, |
| "learning_rate": 7.388316151202749e-07, |
| "loss": 0.0021, |
| "reward": 2.0625, |
| "reward_std": 0.2948285508900881, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.2109375, |
| "rewards/format_reward": 1.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.29296875, |
| "epoch": 0.26403771967423917, |
| "grad_norm": 3.801863021542061, |
| "kl": 0.05523681640625, |
| "learning_rate": 7.353951890034364e-07, |
| "loss": 0.0022, |
| "reward": 2.26171875, |
| "reward_std": 0.310369904153049, |
| "rewards/accuracy_reward_action": 0.9296875, |
| "rewards/accuracy_reward_coord": 0.33203125, |
| "rewards/format_reward": 1.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.5390625, |
| "epoch": 0.2674667809687098, |
| "grad_norm": 1.7906784993530962, |
| "kl": 0.057159423828125, |
| "learning_rate": 7.319587628865978e-07, |
| "loss": 0.0023, |
| "reward": 2.09375, |
| "reward_std": 0.25855094753205776, |
| "rewards/accuracy_reward_action": 0.83984375, |
| "rewards/accuracy_reward_coord": 0.25390625, |
| "rewards/format_reward": 1.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.703125, |
| "epoch": 0.2708958422631805, |
| "grad_norm": 3.2482720424647993, |
| "kl": 0.06103515625, |
| "learning_rate": 7.285223367697594e-07, |
| "loss": 0.0024, |
| "reward": 2.15625, |
| "reward_std": 0.2630355451256037, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.2890625, |
| "rewards/format_reward": 1.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.47265625, |
| "epoch": 0.2743249035576511, |
| "grad_norm": 6.643515027367905, |
| "kl": 0.075408935546875, |
| "learning_rate": 7.250859106529209e-07, |
| "loss": 0.003, |
| "reward": 2.25390625, |
| "reward_std": 0.2786760898306966, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.33984375, |
| "rewards/format_reward": 1.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.171875, |
| "epoch": 0.2777539648521217, |
| "grad_norm": 2.4853316682654345, |
| "kl": 0.058929443359375, |
| "learning_rate": 7.216494845360824e-07, |
| "loss": 0.0024, |
| "reward": 2.04296875, |
| "reward_std": 0.3351527014747262, |
| "rewards/accuracy_reward_action": 0.7734375, |
| "rewards/accuracy_reward_coord": 0.2734375, |
| "rewards/format_reward": 0.99609375, |
| "step": 81 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.640625, |
| "epoch": 0.28118302614659235, |
| "grad_norm": 4.577244916480412, |
| "kl": 0.06158447265625, |
| "learning_rate": 7.182130584192439e-07, |
| "loss": 0.0025, |
| "reward": 2.0625, |
| "reward_std": 0.2432457758113742, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.25390625, |
| "rewards/format_reward": 1.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.34765625, |
| "epoch": 0.28461208744106303, |
| "grad_norm": 4.178949531512928, |
| "kl": 0.054229736328125, |
| "learning_rate": 7.147766323024054e-07, |
| "loss": 0.0022, |
| "reward": 2.11328125, |
| "reward_std": 0.3627553675323725, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.27734375, |
| "rewards/format_reward": 0.9921875, |
| "step": 83 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.85546875, |
| "epoch": 0.28804114873553366, |
| "grad_norm": 1.9207012352711366, |
| "kl": 0.0654296875, |
| "learning_rate": 7.11340206185567e-07, |
| "loss": 0.0026, |
| "reward": 2.1484375, |
| "reward_std": 0.29073167592287064, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.296875, |
| "rewards/format_reward": 0.9921875, |
| "step": 84 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.2421875, |
| "epoch": 0.2914702100300043, |
| "grad_norm": 3.819937446906779, |
| "kl": 0.096038818359375, |
| "learning_rate": 7.079037800687286e-07, |
| "loss": 0.0038, |
| "reward": 2.09375, |
| "reward_std": 0.3331120489165187, |
| "rewards/accuracy_reward_action": 0.79296875, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.9921875, |
| "step": 85 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.3046875, |
| "epoch": 0.2948992713244749, |
| "grad_norm": 2.4595638989820072, |
| "kl": 0.07666015625, |
| "learning_rate": 7.044673539518901e-07, |
| "loss": 0.0031, |
| "reward": 2.16796875, |
| "reward_std": 0.2625892860814929, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 86 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.01171875, |
| "epoch": 0.2983283326189456, |
| "grad_norm": 2.8999588777720158, |
| "kl": 0.066436767578125, |
| "learning_rate": 7.010309278350515e-07, |
| "loss": 0.0027, |
| "reward": 2.07421875, |
| "reward_std": 0.257931400090456, |
| "rewards/accuracy_reward_action": 0.91796875, |
| "rewards/accuracy_reward_coord": 0.1640625, |
| "rewards/format_reward": 0.9921875, |
| "step": 87 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.02734375, |
| "epoch": 0.3017573939134162, |
| "grad_norm": 2.4489745976786357, |
| "kl": 0.064361572265625, |
| "learning_rate": 6.97594501718213e-07, |
| "loss": 0.0026, |
| "reward": 2.19140625, |
| "reward_std": 0.276412351988256, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 1.0, |
| "step": 88 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.01953125, |
| "epoch": 0.30518645520788684, |
| "grad_norm": 3.049686535143818, |
| "kl": 0.059539794921875, |
| "learning_rate": 6.941580756013746e-07, |
| "loss": 0.0024, |
| "reward": 2.140625, |
| "reward_std": 0.253778631798923, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.29296875, |
| "rewards/format_reward": 0.99609375, |
| "step": 89 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2421875, |
| "epoch": 0.30861551650235747, |
| "grad_norm": 4.904039350613208, |
| "kl": 0.05462646484375, |
| "learning_rate": 6.907216494845361e-07, |
| "loss": 0.0022, |
| "reward": 2.13671875, |
| "reward_std": 0.32303678803145885, |
| "rewards/accuracy_reward_action": 0.87109375, |
| "rewards/accuracy_reward_coord": 0.265625, |
| "rewards/format_reward": 1.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.69140625, |
| "epoch": 0.3120445777968281, |
| "grad_norm": 2.9357481693358882, |
| "kl": 0.055145263671875, |
| "learning_rate": 6.872852233676976e-07, |
| "loss": 0.0022, |
| "reward": 2.21875, |
| "reward_std": 0.2909819334745407, |
| "rewards/accuracy_reward_action": 0.84765625, |
| "rewards/accuracy_reward_coord": 0.37890625, |
| "rewards/format_reward": 0.9921875, |
| "step": 91 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.52734375, |
| "epoch": 0.3154736390912988, |
| "grad_norm": 2.5466287089726616, |
| "kl": 0.058929443359375, |
| "learning_rate": 6.838487972508591e-07, |
| "loss": 0.0024, |
| "reward": 2.24609375, |
| "reward_std": 0.23528317362070084, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 0.9921875, |
| "step": 92 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.81640625, |
| "epoch": 0.3189027003857694, |
| "grad_norm": 2.6122685492613713, |
| "kl": 0.062408447265625, |
| "learning_rate": 6.804123711340206e-07, |
| "loss": 0.0025, |
| "reward": 2.14453125, |
| "reward_std": 0.317325497046113, |
| "rewards/accuracy_reward_action": 0.91015625, |
| "rewards/accuracy_reward_coord": 0.23828125, |
| "rewards/format_reward": 0.99609375, |
| "step": 93 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.265625, |
| "epoch": 0.32233176168024, |
| "grad_norm": 2.1487987963276676, |
| "kl": 0.0521392822265625, |
| "learning_rate": 6.769759450171821e-07, |
| "loss": 0.0021, |
| "reward": 2.2421875, |
| "reward_std": 0.31470869667828083, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 94 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.640625, |
| "epoch": 0.32576082297471065, |
| "grad_norm": 2.2669655916615024, |
| "kl": 0.053009033203125, |
| "learning_rate": 6.735395189003437e-07, |
| "loss": 0.0021, |
| "reward": 2.16796875, |
| "reward_std": 0.2151595028117299, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.25390625, |
| "rewards/format_reward": 0.98828125, |
| "step": 95 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.5859375, |
| "epoch": 0.32918988426918133, |
| "grad_norm": 2.167196384717679, |
| "kl": 0.054656982421875, |
| "learning_rate": 6.701030927835051e-07, |
| "loss": 0.0022, |
| "reward": 2.25, |
| "reward_std": 0.29653357807546854, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 96 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 117.77734375, |
| "epoch": 0.33261894556365196, |
| "grad_norm": 3.6750107049628564, |
| "kl": 0.05615234375, |
| "learning_rate": 6.666666666666666e-07, |
| "loss": 0.0022, |
| "reward": 2.125, |
| "reward_std": 0.26538573764264584, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 0.98828125, |
| "step": 97 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.62109375, |
| "epoch": 0.3360480068581226, |
| "grad_norm": 22.80143467477284, |
| "kl": 0.05291748046875, |
| "learning_rate": 6.632302405498281e-07, |
| "loss": 0.0021, |
| "reward": 2.16796875, |
| "reward_std": 0.24659765977412462, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.25, |
| "rewards/format_reward": 0.99609375, |
| "step": 98 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.6015625, |
| "epoch": 0.3394770681525932, |
| "grad_norm": 2.8406924020432385, |
| "kl": 0.055755615234375, |
| "learning_rate": 6.597938144329896e-07, |
| "loss": 0.0022, |
| "reward": 2.33203125, |
| "reward_std": 0.22755195572972298, |
| "rewards/accuracy_reward_action": 0.94921875, |
| "rewards/accuracy_reward_coord": 0.38671875, |
| "rewards/format_reward": 0.99609375, |
| "step": 99 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.59765625, |
| "epoch": 0.3429061294470639, |
| "grad_norm": 2.4016747335432047, |
| "kl": 0.056793212890625, |
| "learning_rate": 6.563573883161512e-07, |
| "loss": 0.0023, |
| "reward": 2.16015625, |
| "reward_std": 0.29211112204939127, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.29296875, |
| "rewards/format_reward": 0.98828125, |
| "step": 100 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.04296875, |
| "epoch": 0.3463351907415345, |
| "grad_norm": 3.724814765953206, |
| "kl": 0.0584716796875, |
| "learning_rate": 6.529209621993127e-07, |
| "loss": 0.0023, |
| "reward": 2.2265625, |
| "reward_std": 0.2973606949672103, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.34765625, |
| "rewards/format_reward": 0.98828125, |
| "step": 101 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.7734375, |
| "epoch": 0.34976425203600514, |
| "grad_norm": 2.903396563380394, |
| "kl": 0.05841064453125, |
| "learning_rate": 6.494845360824742e-07, |
| "loss": 0.0023, |
| "reward": 2.1484375, |
| "reward_std": 0.30724026821553707, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.99609375, |
| "step": 102 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.01171875, |
| "epoch": 0.35319331333047577, |
| "grad_norm": 4.389088874211473, |
| "kl": 0.062530517578125, |
| "learning_rate": 6.460481099656357e-07, |
| "loss": 0.0025, |
| "reward": 2.140625, |
| "reward_std": 0.2465912913903594, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.23828125, |
| "rewards/format_reward": 0.99609375, |
| "step": 103 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.8203125, |
| "epoch": 0.35662237462494645, |
| "grad_norm": 7.379865047803019, |
| "kl": 0.06072998046875, |
| "learning_rate": 6.426116838487972e-07, |
| "loss": 0.0024, |
| "reward": 2.265625, |
| "reward_std": 0.25317383371293545, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 0.98828125, |
| "step": 104 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.44140625, |
| "epoch": 0.3600514359194171, |
| "grad_norm": 10.504549459335141, |
| "kl": 0.052886962890625, |
| "learning_rate": 6.391752577319586e-07, |
| "loss": 0.0021, |
| "reward": 2.33984375, |
| "reward_std": 0.28095037396997213, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.40625, |
| "rewards/format_reward": 0.99609375, |
| "step": 105 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.91796875, |
| "epoch": 0.3634804972138877, |
| "grad_norm": 5.045451831718068, |
| "kl": 0.059295654296875, |
| "learning_rate": 6.357388316151202e-07, |
| "loss": 0.0024, |
| "reward": 2.1875, |
| "reward_std": 0.3055070545524359, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 0.99609375, |
| "step": 106 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.6640625, |
| "epoch": 0.3669095585083583, |
| "grad_norm": 4.063268332407979, |
| "kl": 0.054443359375, |
| "learning_rate": 6.323024054982817e-07, |
| "loss": 0.0022, |
| "reward": 2.24609375, |
| "reward_std": 0.2154239807277918, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 1.0, |
| "step": 107 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.9609375, |
| "epoch": 0.37033861980282895, |
| "grad_norm": 3.041280291099776, |
| "kl": 0.0679931640625, |
| "learning_rate": 6.288659793814433e-07, |
| "loss": 0.0027, |
| "reward": 2.12890625, |
| "reward_std": 0.2933124750852585, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.99609375, |
| "step": 108 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.65234375, |
| "epoch": 0.37376768109729963, |
| "grad_norm": 2.45268780571316, |
| "kl": 0.0626220703125, |
| "learning_rate": 6.254295532646048e-07, |
| "loss": 0.0025, |
| "reward": 2.2421875, |
| "reward_std": 0.21857689507305622, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.3671875, |
| "rewards/format_reward": 1.0, |
| "step": 109 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.7265625, |
| "epoch": 0.37719674239177026, |
| "grad_norm": 6.836912158823584, |
| "kl": 0.056427001953125, |
| "learning_rate": 6.219931271477663e-07, |
| "loss": 0.0023, |
| "reward": 2.26171875, |
| "reward_std": 0.20179079100489616, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 0.99609375, |
| "step": 110 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.28125, |
| "epoch": 0.3806258036862409, |
| "grad_norm": 7.071556300462292, |
| "kl": 0.0528564453125, |
| "learning_rate": 6.185567010309279e-07, |
| "loss": 0.0021, |
| "reward": 2.30078125, |
| "reward_std": 0.3173297345638275, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.46875, |
| "rewards/format_reward": 0.99609375, |
| "step": 111 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.140625, |
| "epoch": 0.3840548649807115, |
| "grad_norm": 3.1256736042468343, |
| "kl": 0.051055908203125, |
| "learning_rate": 6.151202749140894e-07, |
| "loss": 0.002, |
| "reward": 2.08984375, |
| "reward_std": 0.16978827212005854, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.1875, |
| "rewards/format_reward": 1.0, |
| "step": 112 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.12109375, |
| "epoch": 0.3874839262751822, |
| "grad_norm": 4.067305837595256, |
| "kl": 0.07073974609375, |
| "learning_rate": 6.116838487972509e-07, |
| "loss": 0.0028, |
| "reward": 2.265625, |
| "reward_std": 0.32294214703142643, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.4375, |
| "rewards/format_reward": 1.0, |
| "step": 113 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.82421875, |
| "epoch": 0.3909129875696528, |
| "grad_norm": 2.0554187661584153, |
| "kl": 0.06512451171875, |
| "learning_rate": 6.082474226804123e-07, |
| "loss": 0.0026, |
| "reward": 2.15625, |
| "reward_std": 0.28534636087715626, |
| "rewards/accuracy_reward_action": 0.81640625, |
| "rewards/accuracy_reward_coord": 0.34375, |
| "rewards/format_reward": 0.99609375, |
| "step": 114 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.33203125, |
| "epoch": 0.39434204886412344, |
| "grad_norm": 3.3342749231581004, |
| "kl": 0.0616455078125, |
| "learning_rate": 6.048109965635738e-07, |
| "loss": 0.0025, |
| "reward": 2.23828125, |
| "reward_std": 0.32312591560184956, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 1.0, |
| "step": 115 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.9296875, |
| "epoch": 0.39777111015859407, |
| "grad_norm": 3.9108229943071153, |
| "kl": 0.05938720703125, |
| "learning_rate": 6.013745704467354e-07, |
| "loss": 0.0024, |
| "reward": 2.2265625, |
| "reward_std": 0.33316784631460905, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 1.0, |
| "step": 116 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.77734375, |
| "epoch": 0.40120017145306475, |
| "grad_norm": 4.755539569606583, |
| "kl": 0.056365966796875, |
| "learning_rate": 5.979381443298969e-07, |
| "loss": 0.0023, |
| "reward": 2.26171875, |
| "reward_std": 0.31653857976198196, |
| "rewards/accuracy_reward_action": 0.84765625, |
| "rewards/accuracy_reward_coord": 0.421875, |
| "rewards/format_reward": 0.9921875, |
| "step": 117 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.7421875, |
| "epoch": 0.4046292327475354, |
| "grad_norm": 4.463216231770725, |
| "kl": 0.07086181640625, |
| "learning_rate": 5.945017182130584e-07, |
| "loss": 0.0028, |
| "reward": 2.15625, |
| "reward_std": 0.24183008819818497, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.2734375, |
| "rewards/format_reward": 1.0, |
| "step": 118 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.5390625, |
| "epoch": 0.408058294042006, |
| "grad_norm": 2.924845087516111, |
| "kl": 0.063323974609375, |
| "learning_rate": 5.910652920962199e-07, |
| "loss": 0.0025, |
| "reward": 2.10546875, |
| "reward_std": 0.2516478532925248, |
| "rewards/accuracy_reward_action": 0.83203125, |
| "rewards/accuracy_reward_coord": 0.27734375, |
| "rewards/format_reward": 0.99609375, |
| "step": 119 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.78125, |
| "epoch": 0.4114873553364766, |
| "grad_norm": 1.789363566352834, |
| "kl": 0.067291259765625, |
| "learning_rate": 5.876288659793814e-07, |
| "loss": 0.0027, |
| "reward": 2.078125, |
| "reward_std": 0.28578172344714403, |
| "rewards/accuracy_reward_action": 0.796875, |
| "rewards/accuracy_reward_coord": 0.28125, |
| "rewards/format_reward": 1.0, |
| "step": 120 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.8984375, |
| "epoch": 0.4149164166309473, |
| "grad_norm": 3.6455708988858415, |
| "kl": 0.067718505859375, |
| "learning_rate": 5.841924398625429e-07, |
| "loss": 0.0027, |
| "reward": 2.4140625, |
| "reward_std": 0.24929404351860285, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.51171875, |
| "rewards/format_reward": 1.0, |
| "step": 121 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.5, |
| "epoch": 0.41834547792541793, |
| "grad_norm": 1.7899103075940643, |
| "kl": 0.063751220703125, |
| "learning_rate": 5.807560137457045e-07, |
| "loss": 0.0025, |
| "reward": 2.15625, |
| "reward_std": 0.22952503617852926, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.3046875, |
| "rewards/format_reward": 1.0, |
| "step": 122 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.60546875, |
| "epoch": 0.42177453921988856, |
| "grad_norm": 3.293296128480086, |
| "kl": 0.060089111328125, |
| "learning_rate": 5.773195876288659e-07, |
| "loss": 0.0024, |
| "reward": 2.26171875, |
| "reward_std": 0.25149053148925304, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 1.0, |
| "step": 123 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.36328125, |
| "epoch": 0.4252036005143592, |
| "grad_norm": 2.9208531097004053, |
| "kl": 0.05560302734375, |
| "learning_rate": 5.738831615120274e-07, |
| "loss": 0.0022, |
| "reward": 2.1875, |
| "reward_std": 0.24435340333729982, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 124 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.1484375, |
| "epoch": 0.4286326618088298, |
| "grad_norm": 6.435409084870086, |
| "kl": 0.0673828125, |
| "learning_rate": 5.704467353951889e-07, |
| "loss": 0.0027, |
| "reward": 2.17578125, |
| "reward_std": 0.3276154175400734, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.3046875, |
| "rewards/format_reward": 0.99609375, |
| "step": 125 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.1953125, |
| "epoch": 0.4320617231033005, |
| "grad_norm": 1.7152252351092034, |
| "kl": 0.05810546875, |
| "learning_rate": 5.670103092783504e-07, |
| "loss": 0.0023, |
| "reward": 2.08203125, |
| "reward_std": 0.2330364715307951, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.2734375, |
| "rewards/format_reward": 1.0, |
| "step": 126 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.0625, |
| "epoch": 0.4354907843977711, |
| "grad_norm": 2.6261301563581956, |
| "kl": 0.07513427734375, |
| "learning_rate": 5.63573883161512e-07, |
| "loss": 0.003, |
| "reward": 2.4765625, |
| "reward_std": 0.2715753559023142, |
| "rewards/accuracy_reward_action": 0.95703125, |
| "rewards/accuracy_reward_coord": 0.51953125, |
| "rewards/format_reward": 1.0, |
| "step": 127 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.765625, |
| "epoch": 0.43891984569224174, |
| "grad_norm": 2.903714927682652, |
| "kl": 0.06536865234375, |
| "learning_rate": 5.601374570446735e-07, |
| "loss": 0.0026, |
| "reward": 2.32421875, |
| "reward_std": 0.2046777717769146, |
| "rewards/accuracy_reward_action": 0.9609375, |
| "rewards/accuracy_reward_coord": 0.3671875, |
| "rewards/format_reward": 0.99609375, |
| "step": 128 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.91015625, |
| "epoch": 0.44234890698671236, |
| "grad_norm": 3.011327246311379, |
| "kl": 0.0595703125, |
| "learning_rate": 5.56701030927835e-07, |
| "loss": 0.0024, |
| "reward": 2.41796875, |
| "reward_std": 0.2730232598260045, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.5078125, |
| "rewards/format_reward": 0.99609375, |
| "step": 129 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.16796875, |
| "epoch": 0.44577796828118305, |
| "grad_norm": 2.0761484784905995, |
| "kl": 0.0486907958984375, |
| "learning_rate": 5.532646048109966e-07, |
| "loss": 0.0019, |
| "reward": 2.25, |
| "reward_std": 0.26358178071677685, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.38671875, |
| "rewards/format_reward": 1.0, |
| "step": 130 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.5546875, |
| "epoch": 0.44920702957565367, |
| "grad_norm": 5.491907296283607, |
| "kl": 0.066558837890625, |
| "learning_rate": 5.498281786941581e-07, |
| "loss": 0.0027, |
| "reward": 2.2421875, |
| "reward_std": 0.3427243307232857, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 0.9921875, |
| "step": 131 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.234375, |
| "epoch": 0.4526360908701243, |
| "grad_norm": 2.6841999435159627, |
| "kl": 0.06378173828125, |
| "learning_rate": 5.463917525773195e-07, |
| "loss": 0.0026, |
| "reward": 2.140625, |
| "reward_std": 0.18373434152454138, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 132 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.66796875, |
| "epoch": 0.4560651521645949, |
| "grad_norm": 3.417110980260384, |
| "kl": 0.060943603515625, |
| "learning_rate": 5.429553264604811e-07, |
| "loss": 0.0024, |
| "reward": 2.296875, |
| "reward_std": 0.24462180212140083, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 1.0, |
| "step": 133 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.265625, |
| "epoch": 0.4594942134590656, |
| "grad_norm": 5.788338178961461, |
| "kl": 0.066986083984375, |
| "learning_rate": 5.395189003436426e-07, |
| "loss": 0.0027, |
| "reward": 2.22265625, |
| "reward_std": 0.20572047866880894, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 1.0, |
| "step": 134 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.81640625, |
| "epoch": 0.46292327475353623, |
| "grad_norm": 2.963316780716916, |
| "kl": 0.063934326171875, |
| "learning_rate": 5.360824742268041e-07, |
| "loss": 0.0026, |
| "reward": 2.19140625, |
| "reward_std": 0.32010049652308226, |
| "rewards/accuracy_reward_action": 0.8984375, |
| "rewards/accuracy_reward_coord": 0.296875, |
| "rewards/format_reward": 0.99609375, |
| "step": 135 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.42578125, |
| "epoch": 0.46635233604800685, |
| "grad_norm": 6.143892291035493, |
| "kl": 0.0615234375, |
| "learning_rate": 5.326460481099656e-07, |
| "loss": 0.0025, |
| "reward": 2.0859375, |
| "reward_std": 0.31697416584938765, |
| "rewards/accuracy_reward_action": 0.72265625, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 1.0, |
| "step": 136 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.37109375, |
| "epoch": 0.4697813973424775, |
| "grad_norm": 6.9359445377938815, |
| "kl": 0.064300537109375, |
| "learning_rate": 5.292096219931271e-07, |
| "loss": 0.0026, |
| "reward": 2.2578125, |
| "reward_std": 0.33061617612838745, |
| "rewards/accuracy_reward_action": 0.9453125, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 1.0, |
| "step": 137 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.6796875, |
| "epoch": 0.47321045863694816, |
| "grad_norm": 2.588210038877604, |
| "kl": 0.050445556640625, |
| "learning_rate": 5.257731958762887e-07, |
| "loss": 0.002, |
| "reward": 2.25390625, |
| "reward_std": 0.2738156979903579, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.37890625, |
| "rewards/format_reward": 0.99609375, |
| "step": 138 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.6171875, |
| "epoch": 0.4766395199314188, |
| "grad_norm": 1.8678875446907652, |
| "kl": 0.06463623046875, |
| "learning_rate": 5.223367697594502e-07, |
| "loss": 0.0026, |
| "reward": 2.32421875, |
| "reward_std": 0.2723741699010134, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.46484375, |
| "rewards/format_reward": 1.0, |
| "step": 139 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.08203125, |
| "epoch": 0.4800685812258894, |
| "grad_norm": 3.9056747727411705, |
| "kl": 0.05560302734375, |
| "learning_rate": 5.189003436426117e-07, |
| "loss": 0.0022, |
| "reward": 2.3671875, |
| "reward_std": 0.29187846928834915, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.43359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 140 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.125, |
| "epoch": 0.48349764252036004, |
| "grad_norm": 2.073984326464115, |
| "kl": 0.06390380859375, |
| "learning_rate": 5.154639175257731e-07, |
| "loss": 0.0026, |
| "reward": 2.08984375, |
| "reward_std": 0.25432016886770725, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.24609375, |
| "rewards/format_reward": 1.0, |
| "step": 141 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.83984375, |
| "epoch": 0.48692670381483066, |
| "grad_norm": 2.149401200913545, |
| "kl": 0.056243896484375, |
| "learning_rate": 5.120274914089346e-07, |
| "loss": 0.0022, |
| "reward": 2.203125, |
| "reward_std": 0.35073984786868095, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 1.0, |
| "step": 142 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.6875, |
| "epoch": 0.49035576510930134, |
| "grad_norm": 12.159410637716215, |
| "kl": 0.0594482421875, |
| "learning_rate": 5.085910652920962e-07, |
| "loss": 0.0024, |
| "reward": 2.06640625, |
| "reward_std": 0.22434448823332787, |
| "rewards/accuracy_reward_action": 0.76171875, |
| "rewards/accuracy_reward_coord": 0.3046875, |
| "rewards/format_reward": 1.0, |
| "step": 143 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.7265625, |
| "epoch": 0.49378482640377197, |
| "grad_norm": 2.9357482808195083, |
| "kl": 0.060821533203125, |
| "learning_rate": 5.051546391752577e-07, |
| "loss": 0.0024, |
| "reward": 2.171875, |
| "reward_std": 0.25714581832289696, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 144 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.07421875, |
| "epoch": 0.4972138876982426, |
| "grad_norm": 1.6679561448700784, |
| "kl": 0.062225341796875, |
| "learning_rate": 5.017182130584192e-07, |
| "loss": 0.0025, |
| "reward": 2.29296875, |
| "reward_std": 0.2424728237092495, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.421875, |
| "rewards/format_reward": 0.99609375, |
| "step": 145 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.25, |
| "epoch": 0.5006429489927132, |
| "grad_norm": 2.811523326110659, |
| "kl": 0.06048583984375, |
| "learning_rate": 4.982817869415807e-07, |
| "loss": 0.0024, |
| "reward": 2.15625, |
| "reward_std": 0.21675075124949217, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.26171875, |
| "rewards/format_reward": 0.9921875, |
| "step": 146 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.671875, |
| "epoch": 0.5040720102871838, |
| "grad_norm": 9.755349918032064, |
| "kl": 0.059417724609375, |
| "learning_rate": 4.948453608247422e-07, |
| "loss": 0.0024, |
| "reward": 2.36328125, |
| "reward_std": 0.2519870037212968, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.48046875, |
| "rewards/format_reward": 0.9921875, |
| "step": 147 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.1953125, |
| "epoch": 0.5075010715816545, |
| "grad_norm": 3.260159720176464, |
| "kl": 0.0811767578125, |
| "learning_rate": 4.914089347079037e-07, |
| "loss": 0.0032, |
| "reward": 2.30859375, |
| "reward_std": 0.32169321179389954, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.43359375, |
| "rewards/format_reward": 0.9921875, |
| "step": 148 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.390625, |
| "epoch": 0.5109301328761252, |
| "grad_norm": 3.6764074554385893, |
| "kl": 0.059173583984375, |
| "learning_rate": 4.879725085910652e-07, |
| "loss": 0.0024, |
| "reward": 2.13671875, |
| "reward_std": 0.20608290564268827, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.21484375, |
| "rewards/format_reward": 1.0, |
| "step": 149 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.28515625, |
| "epoch": 0.5143591941705958, |
| "grad_norm": 2.4013259367233135, |
| "kl": 0.060577392578125, |
| "learning_rate": 4.845360824742267e-07, |
| "loss": 0.0024, |
| "reward": 2.16796875, |
| "reward_std": 0.2955062361434102, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.27734375, |
| "rewards/format_reward": 0.99609375, |
| "step": 150 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.3984375, |
| "epoch": 0.5177882554650665, |
| "grad_norm": 20.068068245198482, |
| "kl": 0.0634765625, |
| "learning_rate": 4.810996563573884e-07, |
| "loss": 0.0025, |
| "reward": 2.1484375, |
| "reward_std": 0.19920963142067194, |
| "rewards/accuracy_reward_action": 0.80859375, |
| "rewards/accuracy_reward_coord": 0.33984375, |
| "rewards/format_reward": 1.0, |
| "step": 151 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.203125, |
| "epoch": 0.5212173167595371, |
| "grad_norm": 1.9706184774795734, |
| "kl": 0.058929443359375, |
| "learning_rate": 4.776632302405499e-07, |
| "loss": 0.0024, |
| "reward": 2.359375, |
| "reward_std": 0.29422858636826277, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.4765625, |
| "rewards/format_reward": 0.99609375, |
| "step": 152 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.28515625, |
| "epoch": 0.5246463780540077, |
| "grad_norm": 2.0806228304348635, |
| "kl": 0.0726318359375, |
| "learning_rate": 4.742268041237113e-07, |
| "loss": 0.0029, |
| "reward": 2.296875, |
| "reward_std": 0.23042033333331347, |
| "rewards/accuracy_reward_action": 0.9296875, |
| "rewards/accuracy_reward_coord": 0.37109375, |
| "rewards/format_reward": 0.99609375, |
| "step": 153 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.9765625, |
| "epoch": 0.5280754393484783, |
| "grad_norm": 4.3345166524028595, |
| "kl": 0.07025146484375, |
| "learning_rate": 4.707903780068728e-07, |
| "loss": 0.0028, |
| "reward": 2.16015625, |
| "reward_std": 0.35637666657567024, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.265625, |
| "rewards/format_reward": 0.98828125, |
| "step": 154 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.31640625, |
| "epoch": 0.531504500642949, |
| "grad_norm": 7.707571860540067, |
| "kl": 0.061309814453125, |
| "learning_rate": 4.6735395189003437e-07, |
| "loss": 0.0025, |
| "reward": 2.23828125, |
| "reward_std": 0.18736864533275366, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 1.0, |
| "step": 155 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.5859375, |
| "epoch": 0.5349335619374196, |
| "grad_norm": 2.1418912033584734, |
| "kl": 0.0621337890625, |
| "learning_rate": 4.639175257731959e-07, |
| "loss": 0.0025, |
| "reward": 2.17578125, |
| "reward_std": 0.22305511962622404, |
| "rewards/accuracy_reward_action": 0.83984375, |
| "rewards/accuracy_reward_coord": 0.33984375, |
| "rewards/format_reward": 0.99609375, |
| "step": 156 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.3203125, |
| "epoch": 0.5383626232318903, |
| "grad_norm": 2.42747826627425, |
| "kl": 0.072479248046875, |
| "learning_rate": 4.6048109965635733e-07, |
| "loss": 0.0029, |
| "reward": 2.26953125, |
| "reward_std": 0.3015656163915992, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 157 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 112.7578125, |
| "epoch": 0.541791684526361, |
| "grad_norm": 2.1517223333043876, |
| "kl": 0.0494842529296875, |
| "learning_rate": 4.570446735395189e-07, |
| "loss": 0.002, |
| "reward": 2.28125, |
| "reward_std": 0.3337950073182583, |
| "rewards/accuracy_reward_action": 0.91015625, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 0.99609375, |
| "step": 158 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.859375, |
| "epoch": 0.5452207458208316, |
| "grad_norm": 2.5541435290813603, |
| "kl": 0.055023193359375, |
| "learning_rate": 4.536082474226804e-07, |
| "loss": 0.0022, |
| "reward": 2.26171875, |
| "reward_std": 0.15250477846711874, |
| "rewards/accuracy_reward_action": 0.9296875, |
| "rewards/accuracy_reward_coord": 0.3359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 159 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.9609375, |
| "epoch": 0.5486498071153022, |
| "grad_norm": 4.51678749386207, |
| "kl": 0.0596923828125, |
| "learning_rate": 4.501718213058419e-07, |
| "loss": 0.0024, |
| "reward": 2.21875, |
| "reward_std": 0.31441445648670197, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 160 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.73046875, |
| "epoch": 0.5520788684097728, |
| "grad_norm": 3.0458061345691503, |
| "kl": 0.0517120361328125, |
| "learning_rate": 4.4673539518900345e-07, |
| "loss": 0.0021, |
| "reward": 2.1640625, |
| "reward_std": 0.2575791822746396, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 1.0, |
| "step": 161 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2265625, |
| "epoch": 0.5555079297042435, |
| "grad_norm": 5.855833740137747, |
| "kl": 0.066070556640625, |
| "learning_rate": 4.432989690721649e-07, |
| "loss": 0.0026, |
| "reward": 2.21875, |
| "reward_std": 0.40313977375626564, |
| "rewards/accuracy_reward_action": 0.83203125, |
| "rewards/accuracy_reward_coord": 0.390625, |
| "rewards/format_reward": 0.99609375, |
| "step": 162 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.11328125, |
| "epoch": 0.5589369909987141, |
| "grad_norm": 2.9004827339181425, |
| "kl": 0.065338134765625, |
| "learning_rate": 4.398625429553264e-07, |
| "loss": 0.0026, |
| "reward": 2.36328125, |
| "reward_std": 0.18846283666789532, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.4609375, |
| "rewards/format_reward": 1.0, |
| "step": 163 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.9375, |
| "epoch": 0.5623660522931847, |
| "grad_norm": 2.127861485761365, |
| "kl": 0.0596923828125, |
| "learning_rate": 4.3642611683848796e-07, |
| "loss": 0.0024, |
| "reward": 2.1171875, |
| "reward_std": 0.1504594897851348, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.296875, |
| "rewards/format_reward": 1.0, |
| "step": 164 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.36328125, |
| "epoch": 0.5657951135876553, |
| "grad_norm": 3.131594231201492, |
| "kl": 0.05096435546875, |
| "learning_rate": 4.3298969072164947e-07, |
| "loss": 0.002, |
| "reward": 1.99609375, |
| "reward_std": 0.22178402822464705, |
| "rewards/accuracy_reward_action": 0.77734375, |
| "rewards/accuracy_reward_coord": 0.22265625, |
| "rewards/format_reward": 0.99609375, |
| "step": 165 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.11328125, |
| "epoch": 0.5692241748821261, |
| "grad_norm": 2.6116928502512744, |
| "kl": 0.05963134765625, |
| "learning_rate": 4.2955326460481097e-07, |
| "loss": 0.0024, |
| "reward": 2.3515625, |
| "reward_std": 0.21527426224201918, |
| "rewards/accuracy_reward_action": 0.96484375, |
| "rewards/accuracy_reward_coord": 0.38671875, |
| "rewards/format_reward": 1.0, |
| "step": 166 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.90625, |
| "epoch": 0.5726532361765967, |
| "grad_norm": 2.8869141614124287, |
| "kl": 0.055877685546875, |
| "learning_rate": 4.261168384879725e-07, |
| "loss": 0.0022, |
| "reward": 2.0, |
| "reward_std": 0.24724038317799568, |
| "rewards/accuracy_reward_action": 0.78515625, |
| "rewards/accuracy_reward_coord": 0.21875, |
| "rewards/format_reward": 0.99609375, |
| "step": 167 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.80078125, |
| "epoch": 0.5760822974710673, |
| "grad_norm": 2.649000966370819, |
| "kl": 0.05657958984375, |
| "learning_rate": 4.22680412371134e-07, |
| "loss": 0.0023, |
| "reward": 2.1875, |
| "reward_std": 0.23751860670745373, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.3046875, |
| "rewards/format_reward": 0.99609375, |
| "step": 168 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.24609375, |
| "epoch": 0.5795113587655379, |
| "grad_norm": 3.4059941551394037, |
| "kl": 0.0653076171875, |
| "learning_rate": 4.1924398625429554e-07, |
| "loss": 0.0026, |
| "reward": 2.171875, |
| "reward_std": 0.14085884485393763, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.29296875, |
| "rewards/format_reward": 1.0, |
| "step": 169 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.3203125, |
| "epoch": 0.5829404200600086, |
| "grad_norm": 1.824844597127134, |
| "kl": 0.05792236328125, |
| "learning_rate": 4.1580756013745704e-07, |
| "loss": 0.0023, |
| "reward": 2.09765625, |
| "reward_std": 0.17149819899350405, |
| "rewards/accuracy_reward_action": 0.7578125, |
| "rewards/accuracy_reward_coord": 0.34765625, |
| "rewards/format_reward": 0.9921875, |
| "step": 170 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.3984375, |
| "epoch": 0.5863694813544792, |
| "grad_norm": 2.187167981523429, |
| "kl": 0.0566253662109375, |
| "learning_rate": 4.123711340206185e-07, |
| "loss": 0.0023, |
| "reward": 2.203125, |
| "reward_std": 0.303788922727108, |
| "rewards/accuracy_reward_action": 0.76171875, |
| "rewards/accuracy_reward_coord": 0.44140625, |
| "rewards/format_reward": 1.0, |
| "step": 171 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.2890625, |
| "epoch": 0.5897985426489498, |
| "grad_norm": 4.038396242056258, |
| "kl": 0.0596923828125, |
| "learning_rate": 4.0893470790378005e-07, |
| "loss": 0.0024, |
| "reward": 2.296875, |
| "reward_std": 0.2917390838265419, |
| "rewards/accuracy_reward_action": 0.93359375, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 1.0, |
| "step": 172 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.3671875, |
| "epoch": 0.5932276039434204, |
| "grad_norm": 17.79728215989008, |
| "kl": 0.062255859375, |
| "learning_rate": 4.0549828178694155e-07, |
| "loss": 0.0025, |
| "reward": 2.265625, |
| "reward_std": 0.2565556038171053, |
| "rewards/accuracy_reward_action": 0.95703125, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 1.0, |
| "step": 173 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.75, |
| "epoch": 0.5966566652378912, |
| "grad_norm": 3.1203510157697134, |
| "kl": 0.062957763671875, |
| "learning_rate": 4.020618556701031e-07, |
| "loss": 0.0025, |
| "reward": 2.00390625, |
| "reward_std": 0.20635129511356354, |
| "rewards/accuracy_reward_action": 0.76953125, |
| "rewards/accuracy_reward_coord": 0.234375, |
| "rewards/format_reward": 1.0, |
| "step": 174 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.6328125, |
| "epoch": 0.6000857265323618, |
| "grad_norm": 3.6965883400356403, |
| "kl": 0.069488525390625, |
| "learning_rate": 3.986254295532646e-07, |
| "loss": 0.0028, |
| "reward": 2.21875, |
| "reward_std": 0.24308231100440025, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.390625, |
| "rewards/format_reward": 1.0, |
| "step": 175 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.3203125, |
| "epoch": 0.6035147878268324, |
| "grad_norm": 4.26935936224658, |
| "kl": 0.0628662109375, |
| "learning_rate": 3.9518900343642607e-07, |
| "loss": 0.0025, |
| "reward": 2.23828125, |
| "reward_std": 0.2842397876083851, |
| "rewards/accuracy_reward_action": 0.90234375, |
| "rewards/accuracy_reward_coord": 0.3359375, |
| "rewards/format_reward": 1.0, |
| "step": 176 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.75390625, |
| "epoch": 0.6069438491213031, |
| "grad_norm": 2.4715106694078286, |
| "kl": 0.0633544921875, |
| "learning_rate": 3.917525773195876e-07, |
| "loss": 0.0025, |
| "reward": 2.21875, |
| "reward_std": 0.25477858539670706, |
| "rewards/accuracy_reward_action": 0.87109375, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 177 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.7734375, |
| "epoch": 0.6103729104157737, |
| "grad_norm": 3.8819744356958306, |
| "kl": 0.064788818359375, |
| "learning_rate": 3.8831615120274913e-07, |
| "loss": 0.0026, |
| "reward": 2.3515625, |
| "reward_std": 0.25167326908558607, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.4296875, |
| "rewards/format_reward": 1.0, |
| "step": 178 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.6875, |
| "epoch": 0.6138019717102443, |
| "grad_norm": 2.0131016192778795, |
| "kl": 0.0460205078125, |
| "learning_rate": 3.8487972508591063e-07, |
| "loss": 0.0018, |
| "reward": 2.1796875, |
| "reward_std": 0.3403683602809906, |
| "rewards/accuracy_reward_action": 0.80078125, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 0.99609375, |
| "step": 179 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.91015625, |
| "epoch": 0.6172310330047149, |
| "grad_norm": 2.40863521885105, |
| "kl": 0.063385009765625, |
| "learning_rate": 3.8144329896907214e-07, |
| "loss": 0.0025, |
| "reward": 2.28125, |
| "reward_std": 0.230240011587739, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.40234375, |
| "rewards/format_reward": 0.99609375, |
| "step": 180 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.58203125, |
| "epoch": 0.6206600942991856, |
| "grad_norm": 3.1847190717179026, |
| "kl": 0.056396484375, |
| "learning_rate": 3.7800687285223364e-07, |
| "loss": 0.0023, |
| "reward": 2.1328125, |
| "reward_std": 0.3664143308997154, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 0.99609375, |
| "step": 181 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.5078125, |
| "epoch": 0.6240891555936562, |
| "grad_norm": 3.3211334385699303, |
| "kl": 0.05389404296875, |
| "learning_rate": 3.745704467353952e-07, |
| "loss": 0.0022, |
| "reward": 2.25, |
| "reward_std": 0.2401421070098877, |
| "rewards/accuracy_reward_action": 0.83984375, |
| "rewards/accuracy_reward_coord": 0.41015625, |
| "rewards/format_reward": 1.0, |
| "step": 182 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.125, |
| "epoch": 0.6275182168881269, |
| "grad_norm": 3.704018941036333, |
| "kl": 0.065277099609375, |
| "learning_rate": 3.711340206185567e-07, |
| "loss": 0.0026, |
| "reward": 2.1875, |
| "reward_std": 0.1892604036256671, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 1.0, |
| "step": 183 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.8125, |
| "epoch": 0.6309472781825975, |
| "grad_norm": 5.766622845742471, |
| "kl": 0.063873291015625, |
| "learning_rate": 3.676975945017182e-07, |
| "loss": 0.0026, |
| "reward": 2.37109375, |
| "reward_std": 0.26052681915462017, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.44921875, |
| "rewards/format_reward": 0.99609375, |
| "step": 184 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.64453125, |
| "epoch": 0.6343763394770682, |
| "grad_norm": 12.693794819646222, |
| "kl": 0.05938720703125, |
| "learning_rate": 3.642611683848797e-07, |
| "loss": 0.0024, |
| "reward": 2.37890625, |
| "reward_std": 0.28175163455307484, |
| "rewards/accuracy_reward_action": 0.9453125, |
| "rewards/accuracy_reward_coord": 0.4375, |
| "rewards/format_reward": 0.99609375, |
| "step": 185 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.68359375, |
| "epoch": 0.6378054007715388, |
| "grad_norm": 7.235548008451321, |
| "kl": 0.060546875, |
| "learning_rate": 3.608247422680412e-07, |
| "loss": 0.0024, |
| "reward": 2.1171875, |
| "reward_std": 0.28209348395466805, |
| "rewards/accuracy_reward_action": 0.796875, |
| "rewards/accuracy_reward_coord": 0.3203125, |
| "rewards/format_reward": 1.0, |
| "step": 186 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.96484375, |
| "epoch": 0.6412344620660094, |
| "grad_norm": 7.687215435921219, |
| "kl": 0.065704345703125, |
| "learning_rate": 3.573883161512027e-07, |
| "loss": 0.0026, |
| "reward": 2.1484375, |
| "reward_std": 0.1814600545912981, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.24609375, |
| "rewards/format_reward": 0.99609375, |
| "step": 187 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.16015625, |
| "epoch": 0.64466352336048, |
| "grad_norm": 3.825184151511829, |
| "kl": 0.058258056640625, |
| "learning_rate": 3.539518900343643e-07, |
| "loss": 0.0023, |
| "reward": 2.3984375, |
| "reward_std": 0.2644512262195349, |
| "rewards/accuracy_reward_action": 0.96875, |
| "rewards/accuracy_reward_coord": 0.4296875, |
| "rewards/format_reward": 1.0, |
| "step": 188 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.3828125, |
| "epoch": 0.6480925846549507, |
| "grad_norm": 6.231341414590355, |
| "kl": 0.06439208984375, |
| "learning_rate": 3.5051546391752573e-07, |
| "loss": 0.0026, |
| "reward": 2.3125, |
| "reward_std": 0.19818973448127508, |
| "rewards/accuracy_reward_action": 0.953125, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 189 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.7421875, |
| "epoch": 0.6515216459494213, |
| "grad_norm": 2.6776596064402503, |
| "kl": 0.064849853515625, |
| "learning_rate": 3.470790378006873e-07, |
| "loss": 0.0026, |
| "reward": 2.265625, |
| "reward_std": 0.2254919307306409, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.33984375, |
| "rewards/format_reward": 1.0, |
| "step": 190 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.80078125, |
| "epoch": 0.654950707243892, |
| "grad_norm": 2.4403487522323797, |
| "kl": 0.060577392578125, |
| "learning_rate": 3.436426116838488e-07, |
| "loss": 0.0024, |
| "reward": 2.35546875, |
| "reward_std": 0.25711186695843935, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.43359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 191 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.50390625, |
| "epoch": 0.6583797685383627, |
| "grad_norm": 2.726606434781571, |
| "kl": 0.057037353515625, |
| "learning_rate": 3.402061855670103e-07, |
| "loss": 0.0023, |
| "reward": 2.10546875, |
| "reward_std": 0.2689245492219925, |
| "rewards/accuracy_reward_action": 0.75, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 192 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.4375, |
| "epoch": 0.6618088298328333, |
| "grad_norm": 2.2014192461202593, |
| "kl": 0.059173583984375, |
| "learning_rate": 3.3676975945017185e-07, |
| "loss": 0.0024, |
| "reward": 2.16015625, |
| "reward_std": 0.1743851751089096, |
| "rewards/accuracy_reward_action": 0.7734375, |
| "rewards/accuracy_reward_coord": 0.38671875, |
| "rewards/format_reward": 1.0, |
| "step": 193 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.65234375, |
| "epoch": 0.6652378911273039, |
| "grad_norm": 2.6193872685904025, |
| "kl": 0.057861328125, |
| "learning_rate": 3.333333333333333e-07, |
| "loss": 0.0023, |
| "reward": 2.37109375, |
| "reward_std": 0.3137936666607857, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.484375, |
| "rewards/format_reward": 0.9921875, |
| "step": 194 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.796875, |
| "epoch": 0.6686669524217745, |
| "grad_norm": 2.685086268578767, |
| "kl": 0.066864013671875, |
| "learning_rate": 3.298969072164948e-07, |
| "loss": 0.0027, |
| "reward": 2.23046875, |
| "reward_std": 0.2271191542968154, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 1.0, |
| "step": 195 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.04296875, |
| "epoch": 0.6720960137162452, |
| "grad_norm": 3.271710525721659, |
| "kl": 0.06494140625, |
| "learning_rate": 3.2646048109965636e-07, |
| "loss": 0.0026, |
| "reward": 2.0625, |
| "reward_std": 0.32577752880752087, |
| "rewards/accuracy_reward_action": 0.828125, |
| "rewards/accuracy_reward_coord": 0.23828125, |
| "rewards/format_reward": 0.99609375, |
| "step": 196 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 113.484375, |
| "epoch": 0.6755250750107158, |
| "grad_norm": 3.36795886600386, |
| "kl": 0.0592803955078125, |
| "learning_rate": 3.2302405498281787e-07, |
| "loss": 0.0024, |
| "reward": 2.3046875, |
| "reward_std": 0.21713144704699516, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 1.0, |
| "step": 197 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.31640625, |
| "epoch": 0.6789541363051864, |
| "grad_norm": 4.155750547891662, |
| "kl": 0.05615234375, |
| "learning_rate": 3.195876288659793e-07, |
| "loss": 0.0022, |
| "reward": 2.3984375, |
| "reward_std": 0.317256017588079, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.5078125, |
| "rewards/format_reward": 1.0, |
| "step": 198 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.3515625, |
| "epoch": 0.682383197599657, |
| "grad_norm": 2.2382157235308853, |
| "kl": 0.05859375, |
| "learning_rate": 3.161512027491409e-07, |
| "loss": 0.0023, |
| "reward": 2.1484375, |
| "reward_std": 0.26314294897019863, |
| "rewards/accuracy_reward_action": 0.7890625, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 199 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.2734375, |
| "epoch": 0.6858122588941278, |
| "grad_norm": 2.254961340187696, |
| "kl": 0.05914306640625, |
| "learning_rate": 3.127147766323024e-07, |
| "loss": 0.0024, |
| "reward": 2.24609375, |
| "reward_std": 0.16557206399738789, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 0.9921875, |
| "step": 200 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.82421875, |
| "epoch": 0.6892413201885984, |
| "grad_norm": 2.7434719866808837, |
| "kl": 0.05810546875, |
| "learning_rate": 3.0927835051546394e-07, |
| "loss": 0.0023, |
| "reward": 2.234375, |
| "reward_std": 0.22423463128507137, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 1.0, |
| "step": 201 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.7578125, |
| "epoch": 0.692670381483069, |
| "grad_norm": 2.382655775119527, |
| "kl": 0.059539794921875, |
| "learning_rate": 3.0584192439862544e-07, |
| "loss": 0.0024, |
| "reward": 2.3125, |
| "reward_std": 0.34181051328778267, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.390625, |
| "rewards/format_reward": 1.0, |
| "step": 202 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.80859375, |
| "epoch": 0.6960994427775397, |
| "grad_norm": 11.014895178134898, |
| "kl": 0.065826416015625, |
| "learning_rate": 3.024054982817869e-07, |
| "loss": 0.0026, |
| "reward": 2.2265625, |
| "reward_std": 0.19931845646351576, |
| "rewards/accuracy_reward_action": 0.89453125, |
| "rewards/accuracy_reward_coord": 0.33984375, |
| "rewards/format_reward": 0.9921875, |
| "step": 203 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 111.80859375, |
| "epoch": 0.6995285040720103, |
| "grad_norm": 5.45051374488145, |
| "kl": 0.0550994873046875, |
| "learning_rate": 2.9896907216494845e-07, |
| "loss": 0.0022, |
| "reward": 2.17578125, |
| "reward_std": 0.2756984196603298, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.98046875, |
| "step": 204 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.12890625, |
| "epoch": 0.7029575653664809, |
| "grad_norm": 2.5795644690128547, |
| "kl": 0.0765380859375, |
| "learning_rate": 2.9553264604810995e-07, |
| "loss": 0.0031, |
| "reward": 2.3359375, |
| "reward_std": 0.2355452049523592, |
| "rewards/accuracy_reward_action": 0.94140625, |
| "rewards/accuracy_reward_coord": 0.39453125, |
| "rewards/format_reward": 1.0, |
| "step": 205 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.05859375, |
| "epoch": 0.7063866266609515, |
| "grad_norm": 1.9623795208382564, |
| "kl": 0.0682373046875, |
| "learning_rate": 2.9209621993127146e-07, |
| "loss": 0.0027, |
| "reward": 2.41796875, |
| "reward_std": 0.22526716440916061, |
| "rewards/accuracy_reward_action": 0.94140625, |
| "rewards/accuracy_reward_coord": 0.48046875, |
| "rewards/format_reward": 0.99609375, |
| "step": 206 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.40625, |
| "epoch": 0.7098156879554222, |
| "grad_norm": 1.3987661010367967, |
| "kl": 0.05712890625, |
| "learning_rate": 2.8865979381443296e-07, |
| "loss": 0.0023, |
| "reward": 2.203125, |
| "reward_std": 0.18596168421208858, |
| "rewards/accuracy_reward_action": 0.85546875, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 0.9921875, |
| "step": 207 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.296875, |
| "epoch": 0.7132447492498929, |
| "grad_norm": 2.5454111949134366, |
| "kl": 0.0555877685546875, |
| "learning_rate": 2.8522336769759446e-07, |
| "loss": 0.0022, |
| "reward": 2.2109375, |
| "reward_std": 0.2109950603917241, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.36328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 208 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.93359375, |
| "epoch": 0.7166738105443635, |
| "grad_norm": 4.680608047371802, |
| "kl": 0.061920166015625, |
| "learning_rate": 2.81786941580756e-07, |
| "loss": 0.0025, |
| "reward": 2.12109375, |
| "reward_std": 0.19135739468038082, |
| "rewards/accuracy_reward_action": 0.7734375, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 209 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 110.9453125, |
| "epoch": 0.7201028718388341, |
| "grad_norm": 3.8695417245183217, |
| "kl": 0.05340576171875, |
| "learning_rate": 2.783505154639175e-07, |
| "loss": 0.0021, |
| "reward": 2.26953125, |
| "reward_std": 0.3266449635848403, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 1.0, |
| "step": 210 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.078125, |
| "epoch": 0.7235319331333048, |
| "grad_norm": 7.186477052295602, |
| "kl": 0.0621337890625, |
| "learning_rate": 2.7491408934707903e-07, |
| "loss": 0.0025, |
| "reward": 2.18359375, |
| "reward_std": 0.2528016325086355, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.3203125, |
| "rewards/format_reward": 1.0, |
| "step": 211 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.28125, |
| "epoch": 0.7269609944277754, |
| "grad_norm": 2.0631486565417867, |
| "kl": 0.060333251953125, |
| "learning_rate": 2.7147766323024053e-07, |
| "loss": 0.0024, |
| "reward": 2.26171875, |
| "reward_std": 0.21227251924574375, |
| "rewards/accuracy_reward_action": 0.83203125, |
| "rewards/accuracy_reward_coord": 0.4296875, |
| "rewards/format_reward": 1.0, |
| "step": 212 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 107.47265625, |
| "epoch": 0.730390055722246, |
| "grad_norm": 2.3982126246518214, |
| "kl": 0.059661865234375, |
| "learning_rate": 2.6804123711340204e-07, |
| "loss": 0.0024, |
| "reward": 2.375, |
| "reward_std": 0.22252471186220646, |
| "rewards/accuracy_reward_action": 0.98828125, |
| "rewards/accuracy_reward_coord": 0.38671875, |
| "rewards/format_reward": 1.0, |
| "step": 213 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.74609375, |
| "epoch": 0.7338191170167166, |
| "grad_norm": 7.053341803393176, |
| "kl": 0.0509033203125, |
| "learning_rate": 2.6460481099656354e-07, |
| "loss": 0.002, |
| "reward": 2.1875, |
| "reward_std": 0.2063190806657076, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.32421875, |
| "rewards/format_reward": 1.0, |
| "step": 214 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.19140625, |
| "epoch": 0.7372481783111873, |
| "grad_norm": 6.251749314545945, |
| "kl": 0.06671142578125, |
| "learning_rate": 2.611683848797251e-07, |
| "loss": 0.0027, |
| "reward": 2.203125, |
| "reward_std": 0.18767161015421152, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 1.0, |
| "step": 215 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.71484375, |
| "epoch": 0.7406772396056579, |
| "grad_norm": 1.5875978023666912, |
| "kl": 0.066802978515625, |
| "learning_rate": 2.5773195876288655e-07, |
| "loss": 0.0027, |
| "reward": 2.40625, |
| "reward_std": 0.2597773177549243, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.55078125, |
| "rewards/format_reward": 0.99609375, |
| "step": 216 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 109.0859375, |
| "epoch": 0.7441063009001286, |
| "grad_norm": 2.200432153622085, |
| "kl": 0.058349609375, |
| "learning_rate": 2.542955326460481e-07, |
| "loss": 0.0023, |
| "reward": 2.21484375, |
| "reward_std": 0.17895132210105658, |
| "rewards/accuracy_reward_action": 0.8984375, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 1.0, |
| "step": 217 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.2890625, |
| "epoch": 0.7475353621945993, |
| "grad_norm": 1.8948662873298148, |
| "kl": 0.05731201171875, |
| "learning_rate": 2.508591065292096e-07, |
| "loss": 0.0023, |
| "reward": 2.02734375, |
| "reward_std": 0.2757891062647104, |
| "rewards/accuracy_reward_action": 0.77734375, |
| "rewards/accuracy_reward_coord": 0.2578125, |
| "rewards/format_reward": 0.9921875, |
| "step": 218 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 106.51953125, |
| "epoch": 0.7509644234890699, |
| "grad_norm": 3.4206291883450235, |
| "kl": 0.06536865234375, |
| "learning_rate": 2.474226804123711e-07, |
| "loss": 0.0026, |
| "reward": 2.24609375, |
| "reward_std": 0.245152210816741, |
| "rewards/accuracy_reward_action": 0.9375, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 0.99609375, |
| "step": 219 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.96875, |
| "epoch": 0.7543934847835405, |
| "grad_norm": 2.1536521583783585, |
| "kl": 0.0999755859375, |
| "learning_rate": 2.439862542955326e-07, |
| "loss": 0.004, |
| "reward": 2.30859375, |
| "reward_std": 0.18533831275999546, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.453125, |
| "rewards/format_reward": 0.99609375, |
| "step": 220 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.06640625, |
| "epoch": 0.7578225460780111, |
| "grad_norm": 3.5470879643297595, |
| "kl": 0.06903076171875, |
| "learning_rate": 2.405498281786942e-07, |
| "loss": 0.0028, |
| "reward": 2.17578125, |
| "reward_std": 0.22653949353843927, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.26171875, |
| "rewards/format_reward": 1.0, |
| "step": 221 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.58984375, |
| "epoch": 0.7612516073724818, |
| "grad_norm": 1.939359428181939, |
| "kl": 0.063720703125, |
| "learning_rate": 2.3711340206185566e-07, |
| "loss": 0.0025, |
| "reward": 2.375, |
| "reward_std": 0.2126298239454627, |
| "rewards/accuracy_reward_action": 0.97265625, |
| "rewards/accuracy_reward_coord": 0.40625, |
| "rewards/format_reward": 0.99609375, |
| "step": 222 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.68359375, |
| "epoch": 0.7646806686669524, |
| "grad_norm": 2.675302553850158, |
| "kl": 0.058349609375, |
| "learning_rate": 2.3367697594501719e-07, |
| "loss": 0.0023, |
| "reward": 2.40234375, |
| "reward_std": 0.19962881412357092, |
| "rewards/accuracy_reward_action": 0.94140625, |
| "rewards/accuracy_reward_coord": 0.47265625, |
| "rewards/format_reward": 0.98828125, |
| "step": 223 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.3671875, |
| "epoch": 0.768109729961423, |
| "grad_norm": 1.8156057304901947, |
| "kl": 0.06451416015625, |
| "learning_rate": 2.3024054982817866e-07, |
| "loss": 0.0026, |
| "reward": 2.21484375, |
| "reward_std": 0.2694152584299445, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.39453125, |
| "rewards/format_reward": 1.0, |
| "step": 224 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.8828125, |
| "epoch": 0.7715387912558938, |
| "grad_norm": 2.292898497856771, |
| "kl": 0.06561279296875, |
| "learning_rate": 2.268041237113402e-07, |
| "loss": 0.0026, |
| "reward": 2.30078125, |
| "reward_std": 0.24273008853197098, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.4140625, |
| "rewards/format_reward": 0.99609375, |
| "step": 225 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.65625, |
| "epoch": 0.7749678525503644, |
| "grad_norm": 2.4049932524471895, |
| "kl": 0.0560302734375, |
| "learning_rate": 2.2336769759450173e-07, |
| "loss": 0.0022, |
| "reward": 2.2265625, |
| "reward_std": 0.18312525562942028, |
| "rewards/accuracy_reward_action": 0.91796875, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 0.99609375, |
| "step": 226 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.26953125, |
| "epoch": 0.778396913844835, |
| "grad_norm": 3.0510228384132336, |
| "kl": 0.067291259765625, |
| "learning_rate": 2.199312714776632e-07, |
| "loss": 0.0027, |
| "reward": 2.26953125, |
| "reward_std": 0.29204913042485714, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.3828125, |
| "rewards/format_reward": 1.0, |
| "step": 227 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 105.13671875, |
| "epoch": 0.7818259751393056, |
| "grad_norm": 2.801134477925018, |
| "kl": 0.064300537109375, |
| "learning_rate": 2.1649484536082473e-07, |
| "loss": 0.0026, |
| "reward": 2.1875, |
| "reward_std": 0.2845958499237895, |
| "rewards/accuracy_reward_action": 0.87109375, |
| "rewards/accuracy_reward_coord": 0.3203125, |
| "rewards/format_reward": 0.99609375, |
| "step": 228 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.6015625, |
| "epoch": 0.7852550364337763, |
| "grad_norm": 1.5238507681242188, |
| "kl": 0.061279296875, |
| "learning_rate": 2.1305841924398624e-07, |
| "loss": 0.0024, |
| "reward": 2.296875, |
| "reward_std": 0.19438526965677738, |
| "rewards/accuracy_reward_action": 0.94921875, |
| "rewards/accuracy_reward_coord": 0.34765625, |
| "rewards/format_reward": 1.0, |
| "step": 229 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.84375, |
| "epoch": 0.7886840977282469, |
| "grad_norm": 3.2174810156336737, |
| "kl": 0.0709228515625, |
| "learning_rate": 2.0962199312714777e-07, |
| "loss": 0.0028, |
| "reward": 2.27734375, |
| "reward_std": 0.21200903225690126, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.39453125, |
| "rewards/format_reward": 0.99609375, |
| "step": 230 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.6953125, |
| "epoch": 0.7921131590227175, |
| "grad_norm": 3.438022774954148, |
| "kl": 0.080291748046875, |
| "learning_rate": 2.0618556701030925e-07, |
| "loss": 0.0032, |
| "reward": 2.1953125, |
| "reward_std": 0.2185729807242751, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 1.0, |
| "step": 231 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.8125, |
| "epoch": 0.7955422203171881, |
| "grad_norm": 4.55498504791608, |
| "kl": 0.055389404296875, |
| "learning_rate": 2.0274914089347078e-07, |
| "loss": 0.0022, |
| "reward": 2.09375, |
| "reward_std": 0.12073516938835382, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.28125, |
| "rewards/format_reward": 1.0, |
| "step": 232 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 108.6953125, |
| "epoch": 0.7989712816116588, |
| "grad_norm": 9.204589267475633, |
| "kl": 0.059478759765625, |
| "learning_rate": 1.993127147766323e-07, |
| "loss": 0.0024, |
| "reward": 2.1015625, |
| "reward_std": 0.21872024424374104, |
| "rewards/accuracy_reward_action": 0.76953125, |
| "rewards/accuracy_reward_coord": 0.3359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 233 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 94.46484375, |
| "epoch": 0.8024003429061295, |
| "grad_norm": 3.1838861993641356, |
| "kl": 0.061279296875, |
| "learning_rate": 1.958762886597938e-07, |
| "loss": 0.0025, |
| "reward": 2.3828125, |
| "reward_std": 0.26823191065341234, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.515625, |
| "rewards/format_reward": 1.0, |
| "step": 234 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.81640625, |
| "epoch": 0.8058294042006001, |
| "grad_norm": 5.022920175105425, |
| "kl": 0.080108642578125, |
| "learning_rate": 1.9243986254295532e-07, |
| "loss": 0.0032, |
| "reward": 2.19140625, |
| "reward_std": 0.2484198957681656, |
| "rewards/accuracy_reward_action": 0.91796875, |
| "rewards/accuracy_reward_coord": 0.2734375, |
| "rewards/format_reward": 1.0, |
| "step": 235 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.625, |
| "epoch": 0.8092584654950707, |
| "grad_norm": 3.6452089439099224, |
| "kl": 0.069732666015625, |
| "learning_rate": 1.8900343642611682e-07, |
| "loss": 0.0028, |
| "reward": 2.3515625, |
| "reward_std": 0.23472690861672163, |
| "rewards/accuracy_reward_action": 0.953125, |
| "rewards/accuracy_reward_coord": 0.40234375, |
| "rewards/format_reward": 0.99609375, |
| "step": 236 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.62109375, |
| "epoch": 0.8126875267895414, |
| "grad_norm": 2.7630581789606308, |
| "kl": 0.073883056640625, |
| "learning_rate": 1.8556701030927835e-07, |
| "loss": 0.003, |
| "reward": 2.1640625, |
| "reward_std": 0.200991983525455, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.3203125, |
| "rewards/format_reward": 1.0, |
| "step": 237 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 93.6484375, |
| "epoch": 0.816116588084012, |
| "grad_norm": 3.4936750980868094, |
| "kl": 0.0870361328125, |
| "learning_rate": 1.8213058419243986e-07, |
| "loss": 0.0035, |
| "reward": 2.44921875, |
| "reward_std": 0.22026342526078224, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.54296875, |
| "rewards/format_reward": 1.0, |
| "step": 238 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.6953125, |
| "epoch": 0.8195456493784826, |
| "grad_norm": 2.4875655561955297, |
| "kl": 0.07073974609375, |
| "learning_rate": 1.7869415807560136e-07, |
| "loss": 0.0028, |
| "reward": 2.48046875, |
| "reward_std": 0.26424654945731163, |
| "rewards/accuracy_reward_action": 0.94921875, |
| "rewards/accuracy_reward_coord": 0.53125, |
| "rewards/format_reward": 1.0, |
| "step": 239 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.20703125, |
| "epoch": 0.8229747106729532, |
| "grad_norm": 1.7425268860067689, |
| "kl": 0.066192626953125, |
| "learning_rate": 1.7525773195876286e-07, |
| "loss": 0.0027, |
| "reward": 2.125, |
| "reward_std": 0.1960951965302229, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.30078125, |
| "rewards/format_reward": 1.0, |
| "step": 240 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.17578125, |
| "epoch": 0.8264037719674239, |
| "grad_norm": 2.230888311358435, |
| "kl": 0.057891845703125, |
| "learning_rate": 1.718213058419244e-07, |
| "loss": 0.0023, |
| "reward": 2.1640625, |
| "reward_std": 0.23385854996740818, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 0.9921875, |
| "step": 241 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.64453125, |
| "epoch": 0.8298328332618946, |
| "grad_norm": 2.799450768259519, |
| "kl": 0.073486328125, |
| "learning_rate": 1.6838487972508592e-07, |
| "loss": 0.0029, |
| "reward": 2.1640625, |
| "reward_std": 0.14768873061984777, |
| "rewards/accuracy_reward_action": 0.8828125, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 0.99609375, |
| "step": 242 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.8828125, |
| "epoch": 0.8332618945563652, |
| "grad_norm": 2.4868444484547996, |
| "kl": 0.076263427734375, |
| "learning_rate": 1.649484536082474e-07, |
| "loss": 0.0031, |
| "reward": 2.26171875, |
| "reward_std": 0.2543821521103382, |
| "rewards/accuracy_reward_action": 0.8984375, |
| "rewards/accuracy_reward_coord": 0.3671875, |
| "rewards/format_reward": 0.99609375, |
| "step": 243 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.5234375, |
| "epoch": 0.8366909558508359, |
| "grad_norm": 1.6923875186009838, |
| "kl": 0.0662841796875, |
| "learning_rate": 1.6151202749140893e-07, |
| "loss": 0.0026, |
| "reward": 2.234375, |
| "reward_std": 0.17942285630851984, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 1.0, |
| "step": 244 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.28515625, |
| "epoch": 0.8401200171453065, |
| "grad_norm": 2.5018601650619496, |
| "kl": 0.0675048828125, |
| "learning_rate": 1.5807560137457044e-07, |
| "loss": 0.0027, |
| "reward": 2.19140625, |
| "reward_std": 0.18885111529380083, |
| "rewards/accuracy_reward_action": 0.88671875, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.99609375, |
| "step": 245 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.0390625, |
| "epoch": 0.8435490784397771, |
| "grad_norm": 3.1069582033856453, |
| "kl": 0.061248779296875, |
| "learning_rate": 1.5463917525773197e-07, |
| "loss": 0.0025, |
| "reward": 2.18359375, |
| "reward_std": 0.274249623529613, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.37109375, |
| "rewards/format_reward": 1.0, |
| "step": 246 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.8671875, |
| "epoch": 0.8469781397342477, |
| "grad_norm": 4.533280248014449, |
| "kl": 0.0634765625, |
| "learning_rate": 1.5120274914089345e-07, |
| "loss": 0.0025, |
| "reward": 2.1875, |
| "reward_std": 0.20642524771392345, |
| "rewards/accuracy_reward_action": 0.8203125, |
| "rewards/accuracy_reward_coord": 0.37109375, |
| "rewards/format_reward": 0.99609375, |
| "step": 247 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.51171875, |
| "epoch": 0.8504072010287184, |
| "grad_norm": 4.186784641550275, |
| "kl": 0.068511962890625, |
| "learning_rate": 1.4776632302405498e-07, |
| "loss": 0.0027, |
| "reward": 2.1640625, |
| "reward_std": 0.29810838773846626, |
| "rewards/accuracy_reward_action": 0.82421875, |
| "rewards/accuracy_reward_coord": 0.34375, |
| "rewards/format_reward": 0.99609375, |
| "step": 248 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.25, |
| "epoch": 0.853836262323189, |
| "grad_norm": 3.510648421329416, |
| "kl": 0.061370849609375, |
| "learning_rate": 1.4432989690721648e-07, |
| "loss": 0.0025, |
| "reward": 2.2734375, |
| "reward_std": 0.15110857971012592, |
| "rewards/accuracy_reward_action": 0.9140625, |
| "rewards/accuracy_reward_coord": 0.359375, |
| "rewards/format_reward": 1.0, |
| "step": 249 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.171875, |
| "epoch": 0.8572653236176596, |
| "grad_norm": 2.033902237284848, |
| "kl": 0.07513427734375, |
| "learning_rate": 1.40893470790378e-07, |
| "loss": 0.003, |
| "reward": 2.33984375, |
| "reward_std": 0.2207325054332614, |
| "rewards/accuracy_reward_action": 0.8984375, |
| "rewards/accuracy_reward_coord": 0.44140625, |
| "rewards/format_reward": 1.0, |
| "step": 250 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.89453125, |
| "epoch": 0.8606943849121304, |
| "grad_norm": 17.844554886648854, |
| "kl": 0.065521240234375, |
| "learning_rate": 1.3745704467353952e-07, |
| "loss": 0.0026, |
| "reward": 2.1328125, |
| "reward_std": 0.2432925645262003, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.296875, |
| "rewards/format_reward": 1.0, |
| "step": 251 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.05078125, |
| "epoch": 0.864123446206601, |
| "grad_norm": 2.564885218079542, |
| "kl": 0.064483642578125, |
| "learning_rate": 1.3402061855670102e-07, |
| "loss": 0.0026, |
| "reward": 2.40625, |
| "reward_std": 0.23802954237908125, |
| "rewards/accuracy_reward_action": 0.921875, |
| "rewards/accuracy_reward_coord": 0.484375, |
| "rewards/format_reward": 1.0, |
| "step": 252 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.4921875, |
| "epoch": 0.8675525075010716, |
| "grad_norm": 3.73127025667996, |
| "kl": 0.055328369140625, |
| "learning_rate": 1.3058419243986255e-07, |
| "loss": 0.0022, |
| "reward": 2.16015625, |
| "reward_std": 0.14309427700936794, |
| "rewards/accuracy_reward_action": 0.91015625, |
| "rewards/accuracy_reward_coord": 0.25, |
| "rewards/format_reward": 1.0, |
| "step": 253 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.7890625, |
| "epoch": 0.8709815687955422, |
| "grad_norm": 3.118867721818899, |
| "kl": 0.072265625, |
| "learning_rate": 1.2714776632302405e-07, |
| "loss": 0.0029, |
| "reward": 2.2578125, |
| "reward_std": 0.2689540944993496, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.3984375, |
| "rewards/format_reward": 1.0, |
| "step": 254 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.6640625, |
| "epoch": 0.8744106300900129, |
| "grad_norm": 11.341014381766739, |
| "kl": 0.060821533203125, |
| "learning_rate": 1.2371134020618556e-07, |
| "loss": 0.0024, |
| "reward": 2.15625, |
| "reward_std": 0.2020435081794858, |
| "rewards/accuracy_reward_action": 0.84765625, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 1.0, |
| "step": 255 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.55859375, |
| "epoch": 0.8778396913844835, |
| "grad_norm": 2.3194171315590846, |
| "kl": 0.063568115234375, |
| "learning_rate": 1.202749140893471e-07, |
| "loss": 0.0025, |
| "reward": 2.12890625, |
| "reward_std": 0.22880323231220245, |
| "rewards/accuracy_reward_action": 0.81640625, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 0.99609375, |
| "step": 256 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.0859375, |
| "epoch": 0.8812687526789541, |
| "grad_norm": 4.2176091245947305, |
| "kl": 0.050262451171875, |
| "learning_rate": 1.1683848797250859e-07, |
| "loss": 0.002, |
| "reward": 2.08203125, |
| "reward_std": 0.2253336338326335, |
| "rewards/accuracy_reward_action": 0.7578125, |
| "rewards/accuracy_reward_coord": 0.328125, |
| "rewards/format_reward": 0.99609375, |
| "step": 257 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.01953125, |
| "epoch": 0.8846978139734247, |
| "grad_norm": 2.207390288446601, |
| "kl": 0.06121826171875, |
| "learning_rate": 1.134020618556701e-07, |
| "loss": 0.0024, |
| "reward": 2.2734375, |
| "reward_std": 0.2755319904536009, |
| "rewards/accuracy_reward_action": 0.84375, |
| "rewards/accuracy_reward_coord": 0.43359375, |
| "rewards/format_reward": 0.99609375, |
| "step": 258 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.47265625, |
| "epoch": 0.8881268752678955, |
| "grad_norm": 2.186357157969937, |
| "kl": 0.0671844482421875, |
| "learning_rate": 1.099656357388316e-07, |
| "loss": 0.0027, |
| "reward": 2.2578125, |
| "reward_std": 0.20067016035318375, |
| "rewards/accuracy_reward_action": 0.890625, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 0.9921875, |
| "step": 259 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.87109375, |
| "epoch": 0.8915559365623661, |
| "grad_norm": 4.422356099392904, |
| "kl": 0.0606689453125, |
| "learning_rate": 1.0652920962199312e-07, |
| "loss": 0.0024, |
| "reward": 2.2109375, |
| "reward_std": 0.2908977773040533, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 1.0, |
| "step": 260 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.69921875, |
| "epoch": 0.8949849978568367, |
| "grad_norm": 56.35961554033066, |
| "kl": 0.0672607421875, |
| "learning_rate": 1.0309278350515462e-07, |
| "loss": 0.0027, |
| "reward": 2.24609375, |
| "reward_std": 0.33419137727469206, |
| "rewards/accuracy_reward_action": 0.8359375, |
| "rewards/accuracy_reward_coord": 0.41015625, |
| "rewards/format_reward": 1.0, |
| "step": 261 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.671875, |
| "epoch": 0.8984140591513073, |
| "grad_norm": 29.13880298224036, |
| "kl": 0.0675048828125, |
| "learning_rate": 9.965635738831615e-08, |
| "loss": 0.0027, |
| "reward": 2.484375, |
| "reward_std": 0.19108655117452145, |
| "rewards/accuracy_reward_action": 0.953125, |
| "rewards/accuracy_reward_coord": 0.53125, |
| "rewards/format_reward": 1.0, |
| "step": 262 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.28125, |
| "epoch": 0.901843120445778, |
| "grad_norm": 2.920667247726801, |
| "kl": 0.06536865234375, |
| "learning_rate": 9.621993127147766e-08, |
| "loss": 0.0026, |
| "reward": 2.265625, |
| "reward_std": 0.23333880677819252, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.39453125, |
| "rewards/format_reward": 0.99609375, |
| "step": 263 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 103.6953125, |
| "epoch": 0.9052721817402486, |
| "grad_norm": 3.260184200475953, |
| "kl": 0.05902099609375, |
| "learning_rate": 9.278350515463918e-08, |
| "loss": 0.0024, |
| "reward": 2.26953125, |
| "reward_std": 0.29997681826353073, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.41796875, |
| "rewards/format_reward": 1.0, |
| "step": 264 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.39453125, |
| "epoch": 0.9087012430347192, |
| "grad_norm": 1.7797660454793223, |
| "kl": 0.06048583984375, |
| "learning_rate": 8.934707903780068e-08, |
| "loss": 0.0024, |
| "reward": 2.3671875, |
| "reward_std": 0.275061103515327, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.4453125, |
| "rewards/format_reward": 0.99609375, |
| "step": 265 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.48828125, |
| "epoch": 0.9121303043291898, |
| "grad_norm": 9.315295958574032, |
| "kl": 0.058807373046875, |
| "learning_rate": 8.59106529209622e-08, |
| "loss": 0.0024, |
| "reward": 2.12890625, |
| "reward_std": 0.22346731834113598, |
| "rewards/accuracy_reward_action": 0.875, |
| "rewards/accuracy_reward_coord": 0.2578125, |
| "rewards/format_reward": 0.99609375, |
| "step": 266 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.91796875, |
| "epoch": 0.9155593656236605, |
| "grad_norm": 1.7100721169087572, |
| "kl": 0.072113037109375, |
| "learning_rate": 8.24742268041237e-08, |
| "loss": 0.0029, |
| "reward": 2.265625, |
| "reward_std": 0.1885802736505866, |
| "rewards/accuracy_reward_action": 0.95703125, |
| "rewards/accuracy_reward_coord": 0.3125, |
| "rewards/format_reward": 0.99609375, |
| "step": 267 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.3671875, |
| "epoch": 0.9189884269181312, |
| "grad_norm": 2.5827467208411803, |
| "kl": 0.07196044921875, |
| "learning_rate": 7.903780068728522e-08, |
| "loss": 0.0029, |
| "reward": 2.2734375, |
| "reward_std": 0.2126556746661663, |
| "rewards/accuracy_reward_action": 0.8671875, |
| "rewards/accuracy_reward_coord": 0.40625, |
| "rewards/format_reward": 1.0, |
| "step": 268 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 104.4296875, |
| "epoch": 0.9224174882126018, |
| "grad_norm": 2.490432791412204, |
| "kl": 0.06195068359375, |
| "learning_rate": 7.560137457044672e-08, |
| "loss": 0.0025, |
| "reward": 2.22265625, |
| "reward_std": 0.276971060782671, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.3671875, |
| "rewards/format_reward": 0.99609375, |
| "step": 269 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.1171875, |
| "epoch": 0.9258465495070725, |
| "grad_norm": 4.622032205390463, |
| "kl": 0.064910888671875, |
| "learning_rate": 7.216494845360824e-08, |
| "loss": 0.0026, |
| "reward": 2.33984375, |
| "reward_std": 0.18649209756404161, |
| "rewards/accuracy_reward_action": 0.96484375, |
| "rewards/accuracy_reward_coord": 0.375, |
| "rewards/format_reward": 1.0, |
| "step": 270 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.1875, |
| "epoch": 0.9292756108015431, |
| "grad_norm": 8.463978759815769, |
| "kl": 0.067596435546875, |
| "learning_rate": 6.872852233676976e-08, |
| "loss": 0.0027, |
| "reward": 2.1484375, |
| "reward_std": 0.27947098668664694, |
| "rewards/accuracy_reward_action": 0.84765625, |
| "rewards/accuracy_reward_coord": 0.30078125, |
| "rewards/format_reward": 1.0, |
| "step": 271 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 100.359375, |
| "epoch": 0.9327046720960137, |
| "grad_norm": 5.986716399294285, |
| "kl": 0.06573486328125, |
| "learning_rate": 6.529209621993128e-08, |
| "loss": 0.0026, |
| "reward": 2.24609375, |
| "reward_std": 0.22154166735708714, |
| "rewards/accuracy_reward_action": 0.92578125, |
| "rewards/accuracy_reward_coord": 0.3203125, |
| "rewards/format_reward": 1.0, |
| "step": 272 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.65625, |
| "epoch": 0.9361337333904843, |
| "grad_norm": 12.162794110740386, |
| "kl": 0.06353759765625, |
| "learning_rate": 6.185567010309278e-08, |
| "loss": 0.0025, |
| "reward": 2.13671875, |
| "reward_std": 0.2439352972432971, |
| "rewards/accuracy_reward_action": 0.8515625, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 1.0, |
| "step": 273 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 101.953125, |
| "epoch": 0.939562794684955, |
| "grad_norm": 2.06279310839585, |
| "kl": 0.062286376953125, |
| "learning_rate": 5.8419243986254297e-08, |
| "loss": 0.0025, |
| "reward": 2.140625, |
| "reward_std": 0.19306486658751965, |
| "rewards/accuracy_reward_action": 0.85546875, |
| "rewards/accuracy_reward_coord": 0.28515625, |
| "rewards/format_reward": 1.0, |
| "step": 274 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.53125, |
| "epoch": 0.9429918559794256, |
| "grad_norm": 1.9068683121900685, |
| "kl": 0.0609130859375, |
| "learning_rate": 5.49828178694158e-08, |
| "loss": 0.0024, |
| "reward": 2.1640625, |
| "reward_std": 0.277761060744524, |
| "rewards/accuracy_reward_action": 0.8125, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 1.0, |
| "step": 275 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.32421875, |
| "epoch": 0.9464209172738963, |
| "grad_norm": 3.0285990315232683, |
| "kl": 0.067657470703125, |
| "learning_rate": 5.154639175257731e-08, |
| "loss": 0.0027, |
| "reward": 2.328125, |
| "reward_std": 0.290500033646822, |
| "rewards/accuracy_reward_action": 0.91796875, |
| "rewards/accuracy_reward_coord": 0.4140625, |
| "rewards/format_reward": 0.99609375, |
| "step": 276 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.52734375, |
| "epoch": 0.949849978568367, |
| "grad_norm": 1.5774729320269285, |
| "kl": 0.065521240234375, |
| "learning_rate": 4.810996563573883e-08, |
| "loss": 0.0026, |
| "reward": 2.1796875, |
| "reward_std": 0.14032597374171019, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.30859375, |
| "rewards/format_reward": 0.9921875, |
| "step": 277 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.0625, |
| "epoch": 0.9532790398628376, |
| "grad_norm": 2.2857988006535033, |
| "kl": 0.075531005859375, |
| "learning_rate": 4.467353951890034e-08, |
| "loss": 0.003, |
| "reward": 2.4375, |
| "reward_std": 0.27437760680913925, |
| "rewards/accuracy_reward_action": 0.953125, |
| "rewards/accuracy_reward_coord": 0.48828125, |
| "rewards/format_reward": 0.99609375, |
| "step": 278 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.14453125, |
| "epoch": 0.9567081011573082, |
| "grad_norm": 2.9913403629956012, |
| "kl": 0.064788818359375, |
| "learning_rate": 4.123711340206185e-08, |
| "loss": 0.0026, |
| "reward": 2.41015625, |
| "reward_std": 0.24962799157947302, |
| "rewards/accuracy_reward_action": 0.9453125, |
| "rewards/accuracy_reward_coord": 0.46484375, |
| "rewards/format_reward": 1.0, |
| "step": 279 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.4921875, |
| "epoch": 0.9601371624517788, |
| "grad_norm": 15.074431320780395, |
| "kl": 0.067779541015625, |
| "learning_rate": 3.780068728522336e-08, |
| "loss": 0.0027, |
| "reward": 2.31640625, |
| "reward_std": 0.2494807243347168, |
| "rewards/accuracy_reward_action": 0.90625, |
| "rewards/accuracy_reward_coord": 0.41015625, |
| "rewards/format_reward": 1.0, |
| "step": 280 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 102.80859375, |
| "epoch": 0.9635662237462495, |
| "grad_norm": 4.661496714661205, |
| "kl": 0.0673828125, |
| "learning_rate": 3.436426116838488e-08, |
| "loss": 0.0027, |
| "reward": 2.15625, |
| "reward_std": 0.2237510783597827, |
| "rewards/accuracy_reward_action": 0.84765625, |
| "rewards/accuracy_reward_coord": 0.31640625, |
| "rewards/format_reward": 0.9921875, |
| "step": 281 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 97.8046875, |
| "epoch": 0.9669952850407201, |
| "grad_norm": 4.1061268232089025, |
| "kl": 0.063873291015625, |
| "learning_rate": 3.092783505154639e-08, |
| "loss": 0.0026, |
| "reward": 2.30078125, |
| "reward_std": 0.2961772596463561, |
| "rewards/accuracy_reward_action": 0.87109375, |
| "rewards/accuracy_reward_coord": 0.4296875, |
| "rewards/format_reward": 1.0, |
| "step": 282 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.58984375, |
| "epoch": 0.9704243463351907, |
| "grad_norm": 4.3384427281781734, |
| "kl": 0.06597900390625, |
| "learning_rate": 2.74914089347079e-08, |
| "loss": 0.0026, |
| "reward": 2.4140625, |
| "reward_std": 0.1958307158201933, |
| "rewards/accuracy_reward_action": 0.97265625, |
| "rewards/accuracy_reward_coord": 0.44140625, |
| "rewards/format_reward": 1.0, |
| "step": 283 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.4609375, |
| "epoch": 0.9738534076296613, |
| "grad_norm": 2.7479574458607425, |
| "kl": 0.072906494140625, |
| "learning_rate": 2.4054982817869415e-08, |
| "loss": 0.0029, |
| "reward": 2.34765625, |
| "reward_std": 0.3268229737877846, |
| "rewards/accuracy_reward_action": 0.94140625, |
| "rewards/accuracy_reward_coord": 0.41015625, |
| "rewards/format_reward": 0.99609375, |
| "step": 284 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.21875, |
| "epoch": 0.9772824689241321, |
| "grad_norm": 37.19873096068833, |
| "kl": 0.06402587890625, |
| "learning_rate": 2.0618556701030925e-08, |
| "loss": 0.0026, |
| "reward": 2.21484375, |
| "reward_std": 0.2886574361473322, |
| "rewards/accuracy_reward_action": 0.86328125, |
| "rewards/accuracy_reward_coord": 0.35546875, |
| "rewards/format_reward": 0.99609375, |
| "step": 285 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 95.6953125, |
| "epoch": 0.9807115302186027, |
| "grad_norm": 1.3343021057081734, |
| "kl": 0.073028564453125, |
| "learning_rate": 1.718213058419244e-08, |
| "loss": 0.0029, |
| "reward": 2.18359375, |
| "reward_std": 0.22028045170009136, |
| "rewards/accuracy_reward_action": 0.83203125, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 1.0, |
| "step": 286 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 98.89453125, |
| "epoch": 0.9841405915130733, |
| "grad_norm": 1.9072408562440208, |
| "kl": 0.0650634765625, |
| "learning_rate": 1.374570446735395e-08, |
| "loss": 0.0026, |
| "reward": 2.078125, |
| "reward_std": 0.1972224973142147, |
| "rewards/accuracy_reward_action": 0.859375, |
| "rewards/accuracy_reward_coord": 0.22265625, |
| "rewards/format_reward": 0.99609375, |
| "step": 287 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 92.73046875, |
| "epoch": 0.9875696528075439, |
| "grad_norm": 2.7207614693913573, |
| "kl": 0.0624542236328125, |
| "learning_rate": 1.0309278350515463e-08, |
| "loss": 0.0025, |
| "reward": 2.35546875, |
| "reward_std": 0.2721018549054861, |
| "rewards/accuracy_reward_action": 0.91015625, |
| "rewards/accuracy_reward_coord": 0.4453125, |
| "rewards/format_reward": 1.0, |
| "step": 288 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.9453125, |
| "epoch": 0.9909987141020146, |
| "grad_norm": 21.507130433498247, |
| "kl": 0.074371337890625, |
| "learning_rate": 6.872852233676975e-09, |
| "loss": 0.003, |
| "reward": 2.23046875, |
| "reward_std": 0.3140362147241831, |
| "rewards/accuracy_reward_action": 0.87890625, |
| "rewards/accuracy_reward_coord": 0.3515625, |
| "rewards/format_reward": 1.0, |
| "step": 289 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 96.6484375, |
| "epoch": 0.9944277753964852, |
| "grad_norm": 2.92422057668803, |
| "kl": 0.077880859375, |
| "learning_rate": 3.4364261168384875e-09, |
| "loss": 0.0031, |
| "reward": 2.32421875, |
| "reward_std": 0.27937449142336845, |
| "rewards/accuracy_reward_action": 0.9453125, |
| "rewards/accuracy_reward_coord": 0.37890625, |
| "rewards/format_reward": 1.0, |
| "step": 290 |
| }, |
| { |
| "clip_ratio": 0.0, |
| "completion_length": 99.09722232818604, |
| "epoch": 0.9978568366909558, |
| "grad_norm": 5.0178346633813415, |
| "kl": 0.074005126953125, |
| "learning_rate": 0.0, |
| "loss": 0.003, |
| "reward": 1.9513889104127884, |
| "reward_std": 0.13368949433788657, |
| "rewards/accuracy_reward_action": 0.8125000111758709, |
| "rewards/accuracy_reward_coord": 0.14583333441987634, |
| "rewards/format_reward": 0.993055559694767, |
| "step": 291 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 291, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
