diff --git "a/trainer_state.json" "b/trainer_state.json" deleted file mode 100644--- "a/trainer_state.json" +++ /dev/null @@ -1,5434 +0,0 @@ -{ - "best_global_step": null, - "best_metric": null, - "best_model_checkpoint": null, - "epoch": 0.06702693645008588, - "eval_steps": 500, - "global_step": 200, - "is_hyper_param_search": false, - "is_local_process_zero": true, - "is_world_process_zero": true, - "log_history": [ - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 225.875, - "completions/max_terminated_length": 225.875, - "completions/mean_length": 170.796875, - "completions/mean_terminated_length": 170.796875, - "completions/min_length": 125.5, - "completions/min_terminated_length": 125.5, - "entropy": 1.0503418520092964, - "epoch": 0.0003351346822504294, - "frac_reward_zero_std": 0.0, - "grad_norm": 11.864991188049316, - "learning_rate": 1e-06, - "loss": -0.06142277643084526, - "num_tokens": 88123.0, - "reward": 0.28125, - "reward_std": 0.42873119935393333, - "rewards/embodied_reward/mean": 0.28125, - "rewards/embodied_reward/std": 0.42873119935393333, - "step": 1, - "step_time": 216.38424833398312 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 233.625, - "completions/max_terminated_length": 233.625, - "completions/mean_length": 153.203125, - "completions/mean_terminated_length": 153.203125, - "completions/min_length": 92.25, - "completions/min_terminated_length": 92.25, - "entropy": 0.8750532492995262, - "epoch": 0.0006702693645008588, - "frac_reward_zero_std": 0.0, - "grad_norm": 4.950372219085693, - "learning_rate": 9.99888293118856e-07, - "loss": 0.0029853805899620056, - "num_tokens": 172624.0, - "reward": 0.40625, - "reward_std": 0.4560507796704769, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.4560507796704769, - "step": 2, - "step_time": 223.30681869853288 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 236.375, - "completions/max_terminated_length": 236.375, - "completions/mean_length": 175.171875, - "completions/mean_terminated_length": 175.171875, - "completions/min_length": 127.875, - "completions/min_terminated_length": 127.875, - "entropy": 1.095536731183529, - "epoch": 0.0010054040467512883, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.811668872833252, - "learning_rate": 9.997765862377123e-07, - "loss": 0.010467112064361572, - "num_tokens": 261947.0, - "reward": 0.265625, - "reward_std": 0.3934885784983635, - "rewards/embodied_reward/mean": 0.265625, - "rewards/embodied_reward/std": 0.3934885784983635, - "step": 3, - "step_time": 219.62907661870122 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.03125, - "completions/max_length": 273.0, - "completions/max_terminated_length": 194.0, - "completions/mean_length": 139.328125, - "completions/mean_terminated_length": 127.46428728103638, - "completions/min_length": 77.75, - "completions/min_terminated_length": 77.75, - "entropy": 0.8548796810209751, - "epoch": 0.0013405387290017176, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.119979381561279, - "learning_rate": 9.996648793565684e-07, - "loss": 0.03952433168888092, - "num_tokens": 342328.0, - "reward": 0.5, - "reward_std": 0.378745436668396, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.378745436668396, - "step": 4, - "step_time": 249.47778285527602 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.0, - "completions/max_terminated_length": 220.0, - "completions/mean_length": 158.21875, - "completions/mean_terminated_length": 158.21875, - "completions/min_length": 93.75, - "completions/min_terminated_length": 93.75, - "entropy": 0.9264072179794312, - "epoch": 0.001675673411252147, - "frac_reward_zero_std": 0.125, - "grad_norm": 9.573637008666992, - "learning_rate": 9.995531724754244e-07, - "loss": -0.04509415850043297, - "num_tokens": 426582.0, - "reward": 0.453125, - "reward_std": 0.38664888963103294, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.38664888963103294, - "step": 5, - "step_time": 213.6988994004205 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 225.125, - "completions/max_terminated_length": 225.125, - "completions/mean_length": 172.40625, - "completions/mean_terminated_length": 172.40625, - "completions/min_length": 124.875, - "completions/min_terminated_length": 124.875, - "entropy": 1.0127676874399185, - "epoch": 0.0020108080935025766, - "frac_reward_zero_std": 0.0, - "grad_norm": 10.762822151184082, - "learning_rate": 9.994414655942805e-07, - "loss": -0.07516171038150787, - "num_tokens": 510072.0, - "reward": 0.390625, - "reward_std": 0.45346419885754585, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.45346419885754585, - "step": 6, - "step_time": 206.15916877705604 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 214.75, - "completions/max_terminated_length": 214.75, - "completions/mean_length": 149.921875, - "completions/mean_terminated_length": 149.921875, - "completions/min_length": 114.125, - "completions/min_terminated_length": 114.125, - "entropy": 0.9185877703130245, - "epoch": 0.002345942775753006, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.08226203918457, - "learning_rate": 9.993297587131368e-07, - "loss": 0.0013321749866008759, - "num_tokens": 587643.0, - "reward": 0.4375, - "reward_std": 0.3540026433765888, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.3540026433765888, - "step": 7, - "step_time": 198.5258613359183 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 204.25, - "completions/max_terminated_length": 204.25, - "completions/mean_length": 148.640625, - "completions/mean_terminated_length": 148.640625, - "completions/min_length": 98.375, - "completions/min_terminated_length": 98.375, - "entropy": 0.9549025967717171, - "epoch": 0.002681077458003435, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.150115013122559, - "learning_rate": 9.992180518319929e-07, - "loss": -0.10416066646575928, - "num_tokens": 672172.0, - "reward": 0.3125, - "reward_std": 0.4118664003908634, - "rewards/embodied_reward/mean": 0.3125, - "rewards/embodied_reward/std": 0.4118664003908634, - "step": 8, - "step_time": 208.28969044191763 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 202.75, - "completions/max_terminated_length": 202.75, - "completions/mean_length": 149.96875, - "completions/mean_terminated_length": 149.96875, - "completions/min_length": 93.25, - "completions/min_terminated_length": 93.25, - "entropy": 1.0726491585373878, - "epoch": 0.0030162121402538644, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.0146260261535645, - "learning_rate": 9.99106344950849e-07, - "loss": 0.02623813971877098, - "num_tokens": 754490.0, - "reward": 0.4375, - "reward_std": 0.4340128190815449, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.4340128190815449, - "step": 9, - "step_time": 186.34493432706222 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 240.375, - "completions/max_terminated_length": 240.375, - "completions/mean_length": 170.34375, - "completions/mean_terminated_length": 170.34375, - "completions/min_length": 129.75, - "completions/min_terminated_length": 129.75, - "entropy": 1.0655138939619064, - "epoch": 0.003351346822504294, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.189670562744141, - "learning_rate": 9.98994638069705e-07, - "loss": -0.09415297955274582, - "num_tokens": 847584.0, - "reward": 0.53125, - "reward_std": 0.4718420207500458, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.4718420207500458, - "step": 10, - "step_time": 222.44042409723625 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 272.5, - "completions/max_terminated_length": 272.5, - "completions/mean_length": 173.109375, - "completions/mean_terminated_length": 173.109375, - "completions/min_length": 110.5, - "completions/min_terminated_length": 110.5, - "entropy": 0.907024260610342, - "epoch": 0.0036864815047547234, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.223816394805908, - "learning_rate": 9.98882931188561e-07, - "loss": -0.06003513187170029, - "num_tokens": 941031.0, - "reward": 0.453125, - "reward_std": 0.398196816444397, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.398196816444397, - "step": 11, - "step_time": 251.41313650878146 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 213.625, - "completions/max_terminated_length": 213.625, - "completions/mean_length": 159.015625, - "completions/mean_terminated_length": 159.015625, - "completions/min_length": 106.0, - "completions/min_terminated_length": 106.0, - "entropy": 0.979051299393177, - "epoch": 0.004021616187005153, - "frac_reward_zero_std": 0.125, - "grad_norm": 4.801168441772461, - "learning_rate": 9.987712243074174e-07, - "loss": -0.06829184293746948, - "num_tokens": 1027432.0, - "reward": 0.46875, - "reward_std": 0.42976950481534004, - "rewards/embodied_reward/mean": 0.46875, - "rewards/embodied_reward/std": 0.42976950481534004, - "step": 12, - "step_time": 203.89850472984836 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 213.375, - "completions/max_terminated_length": 213.375, - "completions/mean_length": 159.890625, - "completions/mean_terminated_length": 159.890625, - "completions/min_length": 109.5, - "completions/min_terminated_length": 109.5, - "entropy": 0.8931524530053139, - "epoch": 0.004356750869255582, - "frac_reward_zero_std": 0.0, - "grad_norm": 10.420388221740723, - "learning_rate": 9.986595174262734e-07, - "loss": -0.050558820366859436, - "num_tokens": 1117185.0, - "reward": 0.5, - "reward_std": 0.4671337828040123, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.4671337828040123, - "step": 13, - "step_time": 203.29407280683517 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 222.625, - "completions/max_terminated_length": 222.625, - "completions/mean_length": 150.390625, - "completions/mean_terminated_length": 150.390625, - "completions/min_length": 100.125, - "completions/min_terminated_length": 100.125, - "entropy": 0.8581663072109222, - "epoch": 0.004691885551506012, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.973217487335205, - "learning_rate": 9.985478105451295e-07, - "loss": -0.05375608801841736, - "num_tokens": 1202682.0, - "reward": 0.4375, - "reward_std": 0.4918765760958195, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.4918765760958195, - "step": 14, - "step_time": 213.3025799258612 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 234.125, - "completions/max_terminated_length": 234.125, - "completions/mean_length": 196.359375, - "completions/mean_terminated_length": 196.359375, - "completions/min_length": 154.5, - "completions/min_terminated_length": 154.5, - "entropy": 1.1141883954405785, - "epoch": 0.0050270202337564405, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.42173433303833, - "learning_rate": 9.984361036639858e-07, - "loss": -0.019827160984277725, - "num_tokens": 1294921.0, - "reward": 0.515625, - "reward_std": 0.4855116046965122, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.4855116046965122, - "step": 15, - "step_time": 221.23253494221717 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 205.125, - "completions/max_terminated_length": 205.125, - "completions/mean_length": 145.21875, - "completions/mean_terminated_length": 145.21875, - "completions/min_length": 92.75, - "completions/min_terminated_length": 92.75, - "entropy": 0.9090883880853653, - "epoch": 0.00536215491600687, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.822619438171387, - "learning_rate": 9.983243967828418e-07, - "loss": -0.10418610274791718, - "num_tokens": 1372191.0, - "reward": 0.546875, - "reward_std": 0.4186962954699993, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.4186962954699993, - "step": 16, - "step_time": 192.51455764239654 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 187.25, - "completions/max_terminated_length": 187.25, - "completions/mean_length": 148.34375, - "completions/mean_terminated_length": 148.34375, - "completions/min_length": 93.0, - "completions/min_terminated_length": 93.0, - "entropy": 0.8521850630640984, - "epoch": 0.0056972895982573, - "frac_reward_zero_std": 0.0, - "grad_norm": 6.77331018447876, - "learning_rate": 9.98212689901698e-07, - "loss": -0.0018602870404720306, - "num_tokens": 1451797.0, - "reward": 0.59375, - "reward_std": 0.47397347167134285, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.47397347167134285, - "step": 17, - "step_time": 183.43365518562496 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 217.0, - "completions/max_terminated_length": 217.0, - "completions/mean_length": 153.4375, - "completions/mean_terminated_length": 153.4375, - "completions/min_length": 100.625, - "completions/min_terminated_length": 100.625, - "entropy": 0.8924854770302773, - "epoch": 0.006032424280507729, - "frac_reward_zero_std": 0.0, - "grad_norm": 7.525657653808594, - "learning_rate": 9.98100983020554e-07, - "loss": 0.08587130904197693, - "num_tokens": 1534809.0, - "reward": 0.578125, - "reward_std": 0.48080336675047874, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.48080336675047874, - "step": 18, - "step_time": 207.86813628859818 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.015625, - "completions/max_length": 246.375, - "completions/max_terminated_length": 202.125, - "completions/mean_length": 157.53125, - "completions/mean_terminated_length": 150.5535717010498, - "completions/min_length": 105.875, - "completions/min_terminated_length": 105.875, - "entropy": 0.8997205123305321, - "epoch": 0.0063675589627581585, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.017297744750977, - "learning_rate": 9.979892761394103e-07, - "loss": 0.056050967425107956, - "num_tokens": 1620499.0, - "reward": 0.453125, - "reward_std": 0.36767222732305527, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.36767222732305527, - "step": 19, - "step_time": 231.56676801480353 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 239.75, - "completions/max_terminated_length": 239.75, - "completions/mean_length": 167.65625, - "completions/mean_terminated_length": 167.65625, - "completions/min_length": 123.625, - "completions/min_terminated_length": 123.625, - "entropy": 0.9739629626274109, - "epoch": 0.006702693645008588, - "frac_reward_zero_std": 0.0, - "grad_norm": 6.285614013671875, - "learning_rate": 9.978775692582663e-07, - "loss": 0.01535167545080185, - "num_tokens": 1705053.0, - "reward": 0.5625, - "reward_std": 0.45134254172444344, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.45134254172444344, - "step": 20, - "step_time": 216.09783126413822 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 235.5, - "completions/max_terminated_length": 235.5, - "completions/mean_length": 184.5625, - "completions/mean_terminated_length": 184.5625, - "completions/min_length": 128.875, - "completions/min_terminated_length": 128.875, - "entropy": 1.0687043890357018, - "epoch": 0.007037828327259017, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.751375675201416, - "learning_rate": 9.977658623771224e-07, - "loss": -0.022062182426452637, - "num_tokens": 1797609.0, - "reward": 0.40625, - "reward_std": 0.4340128190815449, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.4340128190815449, - "step": 21, - "step_time": 216.57936902716756 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 238.0, - "completions/max_terminated_length": 238.0, - "completions/mean_length": 173.328125, - "completions/mean_terminated_length": 173.328125, - "completions/min_length": 127.5, - "completions/min_terminated_length": 127.5, - "entropy": 1.071356937289238, - "epoch": 0.007372963009509447, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.7314982414245605, - "learning_rate": 9.976541554959785e-07, - "loss": 0.010602042078971863, - "num_tokens": 1882886.0, - "reward": 0.453125, - "reward_std": 0.5034147091209888, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.5034147091209888, - "step": 22, - "step_time": 215.96834176918492 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 246.25, - "completions/max_terminated_length": 246.25, - "completions/mean_length": 191.625, - "completions/mean_terminated_length": 191.625, - "completions/min_length": 146.125, - "completions/min_terminated_length": 146.125, - "entropy": 1.1181199476122856, - "epoch": 0.007708097691759876, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.101344108581543, - "learning_rate": 9.975424486148345e-07, - "loss": -0.029504582285881042, - "num_tokens": 1969774.0, - "reward": 0.40625, - "reward_std": 0.36295419558882713, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.36295419558882713, - "step": 23, - "step_time": 234.35820465069264 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 209.75, - "completions/max_terminated_length": 209.75, - "completions/mean_length": 155.578125, - "completions/mean_terminated_length": 155.578125, - "completions/min_length": 106.125, - "completions/min_terminated_length": 106.125, - "entropy": 0.881608821451664, - "epoch": 0.008043232374010306, - "frac_reward_zero_std": 0.0, - "grad_norm": 8.201946258544922, - "learning_rate": 9.974307417336908e-07, - "loss": -0.0928099974989891, - "num_tokens": 2053963.0, - "reward": 0.546875, - "reward_std": 0.47185181453824043, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.47185181453824043, - "step": 24, - "step_time": 205.3743497808464 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 202.125, - "completions/max_terminated_length": 202.125, - "completions/mean_length": 142.125, - "completions/mean_terminated_length": 142.125, - "completions/min_length": 77.125, - "completions/min_terminated_length": 77.125, - "entropy": 0.8520144671201706, - "epoch": 0.008378367056260735, - "frac_reward_zero_std": 0.125, - "grad_norm": 12.450089454650879, - "learning_rate": 9.97319034852547e-07, - "loss": -0.050880610942840576, - "num_tokens": 2142827.0, - "reward": 0.40625, - "reward_std": 0.42081795260310173, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.42081795260310173, - "step": 25, - "step_time": 190.9370715864934 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 221.75, - "completions/max_terminated_length": 221.75, - "completions/mean_length": 178.765625, - "completions/mean_terminated_length": 178.765625, - "completions/min_length": 128.0, - "completions/min_terminated_length": 128.0, - "entropy": 1.1063818112015724, - "epoch": 0.008713501738511164, - "frac_reward_zero_std": 0.25, - "grad_norm": 15.426373481750488, - "learning_rate": 9.97207327971403e-07, - "loss": 0.0023010745644569397, - "num_tokens": 2231700.0, - "reward": 0.296875, - "reward_std": 0.3445763736963272, - "rewards/embodied_reward/mean": 0.296875, - "rewards/embodied_reward/std": 0.3445763736963272, - "step": 26, - "step_time": 204.8365693972446 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 222.125, - "completions/max_terminated_length": 222.125, - "completions/mean_length": 172.0, - "completions/mean_terminated_length": 172.0, - "completions/min_length": 125.75, - "completions/min_terminated_length": 125.75, - "entropy": 0.9295278117060661, - "epoch": 0.009048636420761593, - "frac_reward_zero_std": 0.0, - "grad_norm": 6.464595317840576, - "learning_rate": 9.97095621090259e-07, - "loss": 0.06612984091043472, - "num_tokens": 2313716.0, - "reward": 0.515625, - "reward_std": 0.4513523355126381, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.4513523355126381, - "step": 27, - "step_time": 203.21224692370743 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 238.125, - "completions/max_terminated_length": 238.125, - "completions/mean_length": 185.1875, - "completions/mean_terminated_length": 185.1875, - "completions/min_length": 137.5, - "completions/min_terminated_length": 137.5, - "entropy": 0.9988782033324242, - "epoch": 0.009383771103012023, - "frac_reward_zero_std": 0.125, - "grad_norm": 4.999497413635254, - "learning_rate": 9.96983914209115e-07, - "loss": -0.02533743530511856, - "num_tokens": 2400248.0, - "reward": 0.484375, - "reward_std": 0.4434390887618065, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.4434390887618065, - "step": 28, - "step_time": 220.5169855831191 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 212.25, - "completions/max_terminated_length": 212.25, - "completions/mean_length": 171.453125, - "completions/mean_terminated_length": 171.453125, - "completions/min_length": 114.375, - "completions/min_terminated_length": 114.375, - "entropy": 1.035932119935751, - "epoch": 0.009718905785262452, - "frac_reward_zero_std": 0.125, - "grad_norm": 4.250608444213867, - "learning_rate": 9.968722073279714e-07, - "loss": -0.0059820376336574554, - "num_tokens": 2487549.0, - "reward": 0.53125, - "reward_std": 0.42976950481534004, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.42976950481534004, - "step": 29, - "step_time": 203.9552340703085 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 199.5, - "completions/max_terminated_length": 199.5, - "completions/mean_length": 141.921875, - "completions/mean_terminated_length": 141.921875, - "completions/min_length": 86.0, - "completions/min_terminated_length": 86.0, - "entropy": 0.9631340950727463, - "epoch": 0.010054040467512881, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.109882831573486, - "learning_rate": 9.967605004468275e-07, - "loss": 0.042611315846443176, - "num_tokens": 2571192.0, - "reward": 0.359375, - "reward_std": 0.4276478476822376, - "rewards/embodied_reward/mean": 0.359375, - "rewards/embodied_reward/std": 0.4276478476822376, - "step": 30, - "step_time": 197.22236177884042 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 242.75, - "completions/max_terminated_length": 242.75, - "completions/mean_length": 189.59375, - "completions/mean_terminated_length": 189.59375, - "completions/min_length": 150.0, - "completions/min_terminated_length": 150.0, - "entropy": 1.0471477508544922, - "epoch": 0.010389175149763312, - "frac_reward_zero_std": 0.25, - "grad_norm": 3.9301867485046387, - "learning_rate": 9.966487935656837e-07, - "loss": 0.007835470139980316, - "num_tokens": 2658766.0, - "reward": 0.453125, - "reward_std": 0.38133201748132706, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.38133201748132706, - "step": 31, - "step_time": 224.33049333421513 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 188.125, - "completions/max_terminated_length": 188.125, - "completions/mean_length": 135.421875, - "completions/mean_terminated_length": 135.421875, - "completions/min_length": 70.5, - "completions/min_terminated_length": 70.5, - "entropy": 0.8054907768964767, - "epoch": 0.01072430983201374, - "frac_reward_zero_std": 0.0, - "grad_norm": 6.475802421569824, - "learning_rate": 9.965370866845398e-07, - "loss": -0.06960076838731766, - "num_tokens": 2743465.0, - "reward": 0.453125, - "reward_std": 0.4287214055657387, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.4287214055657387, - "step": 32, - "step_time": 180.32620351249352 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 233.625, - "completions/max_terminated_length": 233.625, - "completions/mean_length": 193.5625, - "completions/mean_terminated_length": 193.5625, - "completions/min_length": 144.875, - "completions/min_terminated_length": 144.875, - "entropy": 1.0550732016563416, - "epoch": 0.01105944451426417, - "frac_reward_zero_std": 0.5, - "grad_norm": 3.802373170852661, - "learning_rate": 9.964253798033959e-07, - "loss": -0.022170547395944595, - "num_tokens": 2836445.0, - "reward": 0.234375, - "reward_std": 0.23144523426890373, - "rewards/embodied_reward/mean": 0.234375, - "rewards/embodied_reward/std": 0.23144523426890373, - "step": 33, - "step_time": 215.58006748743355 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.375, - "completions/max_terminated_length": 197.375, - "completions/mean_length": 160.015625, - "completions/mean_terminated_length": 160.015625, - "completions/min_length": 118.875, - "completions/min_terminated_length": 118.875, - "entropy": 0.7924338579177856, - "epoch": 0.0113945791965146, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.149416923522949, - "learning_rate": 9.96313672922252e-07, - "loss": 0.014046791940927505, - "num_tokens": 2924806.0, - "reward": 0.328125, - "reward_std": 0.398196816444397, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.398196816444397, - "step": 34, - "step_time": 193.83369634114206 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 235.75, - "completions/max_terminated_length": 235.75, - "completions/mean_length": 185.921875, - "completions/mean_terminated_length": 185.921875, - "completions/min_length": 135.875, - "completions/min_terminated_length": 135.875, - "entropy": 1.0089291036128998, - "epoch": 0.011729713878765029, - "frac_reward_zero_std": 0.0, - "grad_norm": 7.1747894287109375, - "learning_rate": 9.96201966041108e-07, - "loss": -0.018929369747638702, - "num_tokens": 3011849.0, - "reward": 0.484375, - "reward_std": 0.47185181453824043, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.47185181453824043, - "step": 35, - "step_time": 219.9416045746766 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 207.75, - "completions/max_terminated_length": 207.75, - "completions/mean_length": 145.671875, - "completions/mean_terminated_length": 145.671875, - "completions/min_length": 87.375, - "completions/min_terminated_length": 87.375, - "entropy": 0.8565007224678993, - "epoch": 0.012064848561015458, - "frac_reward_zero_std": 0.125, - "grad_norm": 11.541733741760254, - "learning_rate": 9.960902591599643e-07, - "loss": -0.05049588158726692, - "num_tokens": 3098628.0, - "reward": 0.453125, - "reward_std": 0.42553598433732986, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.42553598433732986, - "step": 36, - "step_time": 196.28732378128916 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 230.5, - "completions/max_terminated_length": 230.5, - "completions/mean_length": 181.140625, - "completions/mean_terminated_length": 181.140625, - "completions/min_length": 141.625, - "completions/min_terminated_length": 141.625, - "entropy": 1.063988246023655, - "epoch": 0.012399983243265888, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.367402076721191, - "learning_rate": 9.959785522788204e-07, - "loss": 0.03621979057788849, - "num_tokens": 3189981.0, - "reward": 0.359375, - "reward_std": 0.35612430050969124, - "rewards/embodied_reward/mean": 0.359375, - "rewards/embodied_reward/std": 0.35612430050969124, - "step": 37, - "step_time": 213.88573352945969 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 205.75, - "completions/max_terminated_length": 205.75, - "completions/mean_length": 158.921875, - "completions/mean_terminated_length": 158.921875, - "completions/min_length": 104.625, - "completions/min_terminated_length": 104.625, - "entropy": 0.9690025225281715, - "epoch": 0.012735117925516317, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.441710472106934, - "learning_rate": 9.958668453976764e-07, - "loss": 0.055262207984924316, - "num_tokens": 3280656.0, - "reward": 0.4375, - "reward_std": 0.41610971465706825, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.41610971465706825, - "step": 38, - "step_time": 198.19167350186035 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.25, - "completions/max_terminated_length": 218.25, - "completions/mean_length": 173.59375, - "completions/mean_terminated_length": 173.59375, - "completions/min_length": 119.25, - "completions/min_terminated_length": 119.25, - "entropy": 0.9914040043950081, - "epoch": 0.013070252607766746, - "frac_reward_zero_std": 0.125, - "grad_norm": 12.30889892578125, - "learning_rate": 9.957551385165325e-07, - "loss": 0.059513889253139496, - "num_tokens": 3364334.0, - "reward": 0.578125, - "reward_std": 0.4050365053117275, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.4050365053117275, - "step": 39, - "step_time": 204.2027769451961 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.375, - "completions/max_terminated_length": 218.375, - "completions/mean_length": 171.65625, - "completions/mean_terminated_length": 171.65625, - "completions/min_length": 110.625, - "completions/min_terminated_length": 110.625, - "entropy": 0.9519792571663857, - "epoch": 0.013405387290017176, - "frac_reward_zero_std": 0.0, - "grad_norm": 7.521308422088623, - "learning_rate": 9.956434316353886e-07, - "loss": -0.01916140876710415, - "num_tokens": 3451840.0, - "reward": 0.421875, - "reward_std": 0.4855116046965122, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.4855116046965122, - "step": 40, - "step_time": 207.69160678051412 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 219.0, - "completions/max_terminated_length": 219.0, - "completions/mean_length": 174.3125, - "completions/mean_terminated_length": 174.3125, - "completions/min_length": 116.5, - "completions/min_terminated_length": 116.5, - "entropy": 0.9299773499369621, - "epoch": 0.013740521972267605, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.242437839508057, - "learning_rate": 9.955317247542449e-07, - "loss": -0.04166797176003456, - "num_tokens": 3535412.0, - "reward": 0.5, - "reward_std": 0.4366091936826706, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.4366091936826706, - "step": 41, - "step_time": 211.71333878533915 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 183.25, - "completions/max_terminated_length": 183.25, - "completions/mean_length": 144.96875, - "completions/mean_terminated_length": 144.96875, - "completions/min_length": 93.125, - "completions/min_terminated_length": 93.125, - "entropy": 0.7677692323923111, - "epoch": 0.014075656654518034, - "frac_reward_zero_std": 0.25, - "grad_norm": 8.136746406555176, - "learning_rate": 9.95420017873101e-07, - "loss": 0.031837098300457, - "num_tokens": 3621666.0, - "reward": 0.40625, - "reward_std": 0.31300368532538414, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.31300368532538414, - "step": 42, - "step_time": 175.61713674198836 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 188.5, - "completions/max_terminated_length": 188.5, - "completions/mean_length": 141.515625, - "completions/mean_terminated_length": 141.515625, - "completions/min_length": 83.875, - "completions/min_terminated_length": 83.875, - "entropy": 0.705445758998394, - "epoch": 0.014410791336768463, - "frac_reward_zero_std": 0.125, - "grad_norm": 9.7794771194458, - "learning_rate": 9.953083109919572e-07, - "loss": -0.03957376629114151, - "num_tokens": 3702219.0, - "reward": 0.625, - "reward_std": 0.41610971465706825, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.41610971465706825, - "step": 43, - "step_time": 180.68038399377838 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.625, - "completions/max_terminated_length": 210.625, - "completions/mean_length": 156.5, - "completions/mean_terminated_length": 156.5, - "completions/min_length": 87.875, - "completions/min_terminated_length": 87.875, - "entropy": 0.8889512345194817, - "epoch": 0.014745926019018894, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.727976322174072, - "learning_rate": 9.951966041108133e-07, - "loss": 0.05946221947669983, - "num_tokens": 3787667.0, - "reward": 0.546875, - "reward_std": 0.268809512257576, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.268809512257576, - "step": 44, - "step_time": 202.895914579276 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.0, - "completions/max_terminated_length": 218.0, - "completions/mean_length": 177.96875, - "completions/mean_terminated_length": 177.96875, - "completions/min_length": 131.375, - "completions/min_terminated_length": 131.375, - "entropy": 0.9341032654047012, - "epoch": 0.015081060701269322, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.08352518081665, - "learning_rate": 9.950848972296693e-07, - "loss": 0.011213922873139381, - "num_tokens": 3876761.0, - "reward": 0.59375, - "reward_std": 0.3072218894958496, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.3072218894958496, - "step": 45, - "step_time": 216.47299083694816 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 240.0, - "completions/max_terminated_length": 240.0, - "completions/mean_length": 191.15625, - "completions/mean_terminated_length": 191.15625, - "completions/min_length": 141.125, - "completions/min_terminated_length": 141.125, - "entropy": 1.1357171908020973, - "epoch": 0.015416195383519751, - "frac_reward_zero_std": 0.125, - "grad_norm": 9.986356735229492, - "learning_rate": 9.949731903485254e-07, - "loss": 0.026241466403007507, - "num_tokens": 3965843.0, - "reward": 0.40625, - "reward_std": 0.3845272324979305, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.3845272324979305, - "step": 46, - "step_time": 233.5188608020544 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 201.0, - "completions/max_terminated_length": 201.0, - "completions/mean_length": 153.109375, - "completions/mean_terminated_length": 153.109375, - "completions/min_length": 94.75, - "completions/min_terminated_length": 94.75, - "entropy": 0.855239748954773, - "epoch": 0.015751330065770182, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.805131912231445, - "learning_rate": 9.948614834673815e-07, - "loss": 0.08344872295856476, - "num_tokens": 4052042.0, - "reward": 0.65625, - "reward_std": 0.4255261905491352, - "rewards/embodied_reward/mean": 0.65625, - "rewards/embodied_reward/std": 0.4255261905491352, - "step": 47, - "step_time": 200.79564853245392 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 230.125, - "completions/max_terminated_length": 230.125, - "completions/mean_length": 184.71875, - "completions/mean_terminated_length": 184.71875, - "completions/min_length": 144.375, - "completions/min_terminated_length": 144.375, - "entropy": 0.9989443048834801, - "epoch": 0.016086464748020612, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.195456027984619, - "learning_rate": 9.947497765862378e-07, - "loss": 0.07254484295845032, - "num_tokens": 4139536.0, - "reward": 0.359375, - "reward_std": 0.32878513261675835, - "rewards/embodied_reward/mean": 0.359375, - "rewards/embodied_reward/std": 0.32878513261675835, - "step": 48, - "step_time": 211.37551784096286 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.375, - "completions/max_terminated_length": 210.375, - "completions/mean_length": 161.046875, - "completions/mean_terminated_length": 161.046875, - "completions/min_length": 111.125, - "completions/min_terminated_length": 111.125, - "entropy": 1.0080452486872673, - "epoch": 0.01642159943027104, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.78204345703125, - "learning_rate": 9.946380697050938e-07, - "loss": -0.028072435408830643, - "num_tokens": 4224931.0, - "reward": 0.5625, - "reward_std": 0.4918765760958195, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.4918765760958195, - "step": 49, - "step_time": 209.19759666593745 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 209.625, - "completions/max_terminated_length": 209.625, - "completions/mean_length": 160.984375, - "completions/mean_terminated_length": 160.984375, - "completions/min_length": 95.125, - "completions/min_terminated_length": 95.125, - "entropy": 0.8875817209482193, - "epoch": 0.01675673411252147, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.57839298248291, - "learning_rate": 9.9452636282395e-07, - "loss": -0.02962787076830864, - "num_tokens": 4312674.0, - "reward": 0.578125, - "reward_std": 0.43189116194844246, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.43189116194844246, - "step": 50, - "step_time": 200.16367353871465 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 206.375, - "completions/max_terminated_length": 206.375, - "completions/mean_length": 159.390625, - "completions/mean_terminated_length": 159.390625, - "completions/min_length": 100.375, - "completions/min_terminated_length": 100.375, - "entropy": 0.8942721448838711, - "epoch": 0.017091868794771897, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.174291610717773, - "learning_rate": 9.94414655942806e-07, - "loss": 0.01754888892173767, - "num_tokens": 4396299.0, - "reward": 0.546875, - "reward_std": 0.2777610644698143, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.2777610644698143, - "step": 51, - "step_time": 195.96849506907165 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 247.0, - "completions/max_terminated_length": 247.0, - "completions/mean_length": 199.859375, - "completions/mean_terminated_length": 199.859375, - "completions/min_length": 148.75, - "completions/min_terminated_length": 148.75, - "entropy": 1.0864437967538834, - "epoch": 0.017427003477022328, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.696345329284668, - "learning_rate": 9.94302949061662e-07, - "loss": 0.026493258774280548, - "num_tokens": 4486850.0, - "reward": 0.5625, - "reward_std": 0.3913669213652611, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.3913669213652611, - "step": 52, - "step_time": 227.21073367306963 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 212.5, - "completions/max_terminated_length": 212.5, - "completions/mean_length": 173.671875, - "completions/mean_terminated_length": 173.671875, - "completions/min_length": 125.25, - "completions/min_terminated_length": 125.25, - "entropy": 0.9690144322812557, - "epoch": 0.01776213815927276, - "frac_reward_zero_std": 0.5, - "grad_norm": 5.695949554443359, - "learning_rate": 9.941912421805183e-07, - "loss": 0.014185317791998386, - "num_tokens": 4575397.0, - "reward": 0.640625, - "reward_std": 0.2198973074555397, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.2198973074555397, - "step": 53, - "step_time": 197.31443982152268 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.125, - "completions/max_terminated_length": 197.125, - "completions/mean_length": 143.125, - "completions/mean_terminated_length": 143.125, - "completions/min_length": 90.75, - "completions/min_terminated_length": 90.75, - "entropy": 0.7639511525630951, - "epoch": 0.018097272841523186, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.713415145874023, - "learning_rate": 9.940795352993744e-07, - "loss": -0.059786200523376465, - "num_tokens": 4659917.0, - "reward": 0.484375, - "reward_std": 0.38664888963103294, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.38664888963103294, - "step": 54, - "step_time": 186.15619327500463 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 180.75, - "completions/max_terminated_length": 180.75, - "completions/mean_length": 128.453125, - "completions/mean_terminated_length": 128.453125, - "completions/min_length": 72.125, - "completions/min_terminated_length": 72.125, - "entropy": 0.7642825320363045, - "epoch": 0.018432407523773616, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.463097095489502, - "learning_rate": 9.939678284182305e-07, - "loss": -0.017153847962617874, - "num_tokens": 4741386.0, - "reward": 0.40625, - "reward_std": 0.4092700257897377, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.4092700257897377, - "step": 55, - "step_time": 176.53630728507414 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.25, - "completions/max_terminated_length": 220.25, - "completions/mean_length": 173.9375, - "completions/mean_terminated_length": 173.9375, - "completions/min_length": 125.25, - "completions/min_terminated_length": 125.25, - "entropy": 1.0596941113471985, - "epoch": 0.018767542206024047, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.710636615753174, - "learning_rate": 9.938561215370867e-07, - "loss": 0.05546807497739792, - "num_tokens": 4825918.0, - "reward": 0.40625, - "reward_std": 0.40502671152353287, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.40502671152353287, - "step": 56, - "step_time": 209.2903693257831 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 215.5, - "completions/max_terminated_length": 215.5, - "completions/mean_length": 163.53125, - "completions/mean_terminated_length": 163.53125, - "completions/min_length": 103.125, - "completions/min_terminated_length": 103.125, - "entropy": 0.9143976904451847, - "epoch": 0.019102676888274474, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.851064205169678, - "learning_rate": 9.937444146559428e-07, - "loss": -0.03217067942023277, - "num_tokens": 4910872.0, - "reward": 0.578125, - "reward_std": 0.3403330594301224, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.3403330594301224, - "step": 57, - "step_time": 208.35944880358875 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 191.25, - "completions/max_terminated_length": 191.25, - "completions/mean_length": 133.734375, - "completions/mean_terminated_length": 133.734375, - "completions/min_length": 75.5, - "completions/min_terminated_length": 75.5, - "entropy": 0.7548856288194656, - "epoch": 0.019437811570524904, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.896003246307373, - "learning_rate": 9.936327077747989e-07, - "loss": -0.0976746454834938, - "num_tokens": 4992639.0, - "reward": 0.5, - "reward_std": 0.4003184735774994, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.4003184735774994, - "step": 58, - "step_time": 178.459804341197 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 230.25, - "completions/max_terminated_length": 230.25, - "completions/mean_length": 171.875, - "completions/mean_terminated_length": 171.875, - "completions/min_length": 113.875, - "completions/min_terminated_length": 113.875, - "entropy": 0.8939539827406406, - "epoch": 0.019772946252775335, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.306131362915039, - "learning_rate": 9.93521000893655e-07, - "loss": 0.02398674190044403, - "num_tokens": 5081367.0, - "reward": 0.640625, - "reward_std": 0.35612430050969124, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.35612430050969124, - "step": 59, - "step_time": 209.94198608677834 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 234.875, - "completions/max_terminated_length": 234.875, - "completions/mean_length": 189.078125, - "completions/mean_terminated_length": 189.078125, - "completions/min_length": 144.875, - "completions/min_terminated_length": 144.875, - "entropy": 0.9913431145250797, - "epoch": 0.020108080935025762, - "frac_reward_zero_std": 0.25, - "grad_norm": 14.217759132385254, - "learning_rate": 9.934092940125112e-07, - "loss": 0.01496008038520813, - "num_tokens": 5171428.0, - "reward": 0.609375, - "reward_std": 0.3403330594301224, - "rewards/embodied_reward/mean": 0.609375, - "rewards/embodied_reward/std": 0.3403330594301224, - "step": 60, - "step_time": 220.35051255812868 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 215.75, - "completions/max_terminated_length": 215.75, - "completions/mean_length": 177.640625, - "completions/mean_terminated_length": 177.640625, - "completions/min_length": 131.5, - "completions/min_terminated_length": 131.5, - "entropy": 0.9791299179196358, - "epoch": 0.020443215617276193, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.407893657684326, - "learning_rate": 9.932975871313673e-07, - "loss": 0.010704033076763153, - "num_tokens": 5258365.0, - "reward": 0.5625, - "reward_std": 0.4045617878437042, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.4045617878437042, - "step": 61, - "step_time": 198.50113070290536 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.5, - "completions/max_terminated_length": 210.5, - "completions/mean_length": 167.984375, - "completions/mean_terminated_length": 167.984375, - "completions/min_length": 128.75, - "completions/min_terminated_length": 128.75, - "entropy": 1.0178060978651047, - "epoch": 0.020778350299526623, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.377078056335449, - "learning_rate": 9.931858802502234e-07, - "loss": 0.026623107492923737, - "num_tokens": 5342956.0, - "reward": 0.390625, - "reward_std": 0.3492846116423607, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.3492846116423607, - "step": 62, - "step_time": 214.5232614125125 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 200.75, - "completions/max_terminated_length": 200.75, - "completions/mean_length": 150.6875, - "completions/mean_terminated_length": 150.6875, - "completions/min_length": 87.375, - "completions/min_terminated_length": 87.375, - "entropy": 0.8333362564444542, - "epoch": 0.02111348498177705, - "frac_reward_zero_std": 0.125, - "grad_norm": 4.844173908233643, - "learning_rate": 9.930741733690794e-07, - "loss": -0.08832010626792908, - "num_tokens": 5427496.0, - "reward": 0.5, - "reward_std": 0.42081795260310173, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.42081795260310173, - "step": 63, - "step_time": 187.64319113083184 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 256.5, - "completions/max_terminated_length": 256.5, - "completions/mean_length": 195.625, - "completions/mean_terminated_length": 195.625, - "completions/min_length": 143.375, - "completions/min_terminated_length": 143.375, - "entropy": 1.0594770833849907, - "epoch": 0.02144861966402748, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.9388766288757324, - "learning_rate": 9.929624664879355e-07, - "loss": 0.015371249988675117, - "num_tokens": 5512496.0, - "reward": 0.421875, - "reward_std": 0.318760022521019, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.318760022521019, - "step": 64, - "step_time": 237.41667456598952 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 200.5, - "completions/max_terminated_length": 200.5, - "completions/mean_length": 150.65625, - "completions/mean_terminated_length": 150.65625, - "completions/min_length": 83.75, - "completions/min_terminated_length": 83.75, - "entropy": 0.7539147213101387, - "epoch": 0.02178375434627791, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.464312553405762, - "learning_rate": 9.928507596067918e-07, - "loss": 0.0008958950638771057, - "num_tokens": 5595842.0, - "reward": 0.34375, - "reward_std": 0.2756394073367119, - "rewards/embodied_reward/mean": 0.34375, - "rewards/embodied_reward/std": 0.2756394073367119, - "step": 65, - "step_time": 191.98809132073075 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 199.125, - "completions/max_terminated_length": 199.125, - "completions/mean_length": 159.078125, - "completions/mean_terminated_length": 159.078125, - "completions/min_length": 113.25, - "completions/min_terminated_length": 113.25, - "entropy": 0.8307245783507824, - "epoch": 0.02211888902852834, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.4932355880737305, - "learning_rate": 9.927390527256479e-07, - "loss": 0.006950791925191879, - "num_tokens": 5681847.0, - "reward": 0.578125, - "reward_std": 0.4276478476822376, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.4276478476822376, - "step": 66, - "step_time": 186.08901463449 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 233.875, - "completions/max_terminated_length": 233.875, - "completions/mean_length": 189.84375, - "completions/mean_terminated_length": 189.84375, - "completions/min_length": 132.0, - "completions/min_terminated_length": 132.0, - "entropy": 1.0786075219511986, - "epoch": 0.02245402371077877, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.9902422428131104, - "learning_rate": 9.92627345844504e-07, - "loss": 0.020965784788131714, - "num_tokens": 5770749.0, - "reward": 0.40625, - "reward_std": 0.29143064841628075, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.29143064841628075, - "step": 67, - "step_time": 223.2375891925767 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 224.875, - "completions/max_terminated_length": 224.875, - "completions/mean_length": 177.546875, - "completions/mean_terminated_length": 177.546875, - "completions/min_length": 129.0, - "completions/min_terminated_length": 129.0, - "entropy": 0.9957886263728142, - "epoch": 0.0227891583930292, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.748973846435547, - "learning_rate": 9.9251563896336e-07, - "loss": 0.11500003188848495, - "num_tokens": 5858576.0, - "reward": 0.34375, - "reward_std": 0.3540026433765888, - "rewards/embodied_reward/mean": 0.34375, - "rewards/embodied_reward/std": 0.3540026433765888, - "step": 68, - "step_time": 213.66483664978296 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 237.375, - "completions/max_terminated_length": 237.375, - "completions/mean_length": 185.890625, - "completions/mean_terminated_length": 185.890625, - "completions/min_length": 133.0, - "completions/min_terminated_length": 133.0, - "entropy": 0.9603854268789291, - "epoch": 0.023124293075279627, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.954776287078857, - "learning_rate": 9.92403932082216e-07, - "loss": -0.007677771151065826, - "num_tokens": 5946569.0, - "reward": 0.640625, - "reward_std": 0.41398805752396584, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.41398805752396584, - "step": 69, - "step_time": 222.69557159300894 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.0, - "completions/max_terminated_length": 210.0, - "completions/mean_length": 174.40625, - "completions/mean_terminated_length": 174.40625, - "completions/min_length": 120.25, - "completions/min_terminated_length": 120.25, - "entropy": 0.9372003450989723, - "epoch": 0.023459427757530057, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.32116174697876, - "learning_rate": 9.922922252010724e-07, - "loss": -0.020711228251457214, - "num_tokens": 6035595.0, - "reward": 0.625, - "reward_std": 0.34716295450925827, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.34716295450925827, - "step": 70, - "step_time": 199.17423324426636 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 189.25, - "completions/max_terminated_length": 189.25, - "completions/mean_length": 148.8125, - "completions/mean_terminated_length": 148.8125, - "completions/min_length": 91.625, - "completions/min_terminated_length": 91.625, - "entropy": 0.7551050782203674, - "epoch": 0.023794562439780488, - "frac_reward_zero_std": 0.25, - "grad_norm": 8.986921310424805, - "learning_rate": 9.921805183199284e-07, - "loss": 0.020300909876823425, - "num_tokens": 6121079.0, - "reward": 0.515625, - "reward_std": 0.3445763736963272, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.3445763736963272, - "step": 71, - "step_time": 191.6679937769659 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.125, - "completions/max_terminated_length": 211.125, - "completions/mean_length": 165.109375, - "completions/mean_terminated_length": 165.109375, - "completions/min_length": 122.125, - "completions/min_terminated_length": 122.125, - "entropy": 0.9408272057771683, - "epoch": 0.024129697122030915, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.95114803314209, - "learning_rate": 9.920688114387847e-07, - "loss": 0.029330603778362274, - "num_tokens": 6205694.0, - "reward": 0.671875, - "reward_std": 0.398196816444397, - "rewards/embodied_reward/mean": 0.671875, - "rewards/embodied_reward/std": 0.398196816444397, - "step": 72, - "step_time": 203.05277393152937 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 209.875, - "completions/max_terminated_length": 209.875, - "completions/mean_length": 178.21875, - "completions/mean_terminated_length": 178.21875, - "completions/min_length": 136.25, - "completions/min_terminated_length": 136.25, - "entropy": 0.9317203685641289, - "epoch": 0.024464831804281346, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.812159538269043, - "learning_rate": 9.919571045576408e-07, - "loss": 0.004898030310869217, - "num_tokens": 6298412.0, - "reward": 0.578125, - "reward_std": 0.3356248214840889, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.3356248214840889, - "step": 73, - "step_time": 199.14966173423454 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 207.0, - "completions/max_terminated_length": 207.0, - "completions/mean_length": 166.515625, - "completions/mean_terminated_length": 166.515625, - "completions/min_length": 119.875, - "completions/min_terminated_length": 119.875, - "entropy": 0.8411936052143574, - "epoch": 0.024799966486531776, - "frac_reward_zero_std": 0.375, - "grad_norm": 10.319972038269043, - "learning_rate": 9.918453976764968e-07, - "loss": 0.0018398091197013855, - "num_tokens": 6386021.0, - "reward": 0.421875, - "reward_std": 0.30721209570765495, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.30721209570765495, - "step": 74, - "step_time": 197.81971333129331 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 207.875, - "completions/max_terminated_length": 207.875, - "completions/mean_length": 161.71875, - "completions/mean_terminated_length": 161.71875, - "completions/min_length": 108.0, - "completions/min_terminated_length": 108.0, - "entropy": 0.9432130604982376, - "epoch": 0.025135101168782203, - "frac_reward_zero_std": 0.5, - "grad_norm": 7.760217189788818, - "learning_rate": 9.91733690795353e-07, - "loss": -0.052141062915325165, - "num_tokens": 6470571.0, - "reward": 0.703125, - "reward_std": 0.2608962655067444, - "rewards/embodied_reward/mean": 0.703125, - "rewards/embodied_reward/std": 0.2608962655067444, - "step": 75, - "step_time": 194.0219321786426 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 209.375, - "completions/max_terminated_length": 209.375, - "completions/mean_length": 170.96875, - "completions/mean_terminated_length": 170.96875, - "completions/min_length": 116.75, - "completions/min_terminated_length": 116.75, - "entropy": 0.8571877591311932, - "epoch": 0.025470235851032634, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.1056318283081055, - "learning_rate": 9.91621983914209e-07, - "loss": -0.04306042939424515, - "num_tokens": 6557513.0, - "reward": 0.328125, - "reward_std": 0.3751009628176689, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.3751009628176689, - "step": 76, - "step_time": 209.57417521765456 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 248.25, - "completions/max_terminated_length": 248.25, - "completions/mean_length": 199.703125, - "completions/mean_terminated_length": 199.703125, - "completions/min_length": 155.375, - "completions/min_terminated_length": 155.375, - "entropy": 1.1159206479787827, - "epoch": 0.025805370533283065, - "frac_reward_zero_std": 0.0, - "grad_norm": 13.611446380615234, - "learning_rate": 9.915102770330653e-07, - "loss": -0.025778450071811676, - "num_tokens": 6644846.0, - "reward": 0.515625, - "reward_std": 0.48080336675047874, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.48080336675047874, - "step": 77, - "step_time": 235.0922355405055 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 204.625, - "completions/max_terminated_length": 204.625, - "completions/mean_length": 161.515625, - "completions/mean_terminated_length": 161.515625, - "completions/min_length": 107.375, - "completions/min_terminated_length": 107.375, - "entropy": 0.8515128120779991, - "epoch": 0.026140505215533492, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.986156940460205, - "learning_rate": 9.913985701519213e-07, - "loss": -0.04421810805797577, - "num_tokens": 6724791.0, - "reward": 0.765625, - "reward_std": 0.3403330594301224, - "rewards/embodied_reward/mean": 0.765625, - "rewards/embodied_reward/std": 0.3403330594301224, - "step": 78, - "step_time": 188.41672952007502 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.25, - "completions/max_terminated_length": 211.25, - "completions/mean_length": 165.453125, - "completions/mean_terminated_length": 165.453125, - "completions/min_length": 113.0, - "completions/min_terminated_length": 113.0, - "entropy": 0.9038165584206581, - "epoch": 0.026475639897783922, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.6946797370910645, - "learning_rate": 9.912868632707774e-07, - "loss": 0.043395332992076874, - "num_tokens": 6813828.0, - "reward": 0.734375, - "reward_std": 0.28460075333714485, - "rewards/embodied_reward/mean": 0.734375, - "rewards/embodied_reward/std": 0.28460075333714485, - "step": 79, - "step_time": 209.70491663040593 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 219.75, - "completions/max_terminated_length": 219.75, - "completions/mean_length": 174.40625, - "completions/mean_terminated_length": 174.40625, - "completions/min_length": 126.0, - "completions/min_terminated_length": 126.0, - "entropy": 0.9217192307114601, - "epoch": 0.026810774580034353, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.2781400680542, - "learning_rate": 9.911751563896335e-07, - "loss": 0.06183397397398949, - "num_tokens": 6894070.0, - "reward": 0.515625, - "reward_std": 0.42293960973620415, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.42293960973620415, - "step": 80, - "step_time": 206.75619478616863 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 229.625, - "completions/max_terminated_length": 229.625, - "completions/mean_length": 186.4375, - "completions/mean_terminated_length": 186.4375, - "completions/min_length": 135.875, - "completions/min_terminated_length": 135.875, - "entropy": 0.9810127504169941, - "epoch": 0.02714590926228478, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.398207664489746, - "learning_rate": 9.910634495084895e-07, - "loss": -0.030881110578775406, - "num_tokens": 6986762.0, - "reward": 0.5, - "reward_std": 0.3119301274418831, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.3119301274418831, - "step": 81, - "step_time": 216.07624338474125 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 184.25, - "completions/max_terminated_length": 184.25, - "completions/mean_length": 144.96875, - "completions/mean_terminated_length": 144.96875, - "completions/min_length": 86.25, - "completions/min_terminated_length": 86.25, - "entropy": 0.7796471565961838, - "epoch": 0.02748104394453521, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.205010414123535, - "learning_rate": 9.909517426273458e-07, - "loss": 0.002061324194073677, - "num_tokens": 7073352.0, - "reward": 0.65625, - "reward_std": 0.4092700257897377, - "rewards/embodied_reward/mean": 0.65625, - "rewards/embodied_reward/std": 0.4092700257897377, - "step": 82, - "step_time": 180.68970756977797 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 256.0, - "completions/max_terminated_length": 256.0, - "completions/mean_length": 189.640625, - "completions/mean_terminated_length": 189.640625, - "completions/min_length": 139.125, - "completions/min_terminated_length": 139.125, - "entropy": 0.980815589427948, - "epoch": 0.02781617862678564, - "frac_reward_zero_std": 0.125, - "grad_norm": 23.006103515625, - "learning_rate": 9.90840035746202e-07, - "loss": -0.025218235328793526, - "num_tokens": 7165777.0, - "reward": 0.5625, - "reward_std": 0.42081795260310173, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.42081795260310173, - "step": 83, - "step_time": 238.36746614379808 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 229.5, - "completions/max_terminated_length": 229.5, - "completions/mean_length": 184.609375, - "completions/mean_terminated_length": 184.609375, - "completions/min_length": 139.625, - "completions/min_terminated_length": 139.625, - "entropy": 1.0197813585400581, - "epoch": 0.02815131330903607, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.2557783126831055, - "learning_rate": 9.907283288650582e-07, - "loss": -0.04022667929530144, - "num_tokens": 7253272.0, - "reward": 0.46875, - "reward_std": 0.32195523753762245, - "rewards/embodied_reward/mean": 0.46875, - "rewards/embodied_reward/std": 0.32195523753762245, - "step": 84, - "step_time": 215.34704032959417 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 226.625, - "completions/max_terminated_length": 226.625, - "completions/mean_length": 184.203125, - "completions/mean_terminated_length": 184.203125, - "completions/min_length": 134.625, - "completions/min_terminated_length": 134.625, - "entropy": 1.0019407831132412, - "epoch": 0.0284864479912865, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.050793647766113, - "learning_rate": 9.906166219839142e-07, - "loss": 0.015175912529230118, - "num_tokens": 7336885.0, - "reward": 0.53125, - "reward_std": 0.34929440543055534, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.34929440543055534, - "step": 85, - "step_time": 221.63577098492533 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 228.625, - "completions/max_terminated_length": 228.625, - "completions/mean_length": 190.546875, - "completions/mean_terminated_length": 190.546875, - "completions/min_length": 153.0, - "completions/min_terminated_length": 153.0, - "entropy": 0.9781728237867355, - "epoch": 0.028821582673536926, - "frac_reward_zero_std": 0.5, - "grad_norm": 8.575288772583008, - "learning_rate": 9.905049151027703e-07, - "loss": -0.029037674888968468, - "num_tokens": 7430968.0, - "reward": 0.296875, - "reward_std": 0.23144523426890373, - "rewards/embodied_reward/mean": 0.296875, - "rewards/embodied_reward/std": 0.23144523426890373, - "step": 86, - "step_time": 220.05865199910477 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.875, - "completions/max_terminated_length": 220.875, - "completions/mean_length": 183.28125, - "completions/mean_terminated_length": 183.28125, - "completions/min_length": 130.125, - "completions/min_terminated_length": 130.125, - "entropy": 0.9374861568212509, - "epoch": 0.029156717355787357, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.907240867614746, - "learning_rate": 9.903932082216264e-07, - "loss": -0.020480964332818985, - "num_tokens": 7518146.0, - "reward": 0.609375, - "reward_std": 0.318760022521019, - "rewards/embodied_reward/mean": 0.609375, - "rewards/embodied_reward/std": 0.318760022521019, - "step": 87, - "step_time": 212.32184752402827 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 202.375, - "completions/max_terminated_length": 202.375, - "completions/mean_length": 162.171875, - "completions/mean_terminated_length": 162.171875, - "completions/min_length": 110.875, - "completions/min_terminated_length": 110.875, - "entropy": 0.8426440693438053, - "epoch": 0.029491852038037787, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.96895694732666, - "learning_rate": 9.902815013404825e-07, - "loss": -0.005457162857055664, - "num_tokens": 7606741.0, - "reward": 0.578125, - "reward_std": 0.308285653591156, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.308285653591156, - "step": 88, - "step_time": 193.5487476317212 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 191.625, - "completions/max_terminated_length": 191.625, - "completions/mean_length": 152.921875, - "completions/mean_terminated_length": 152.921875, - "completions/min_length": 85.0, - "completions/min_terminated_length": 85.0, - "entropy": 0.8070343993604183, - "epoch": 0.029826986720288214, - "frac_reward_zero_std": 0.375, - "grad_norm": 31.337604522705078, - "learning_rate": 9.901697944593387e-07, - "loss": -0.004365298897027969, - "num_tokens": 7691480.0, - "reward": 0.375, - "reward_std": 0.30509043857455254, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.30509043857455254, - "step": 89, - "step_time": 190.32073785597458 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 245.5, - "completions/max_terminated_length": 245.5, - "completions/mean_length": 208.578125, - "completions/mean_terminated_length": 208.578125, - "completions/min_length": 171.125, - "completions/min_terminated_length": 171.125, - "entropy": 1.1080514267086983, - "epoch": 0.030162121402538645, - "frac_reward_zero_std": 0.25, - "grad_norm": 8.556970596313477, - "learning_rate": 9.900580875781948e-07, - "loss": 0.026748180389404297, - "num_tokens": 7780005.0, - "reward": 0.34375, - "reward_std": 0.35824595764279366, - "rewards/embodied_reward/mean": 0.34375, - "rewards/embodied_reward/std": 0.35824595764279366, - "step": 90, - "step_time": 227.7664185743779 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 231.375, - "completions/max_terminated_length": 231.375, - "completions/mean_length": 192.953125, - "completions/mean_terminated_length": 192.953125, - "completions/min_length": 152.375, - "completions/min_terminated_length": 152.375, - "entropy": 1.046707570552826, - "epoch": 0.030497256084789075, - "frac_reward_zero_std": 0.5, - "grad_norm": 6.1478400230407715, - "learning_rate": 9.899463806970509e-07, - "loss": 0.00112919881939888, - "num_tokens": 7869082.0, - "reward": 0.53125, - "reward_std": 0.24511481821537018, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.24511481821537018, - "step": 91, - "step_time": 220.76753175677732 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 230.75, - "completions/max_terminated_length": 230.75, - "completions/mean_length": 184.421875, - "completions/mean_terminated_length": 184.421875, - "completions/min_length": 130.625, - "completions/min_terminated_length": 130.625, - "entropy": 0.9702646285295486, - "epoch": 0.030832390767039503, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.907110214233398, - "learning_rate": 9.89834673815907e-07, - "loss": -0.03570472076535225, - "num_tokens": 7955277.0, - "reward": 0.484375, - "reward_std": 0.35612430050969124, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.35612430050969124, - "step": 92, - "step_time": 217.85750008001924 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 217.625, - "completions/max_terminated_length": 217.625, - "completions/mean_length": 182.703125, - "completions/mean_terminated_length": 182.703125, - "completions/min_length": 147.625, - "completions/min_terminated_length": 147.625, - "entropy": 0.956513024866581, - "epoch": 0.031167525449289933, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.574207782745361, - "learning_rate": 9.89722966934763e-07, - "loss": -0.007923200726509094, - "num_tokens": 8047114.0, - "reward": 0.375, - "reward_std": 0.2346404492855072, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.2346404492855072, - "step": 93, - "step_time": 206.50197992520407 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.25, - "completions/max_terminated_length": 196.25, - "completions/mean_length": 161.765625, - "completions/mean_terminated_length": 161.765625, - "completions/min_length": 128.75, - "completions/min_terminated_length": 128.75, - "entropy": 0.8043352290987968, - "epoch": 0.031502660131540364, - "frac_reward_zero_std": 0.5, - "grad_norm": 3.4580090045928955, - "learning_rate": 9.896112600536193e-07, - "loss": -0.006197340786457062, - "num_tokens": 8130683.0, - "reward": 0.46875, - "reward_std": 0.22461533918976784, - "rewards/embodied_reward/mean": 0.46875, - "rewards/embodied_reward/std": 0.22461533918976784, - "step": 94, - "step_time": 183.87887805281207 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.125, - "completions/max_terminated_length": 211.125, - "completions/mean_length": 180.625, - "completions/mean_terminated_length": 180.625, - "completions/min_length": 141.75, - "completions/min_terminated_length": 141.75, - "entropy": 0.9359825924038887, - "epoch": 0.031837794813790794, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.559009552001953, - "learning_rate": 9.894995531724754e-07, - "loss": 0.04006478190422058, - "num_tokens": 8222299.0, - "reward": 0.359375, - "reward_std": 0.28930899128317833, - "rewards/embodied_reward/mean": 0.359375, - "rewards/embodied_reward/std": 0.28930899128317833, - "step": 95, - "step_time": 204.72844316298142 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.125, - "completions/max_terminated_length": 220.125, - "completions/mean_length": 185.0, - "completions/mean_terminated_length": 185.0, - "completions/min_length": 142.75, - "completions/min_terminated_length": 142.75, - "entropy": 0.9245229177176952, - "epoch": 0.032172929496041225, - "frac_reward_zero_std": 0.125, - "grad_norm": 4.620196342468262, - "learning_rate": 9.893878462913314e-07, - "loss": -0.01407700777053833, - "num_tokens": 8307275.0, - "reward": 0.640625, - "reward_std": 0.4113916829228401, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.4113916829228401, - "step": 96, - "step_time": 206.49908239534125 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.875, - "completions/max_terminated_length": 211.875, - "completions/mean_length": 176.703125, - "completions/mean_terminated_length": 176.703125, - "completions/min_length": 133.375, - "completions/min_terminated_length": 133.375, - "entropy": 0.8906809911131859, - "epoch": 0.03250806417829165, - "frac_reward_zero_std": 0.0, - "grad_norm": 10.02591609954834, - "learning_rate": 9.892761394101877e-07, - "loss": 0.011706829071044922, - "num_tokens": 8388792.0, - "reward": 0.4375, - "reward_std": 0.4492306783795357, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.4492306783795357, - "step": 97, - "step_time": 204.16613153694198 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 236.0, - "completions/max_terminated_length": 236.0, - "completions/mean_length": 187.6875, - "completions/mean_terminated_length": 187.6875, - "completions/min_length": 133.25, - "completions/min_terminated_length": 133.25, - "entropy": 0.9518446177244186, - "epoch": 0.03284319886054208, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.2588419914245605, - "learning_rate": 9.891644325290438e-07, - "loss": 0.026919275522232056, - "num_tokens": 8469580.0, - "reward": 0.5, - "reward_std": 0.30038220062851906, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.30038220062851906, - "step": 98, - "step_time": 213.8572158915922 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 221.125, - "completions/max_terminated_length": 221.125, - "completions/mean_length": 183.203125, - "completions/mean_terminated_length": 183.203125, - "completions/min_length": 133.75, - "completions/min_terminated_length": 133.75, - "entropy": 0.9317264258861542, - "epoch": 0.03317833354279251, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.213716506958008, - "learning_rate": 9.890527256478999e-07, - "loss": 0.02925826609134674, - "num_tokens": 8553161.0, - "reward": 0.640625, - "reward_std": 0.38664888963103294, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.38664888963103294, - "step": 99, - "step_time": 210.57451188610867 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 216.125, - "completions/max_terminated_length": 216.125, - "completions/mean_length": 172.125, - "completions/mean_terminated_length": 172.125, - "completions/min_length": 131.625, - "completions/min_terminated_length": 131.625, - "entropy": 0.8997547477483749, - "epoch": 0.03351346822504294, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.632416248321533, - "learning_rate": 9.88941018766756e-07, - "loss": 0.01743157207965851, - "num_tokens": 8638017.0, - "reward": 0.609375, - "reward_std": 0.3403330594301224, - "rewards/embodied_reward/mean": 0.609375, - "rewards/embodied_reward/std": 0.3403330594301224, - "step": 100, - "step_time": 198.48028217069805 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 241.625, - "completions/max_terminated_length": 241.625, - "completions/mean_length": 194.265625, - "completions/mean_terminated_length": 194.265625, - "completions/min_length": 155.375, - "completions/min_terminated_length": 155.375, - "entropy": 1.0804472789168358, - "epoch": 0.03384860290729337, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.7198073863983154, - "learning_rate": 9.888293118856122e-07, - "loss": 0.004325300455093384, - "num_tokens": 8724642.0, - "reward": 0.375, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 101, - "step_time": 218.62928325030953 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.75, - "completions/max_terminated_length": 196.75, - "completions/mean_length": 168.53125, - "completions/mean_terminated_length": 168.53125, - "completions/min_length": 139.25, - "completions/min_terminated_length": 139.25, - "entropy": 0.8299813941121101, - "epoch": 0.034183737589543794, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.734699249267578, - "learning_rate": 9.887176050044683e-07, - "loss": -0.02203398570418358, - "num_tokens": 8807764.0, - "reward": 0.5, - "reward_std": 0.1462521031498909, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.1462521031498909, - "step": 102, - "step_time": 193.44972557481378 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.375, - "completions/max_terminated_length": 210.375, - "completions/mean_length": 171.34375, - "completions/mean_terminated_length": 171.34375, - "completions/min_length": 130.875, - "completions/min_terminated_length": 130.875, - "entropy": 0.8697973489761353, - "epoch": 0.034518872271794225, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.3509721755981445, - "learning_rate": 9.886058981233243e-07, - "loss": 0.08508220314979553, - "num_tokens": 8896610.0, - "reward": 0.484375, - "reward_std": 0.3934885784983635, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.3934885784983635, - "step": 103, - "step_time": 197.23978856252506 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 204.5, - "completions/max_terminated_length": 204.5, - "completions/mean_length": 165.984375, - "completions/mean_terminated_length": 165.984375, - "completions/min_length": 103.375, - "completions/min_terminated_length": 103.375, - "entropy": 0.8705656081438065, - "epoch": 0.034854006954044656, - "frac_reward_zero_std": 0.0, - "grad_norm": 6.983897686004639, - "learning_rate": 9.884941912421804e-07, - "loss": 0.01908813789486885, - "num_tokens": 8981265.0, - "reward": 0.546875, - "reward_std": 0.4902198426425457, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.4902198426425457, - "step": 104, - "step_time": 190.66913616657257 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.078125, - "completions/max_length": 328.375, - "completions/max_terminated_length": 253.625, - "completions/mean_length": 212.703125, - "completions/mean_terminated_length": 183.8052101135254, - "completions/min_length": 129.75, - "completions/min_terminated_length": 129.75, - "entropy": 0.9399808011949062, - "epoch": 0.035189141636295086, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.7466864585876465, - "learning_rate": 9.883824843610365e-07, - "loss": 0.0831226110458374, - "num_tokens": 9076302.0, - "reward": 0.4375, - "reward_std": 0.30038220062851906, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.30038220062851906, - "step": 105, - "step_time": 289.55810589902103 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 223.25, - "completions/max_terminated_length": 223.25, - "completions/mean_length": 191.5, - "completions/mean_terminated_length": 191.5, - "completions/min_length": 149.75, - "completions/min_terminated_length": 149.75, - "entropy": 1.0472619011998177, - "epoch": 0.03552427631854552, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.979881286621094, - "learning_rate": 9.882707774798928e-07, - "loss": 0.015450619161128998, - "num_tokens": 9166062.0, - "reward": 0.515625, - "reward_std": 0.3845370262861252, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.3845370262861252, - "step": 106, - "step_time": 217.16940369829535 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 234.75, - "completions/max_terminated_length": 234.75, - "completions/mean_length": 192.78125, - "completions/mean_terminated_length": 192.78125, - "completions/min_length": 142.125, - "completions/min_terminated_length": 142.125, - "entropy": 1.0660487785935402, - "epoch": 0.03585941100079595, - "frac_reward_zero_std": 0.0, - "grad_norm": 9.938304901123047, - "learning_rate": 9.881590705987488e-07, - "loss": -0.025193262845277786, - "num_tokens": 9252528.0, - "reward": 0.484375, - "reward_std": 0.45346419885754585, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.45346419885754585, - "step": 107, - "step_time": 212.00460273167118 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 181.5, - "completions/max_terminated_length": 181.5, - "completions/mean_length": 148.859375, - "completions/mean_terminated_length": 148.859375, - "completions/min_length": 89.25, - "completions/min_terminated_length": 89.25, - "entropy": 0.7694782540202141, - "epoch": 0.03619454568304637, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.514969348907471, - "learning_rate": 9.88047363717605e-07, - "loss": -0.0032337866723537445, - "num_tokens": 9341151.0, - "reward": 0.515625, - "reward_std": 0.28930899128317833, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.28930899128317833, - "step": 108, - "step_time": 176.81772552942857 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 238.625, - "completions/max_terminated_length": 238.625, - "completions/mean_length": 203.34375, - "completions/mean_terminated_length": 203.34375, - "completions/min_length": 162.875, - "completions/min_terminated_length": 162.875, - "entropy": 1.0209926925599575, - "epoch": 0.0365296803652968, - "frac_reward_zero_std": 0.125, - "grad_norm": 11.942022323608398, - "learning_rate": 9.87935656836461e-07, - "loss": -0.003876522183418274, - "num_tokens": 9432005.0, - "reward": 0.375, - "reward_std": 0.4045617878437042, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.4045617878437042, - "step": 109, - "step_time": 219.50001886766404 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 246.625, - "completions/max_terminated_length": 246.625, - "completions/mean_length": 208.703125, - "completions/mean_terminated_length": 208.703125, - "completions/min_length": 174.875, - "completions/min_terminated_length": 174.875, - "entropy": 1.1984052136540413, - "epoch": 0.03686481504754723, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.9616780281066895, - "learning_rate": 9.87823949955317e-07, - "loss": -0.017085345461964607, - "num_tokens": 9523794.0, - "reward": 0.484375, - "reward_std": 0.42293960973620415, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.42293960973620415, - "step": 110, - "step_time": 239.3317134794779 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.875, - "completions/max_terminated_length": 220.875, - "completions/mean_length": 183.109375, - "completions/mean_terminated_length": 183.109375, - "completions/min_length": 134.75, - "completions/min_terminated_length": 134.75, - "entropy": 1.0686755552887917, - "epoch": 0.03719994972979766, - "frac_reward_zero_std": 0.75, - "grad_norm": 1.8419451713562012, - "learning_rate": 9.877122430741733e-07, - "loss": -0.015778914093971252, - "num_tokens": 9611729.0, - "reward": 0.390625, - "reward_std": 0.10205793008208275, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.10205793008208275, - "step": 111, - "step_time": 207.77252494171262 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 216.5, - "completions/max_terminated_length": 216.5, - "completions/mean_length": 184.09375, - "completions/mean_terminated_length": 184.09375, - "completions/min_length": 154.125, - "completions/min_terminated_length": 154.125, - "entropy": 0.9815471097826958, - "epoch": 0.03753508441204809, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.5624022483825684, - "learning_rate": 9.876005361930294e-07, - "loss": -0.016217069700360298, - "num_tokens": 9697015.0, - "reward": 0.390625, - "reward_std": 0.29355230554938316, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.29355230554938316, - "step": 112, - "step_time": 201.98684726655483 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 241.375, - "completions/max_terminated_length": 241.375, - "completions/mean_length": 205.5, - "completions/mean_terminated_length": 205.5, - "completions/min_length": 177.75, - "completions/min_terminated_length": 177.75, - "entropy": 1.0837717577815056, - "epoch": 0.037870219094298524, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.466866970062256, - "learning_rate": 9.874888293118857e-07, - "loss": 0.007086925208568573, - "num_tokens": 9783959.0, - "reward": 0.453125, - "reward_std": 0.2824693024158478, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.2824693024158478, - "step": 113, - "step_time": 223.83294124295935 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 221.0, - "completions/max_terminated_length": 221.0, - "completions/mean_length": 176.6875, - "completions/mean_terminated_length": 176.6875, - "completions/min_length": 129.125, - "completions/min_terminated_length": 129.125, - "entropy": 0.9399269595742226, - "epoch": 0.03820535377654895, - "frac_reward_zero_std": 0.125, - "grad_norm": 9.149439811706543, - "learning_rate": 9.873771224307418e-07, - "loss": 0.05056214705109596, - "num_tokens": 9867755.0, - "reward": 0.453125, - "reward_std": 0.3845370262861252, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.3845370262861252, - "step": 114, - "step_time": 207.43934958940372 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.25, - "completions/max_terminated_length": 210.25, - "completions/mean_length": 171.65625, - "completions/mean_terminated_length": 171.65625, - "completions/min_length": 126.625, - "completions/min_terminated_length": 126.625, - "entropy": 0.9491228461265564, - "epoch": 0.03854048845879938, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.922851085662842, - "learning_rate": 9.872654155495978e-07, - "loss": 0.02142062783241272, - "num_tokens": 9958973.0, - "reward": 0.203125, - "reward_std": 0.2414703443646431, - "rewards/embodied_reward/mean": 0.203125, - "rewards/embodied_reward/std": 0.2414703443646431, - "step": 115, - "step_time": 199.9031821480021 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 199.375, - "completions/max_terminated_length": 199.375, - "completions/mean_length": 160.953125, - "completions/mean_terminated_length": 160.953125, - "completions/min_length": 102.25, - "completions/min_terminated_length": 102.25, - "entropy": 0.8263540603220463, - "epoch": 0.03887562314104981, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.9248952865600586, - "learning_rate": 9.871537086684539e-07, - "loss": -0.015141937881708145, - "num_tokens": 10039002.0, - "reward": 0.828125, - "reward_std": 0.2824693024158478, - "rewards/embodied_reward/mean": 0.828125, - "rewards/embodied_reward/std": 0.2824693024158478, - "step": 116, - "step_time": 186.60126484185457 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 194.875, - "completions/max_terminated_length": 194.875, - "completions/mean_length": 152.640625, - "completions/mean_terminated_length": 152.640625, - "completions/min_length": 97.625, - "completions/min_terminated_length": 97.625, - "entropy": 0.7522408217191696, - "epoch": 0.03921075782330024, - "frac_reward_zero_std": 0.5, - "grad_norm": 5.210011959075928, - "learning_rate": 9.8704200178731e-07, - "loss": -0.005578596144914627, - "num_tokens": 10122451.0, - "reward": 0.625, - "reward_std": 0.2177756503224373, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.2177756503224373, - "step": 117, - "step_time": 189.74704708578065 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 204.0, - "completions/max_terminated_length": 204.0, - "completions/mean_length": 161.046875, - "completions/mean_terminated_length": 161.046875, - "completions/min_length": 116.375, - "completions/min_terminated_length": 116.375, - "entropy": 0.8503235131502151, - "epoch": 0.03954589250555067, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.865120887756348, - "learning_rate": 9.869302949061662e-07, - "loss": -0.015861500054597855, - "num_tokens": 10206942.0, - "reward": 0.546875, - "reward_std": 0.22673699632287025, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.22673699632287025, - "step": 118, - "step_time": 194.2439356436953 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.5, - "completions/max_terminated_length": 197.5, - "completions/mean_length": 158.296875, - "completions/mean_terminated_length": 158.296875, - "completions/min_length": 105.625, - "completions/min_terminated_length": 105.625, - "entropy": 0.8116026408970356, - "epoch": 0.0398810271878011, - "frac_reward_zero_std": 0.5, - "grad_norm": 19.592626571655273, - "learning_rate": 9.868185880250223e-07, - "loss": 0.027666009962558746, - "num_tokens": 10285945.0, - "reward": 0.515625, - "reward_std": 0.23144523426890373, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.23144523426890373, - "step": 119, - "step_time": 189.67216804251075 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 216.375, - "completions/max_terminated_length": 216.375, - "completions/mean_length": 186.875, - "completions/mean_terminated_length": 186.875, - "completions/min_length": 155.75, - "completions/min_terminated_length": 155.75, - "entropy": 0.9435580670833588, - "epoch": 0.040216161870051524, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.289516925811768, - "learning_rate": 9.867068811438784e-07, - "loss": 0.03222986310720444, - "num_tokens": 10369257.0, - "reward": 0.578125, - "reward_std": 0.2777610644698143, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.2777610644698143, - "step": 120, - "step_time": 198.3163980692625 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 206.75, - "completions/max_terminated_length": 206.75, - "completions/mean_length": 175.671875, - "completions/mean_terminated_length": 175.671875, - "completions/min_length": 139.875, - "completions/min_terminated_length": 139.875, - "entropy": 0.8904099427163601, - "epoch": 0.040551296552301955, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.4009881019592285, - "learning_rate": 9.865951742627345e-07, - "loss": -0.02880970388650894, - "num_tokens": 10458964.0, - "reward": 0.421875, - "reward_std": 0.35141606256365776, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.35141606256365776, - "step": 121, - "step_time": 195.13523684209213 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.125, - "completions/max_terminated_length": 218.125, - "completions/mean_length": 181.71875, - "completions/mean_terminated_length": 181.71875, - "completions/min_length": 132.125, - "completions/min_terminated_length": 132.125, - "entropy": 0.9509338587522507, - "epoch": 0.040886431234552385, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.7731804847717285, - "learning_rate": 9.864834673815905e-07, - "loss": 0.03125741332769394, - "num_tokens": 10547738.0, - "reward": 0.578125, - "reward_std": 0.2777610644698143, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.2777610644698143, - "step": 122, - "step_time": 211.1463377168402 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 203.75, - "completions/max_terminated_length": 203.75, - "completions/mean_length": 171.4375, - "completions/mean_terminated_length": 171.4375, - "completions/min_length": 139.375, - "completions/min_terminated_length": 139.375, - "entropy": 0.8699133098125458, - "epoch": 0.041221565916802816, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.7492146492004395, - "learning_rate": 9.863717605004468e-07, - "loss": -0.006158888339996338, - "num_tokens": 10630286.0, - "reward": 0.40625, - "reward_std": 0.35247982665896416, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.35247982665896416, - "step": 123, - "step_time": 194.94902970036492 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 225.25, - "completions/max_terminated_length": 225.25, - "completions/mean_length": 190.546875, - "completions/mean_terminated_length": 190.546875, - "completions/min_length": 143.375, - "completions/min_terminated_length": 143.375, - "entropy": 0.9575192704796791, - "epoch": 0.041556700599053246, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.868796348571777, - "learning_rate": 9.862600536193029e-07, - "loss": -0.018613725900650024, - "num_tokens": 10722417.0, - "reward": 0.546875, - "reward_std": 0.2961486801505089, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.2961486801505089, - "step": 124, - "step_time": 216.1616349206306 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 202.125, - "completions/max_terminated_length": 202.125, - "completions/mean_length": 165.015625, - "completions/mean_terminated_length": 165.015625, - "completions/min_length": 126.0, - "completions/min_terminated_length": 126.0, - "entropy": 0.8251283094286919, - "epoch": 0.04189183528130368, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.1137168407440186, - "learning_rate": 9.861483467381592e-07, - "loss": 0.006345492787659168, - "num_tokens": 10804242.0, - "reward": 0.46875, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.46875, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 125, - "step_time": 190.0629370426759 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 254.375, - "completions/max_terminated_length": 254.375, - "completions/mean_length": 214.5, - "completions/mean_terminated_length": 214.5, - "completions/min_length": 180.625, - "completions/min_terminated_length": 180.625, - "entropy": 1.1181562840938568, - "epoch": 0.0422269699635541, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.6745829582214355, - "learning_rate": 9.860366398570152e-07, - "loss": 0.004191786982119083, - "num_tokens": 10895130.0, - "reward": 0.734375, - "reward_std": 0.3356248214840889, - "rewards/embodied_reward/mean": 0.734375, - "rewards/embodied_reward/std": 0.3356248214840889, - "step": 126, - "step_time": 241.23223825497553 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 238.75, - "completions/max_terminated_length": 238.75, - "completions/mean_length": 198.640625, - "completions/mean_terminated_length": 198.640625, - "completions/min_length": 162.75, - "completions/min_terminated_length": 162.75, - "entropy": 0.9970330484211445, - "epoch": 0.04256210464580453, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.021557331085205, - "learning_rate": 9.859249329758713e-07, - "loss": -0.0051622893661260605, - "num_tokens": 10977995.0, - "reward": 0.421875, - "reward_std": 0.37769733741879463, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.37769733741879463, - "step": 127, - "step_time": 223.74969706498086 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 213.125, - "completions/max_terminated_length": 213.125, - "completions/mean_length": 180.328125, - "completions/mean_terminated_length": 180.328125, - "completions/min_length": 135.0, - "completions/min_terminated_length": 135.0, - "entropy": 0.8808479160070419, - "epoch": 0.04289723932805496, - "frac_reward_zero_std": 0.125, - "grad_norm": 6.625707626342773, - "learning_rate": 9.858132260947274e-07, - "loss": 0.008904539048671722, - "num_tokens": 11069720.0, - "reward": 0.59375, - "reward_std": 0.42976950481534004, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.42976950481534004, - "step": 128, - "step_time": 197.36914596473798 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.125, - "completions/max_terminated_length": 211.125, - "completions/mean_length": 173.21875, - "completions/mean_terminated_length": 173.21875, - "completions/min_length": 136.625, - "completions/min_terminated_length": 136.625, - "entropy": 0.8288759477436543, - "epoch": 0.04323237401030539, - "frac_reward_zero_std": 0.625, - "grad_norm": 2.503950834274292, - "learning_rate": 9.857015192135834e-07, - "loss": -0.009999081492424011, - "num_tokens": 11151118.0, - "reward": 0.515625, - "reward_std": 0.17358147725462914, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.17358147725462914, - "step": 129, - "step_time": 196.37084928387776 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 240.125, - "completions/max_terminated_length": 240.125, - "completions/mean_length": 198.5, - "completions/mean_terminated_length": 198.5, - "completions/min_length": 159.5, - "completions/min_terminated_length": 159.5, - "entropy": 1.0744335427880287, - "epoch": 0.04356750869255582, - "frac_reward_zero_std": 0.125, - "grad_norm": 9.823932647705078, - "learning_rate": 9.855898123324397e-07, - "loss": -0.016535189002752304, - "num_tokens": 11240286.0, - "reward": 0.265625, - "reward_std": 0.3845370262861252, - "rewards/embodied_reward/mean": 0.265625, - "rewards/embodied_reward/std": 0.3845370262861252, - "step": 130, - "step_time": 229.04733150964603 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 215.25, - "completions/max_terminated_length": 215.25, - "completions/mean_length": 187.0625, - "completions/mean_terminated_length": 187.0625, - "completions/min_length": 158.125, - "completions/min_terminated_length": 158.125, - "entropy": 0.9040943942964077, - "epoch": 0.043902643374806254, - "frac_reward_zero_std": 0.375, - "grad_norm": 16.47975730895996, - "learning_rate": 9.854781054512958e-07, - "loss": 0.0006500370800495148, - "num_tokens": 11325890.0, - "reward": 0.4375, - "reward_std": 0.2845909595489502, - "rewards/embodied_reward/mean": 0.4375, - "rewards/embodied_reward/std": 0.2845909595489502, - "step": 131, - "step_time": 200.2203974914737 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.625, - "completions/max_terminated_length": 196.625, - "completions/mean_length": 169.609375, - "completions/mean_terminated_length": 169.609375, - "completions/min_length": 121.0, - "completions/min_terminated_length": 121.0, - "entropy": 0.8869076743721962, - "epoch": 0.04423777805705668, - "frac_reward_zero_std": 0.5, - "grad_norm": 3.4651167392730713, - "learning_rate": 9.853663985701519e-07, - "loss": -0.02039331942796707, - "num_tokens": 11419801.0, - "reward": 0.21875, - "reward_std": 0.19727617129683495, - "rewards/embodied_reward/mean": 0.21875, - "rewards/embodied_reward/std": 0.19727617129683495, - "step": 132, - "step_time": 191.0084969229065 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 246.875, - "completions/max_terminated_length": 246.875, - "completions/mean_length": 192.171875, - "completions/mean_terminated_length": 192.171875, - "completions/min_length": 147.875, - "completions/min_terminated_length": 147.875, - "entropy": 0.9271059706807137, - "epoch": 0.04457291273930711, - "frac_reward_zero_std": 0.25, - "grad_norm": 6.789852142333984, - "learning_rate": 9.85254691689008e-07, - "loss": 0.011246412992477417, - "num_tokens": 11500340.0, - "reward": 0.421875, - "reward_std": 0.3403330594301224, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.3403330594301224, - "step": 133, - "step_time": 226.05935780750588 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.25, - "completions/max_terminated_length": 210.25, - "completions/mean_length": 181.796875, - "completions/mean_terminated_length": 181.796875, - "completions/min_length": 148.125, - "completions/min_terminated_length": 148.125, - "entropy": 0.9359023831784725, - "epoch": 0.04490804742155754, - "frac_reward_zero_std": 0.5, - "grad_norm": 5.992682933807373, - "learning_rate": 9.851429848078642e-07, - "loss": -0.004808422178030014, - "num_tokens": 11587351.0, - "reward": 0.578125, - "reward_std": 0.23568854853510857, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.23568854853510857, - "step": 134, - "step_time": 194.65654002223164 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 200.25, - "completions/max_terminated_length": 200.25, - "completions/mean_length": 162.6875, - "completions/mean_terminated_length": 162.6875, - "completions/min_length": 118.375, - "completions/min_terminated_length": 118.375, - "entropy": 0.8393921032547951, - "epoch": 0.04524318210380797, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.590486526489258, - "learning_rate": 9.850312779267203e-07, - "loss": 0.008204620331525803, - "num_tokens": 11671595.0, - "reward": 0.640625, - "reward_std": 0.2993341013789177, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.2993341013789177, - "step": 135, - "step_time": 186.0883161877282 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 212.125, - "completions/max_terminated_length": 212.125, - "completions/mean_length": 179.78125, - "completions/mean_terminated_length": 179.78125, - "completions/min_length": 138.625, - "completions/min_terminated_length": 138.625, - "entropy": 0.9086024165153503, - "epoch": 0.0455783167860584, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.536170482635498, - "learning_rate": 9.849195710455763e-07, - "loss": -0.0087600601837039, - "num_tokens": 11754845.0, - "reward": 0.515625, - "reward_std": 0.2777610644698143, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.2777610644698143, - "step": 136, - "step_time": 197.19068260118365 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 192.25, - "completions/max_terminated_length": 192.25, - "completions/mean_length": 165.765625, - "completions/mean_terminated_length": 165.765625, - "completions/min_length": 128.75, - "completions/min_terminated_length": 128.75, - "entropy": 0.7794381082057953, - "epoch": 0.04591345146830882, - "frac_reward_zero_std": 0.625, - "grad_norm": 5.841657638549805, - "learning_rate": 9.848078641644324e-07, - "loss": 0.0435146801173687, - "num_tokens": 11840678.0, - "reward": 0.5625, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 137, - "step_time": 183.12401512498036 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 235.5, - "completions/max_terminated_length": 235.5, - "completions/mean_length": 205.421875, - "completions/mean_terminated_length": 205.421875, - "completions/min_length": 168.125, - "completions/min_terminated_length": 168.125, - "entropy": 1.0118607841432095, - "epoch": 0.046248586150559254, - "frac_reward_zero_std": 0.0, - "grad_norm": 8.494194030761719, - "learning_rate": 9.846961572832887e-07, - "loss": -0.008456829935312271, - "num_tokens": 11928433.0, - "reward": 0.375, - "reward_std": 0.3898441046476364, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.3898441046476364, - "step": 138, - "step_time": 220.0794798238203 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 222.875, - "completions/max_terminated_length": 222.875, - "completions/mean_length": 175.03125, - "completions/mean_terminated_length": 175.03125, - "completions/min_length": 123.375, - "completions/min_terminated_length": 123.375, - "entropy": 0.9574659615755081, - "epoch": 0.046583720832809684, - "frac_reward_zero_std": 0.5, - "grad_norm": 6.827114582061768, - "learning_rate": 9.845844504021448e-07, - "loss": -0.008851293474435806, - "num_tokens": 12012699.0, - "reward": 0.578125, - "reward_std": 0.23144523426890373, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.23144523426890373, - "step": 139, - "step_time": 204.18031763657928 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 236.25, - "completions/max_terminated_length": 236.25, - "completions/mean_length": 199.328125, - "completions/mean_terminated_length": 199.328125, - "completions/min_length": 163.25, - "completions/min_terminated_length": 163.25, - "entropy": 0.9943242222070694, - "epoch": 0.046918855515060115, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.817070007324219, - "learning_rate": 9.844727435210008e-07, - "loss": -0.015749923884868622, - "num_tokens": 12102256.0, - "reward": 0.390625, - "reward_std": 0.29826054349541664, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.29826054349541664, - "step": 140, - "step_time": 217.98478758940473 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 187.0, - "completions/max_terminated_length": 187.0, - "completions/mean_length": 157.53125, - "completions/mean_terminated_length": 157.53125, - "completions/min_length": 108.5, - "completions/min_terminated_length": 108.5, - "entropy": 0.8330435045063496, - "epoch": 0.047253990197310546, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.633111953735352, - "learning_rate": 9.84361036639857e-07, - "loss": 0.025790005922317505, - "num_tokens": 12186986.0, - "reward": 0.3125, - "reward_std": 0.3540026433765888, - "rewards/embodied_reward/mean": 0.3125, - "rewards/embodied_reward/std": 0.3540026433765888, - "step": 141, - "step_time": 178.18095367914066 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 243.125, - "completions/max_terminated_length": 243.125, - "completions/mean_length": 202.4375, - "completions/mean_terminated_length": 202.4375, - "completions/min_length": 157.75, - "completions/min_terminated_length": 157.75, - "entropy": 1.0501012429594994, - "epoch": 0.047589124879560976, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.999776363372803, - "learning_rate": 9.842493297587132e-07, - "loss": 0.006799619644880295, - "num_tokens": 12274902.0, - "reward": 0.5, - "reward_std": 0.3119301274418831, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.3119301274418831, - "step": 142, - "step_time": 224.1898542540148 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 206.5, - "completions/max_terminated_length": 206.5, - "completions/mean_length": 182.296875, - "completions/mean_terminated_length": 182.296875, - "completions/min_length": 157.0, - "completions/min_terminated_length": 157.0, - "entropy": 0.8859607204794884, - "epoch": 0.0479242595618114, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.806723117828369, - "learning_rate": 9.841376228775693e-07, - "loss": 0.009106101468205452, - "num_tokens": 12363993.0, - "reward": 0.625, - "reward_std": 0.2346404492855072, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.2346404492855072, - "step": 143, - "step_time": 205.14150770148262 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 224.625, - "completions/max_terminated_length": 224.625, - "completions/mean_length": 193.546875, - "completions/mean_terminated_length": 193.546875, - "completions/min_length": 158.625, - "completions/min_terminated_length": 158.625, - "entropy": 0.9463705867528915, - "epoch": 0.04825939424406183, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.786468029022217, - "learning_rate": 9.840259159964253e-07, - "loss": -0.008898560889065266, - "num_tokens": 12449508.0, - "reward": 0.5, - "reward_std": 0.4003184735774994, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.4003184735774994, - "step": 144, - "step_time": 208.31075994903222 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 229.75, - "completions/max_terminated_length": 229.75, - "completions/mean_length": 185.375, - "completions/mean_terminated_length": 185.375, - "completions/min_length": 146.375, - "completions/min_terminated_length": 146.375, - "entropy": 0.965535007417202, - "epoch": 0.04859452892631226, - "frac_reward_zero_std": 0.5, - "grad_norm": 6.919014930725098, - "learning_rate": 9.839142091152814e-07, - "loss": -0.02011173591017723, - "num_tokens": 12540708.0, - "reward": 0.375, - "reward_std": 0.23356689140200615, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.23356689140200615, - "step": 145, - "step_time": 222.95655853115022 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.375, - "completions/max_terminated_length": 197.375, - "completions/mean_length": 163.875, - "completions/mean_terminated_length": 163.875, - "completions/min_length": 120.25, - "completions/min_terminated_length": 120.25, - "entropy": 0.7842236869037151, - "epoch": 0.04892966360856269, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.512155055999756, - "learning_rate": 9.838025022341377e-07, - "loss": -0.00862329825758934, - "num_tokens": 12627156.0, - "reward": 0.734375, - "reward_std": 0.3140517845749855, - "rewards/embodied_reward/mean": 0.734375, - "rewards/embodied_reward/std": 0.3140517845749855, - "step": 146, - "step_time": 198.73714059218764 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 214.875, - "completions/max_terminated_length": 214.875, - "completions/mean_length": 181.46875, - "completions/mean_terminated_length": 181.46875, - "completions/min_length": 145.625, - "completions/min_terminated_length": 145.625, - "entropy": 0.9076977670192719, - "epoch": 0.04926479829081312, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.930388927459717, - "learning_rate": 9.836907953529937e-07, - "loss": 0.037006743252277374, - "num_tokens": 12712290.0, - "reward": 0.5, - "reward_std": 0.3845272324979305, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.3845272324979305, - "step": 147, - "step_time": 202.0591345494613 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 227.875, - "completions/max_terminated_length": 227.875, - "completions/mean_length": 194.84375, - "completions/mean_terminated_length": 194.84375, - "completions/min_length": 163.125, - "completions/min_terminated_length": 163.125, - "entropy": 0.9697003066539764, - "epoch": 0.04959993297306355, - "frac_reward_zero_std": 0.625, - "grad_norm": 6.412586688995361, - "learning_rate": 9.835790884718498e-07, - "loss": 0.02391548454761505, - "num_tokens": 12801608.0, - "reward": 0.59375, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 148, - "step_time": 217.07399566099048 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.5, - "completions/max_terminated_length": 210.5, - "completions/mean_length": 173.90625, - "completions/mean_terminated_length": 173.90625, - "completions/min_length": 126.125, - "completions/min_terminated_length": 126.125, - "entropy": 0.9583265632390976, - "epoch": 0.049935067655313976, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.224364757537842, - "learning_rate": 9.834673815907059e-07, - "loss": 0.012125473469495773, - "num_tokens": 12888442.0, - "reward": 0.328125, - "reward_std": 0.29826054349541664, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.29826054349541664, - "step": 149, - "step_time": 199.11138844676316 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 229.125, - "completions/max_terminated_length": 229.125, - "completions/mean_length": 198.828125, - "completions/mean_terminated_length": 198.828125, - "completions/min_length": 164.625, - "completions/min_terminated_length": 164.625, - "entropy": 0.9932208023965359, - "epoch": 0.05027020233756441, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.610958099365234, - "learning_rate": 9.83355674709562e-07, - "loss": 0.005720086395740509, - "num_tokens": 12972847.0, - "reward": 0.765625, - "reward_std": 0.2109457552433014, - "rewards/embodied_reward/mean": 0.765625, - "rewards/embodied_reward/std": 0.2109457552433014, - "step": 150, - "step_time": 211.97514683194458 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.875, - "completions/max_terminated_length": 211.875, - "completions/mean_length": 178.046875, - "completions/mean_terminated_length": 178.046875, - "completions/min_length": 134.5, - "completions/min_terminated_length": 134.5, - "entropy": 0.958558663725853, - "epoch": 0.05060533701981484, - "frac_reward_zero_std": 0.5, - "grad_norm": 5.228827953338623, - "learning_rate": 9.832439678284182e-07, - "loss": 0.0062184324488043785, - "num_tokens": 13060258.0, - "reward": 0.703125, - "reward_std": 0.2109457552433014, - "rewards/embodied_reward/mean": 0.703125, - "rewards/embodied_reward/std": 0.2109457552433014, - "step": 151, - "step_time": 199.21716672787443 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 168.75, - "completions/max_terminated_length": 168.75, - "completions/mean_length": 140.828125, - "completions/mean_terminated_length": 140.828125, - "completions/min_length": 85.875, - "completions/min_terminated_length": 85.875, - "entropy": 0.7318771071732044, - "epoch": 0.05094047170206527, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.470655918121338, - "learning_rate": 9.831322609472743e-07, - "loss": -0.01913967728614807, - "num_tokens": 13143303.0, - "reward": 0.75, - "reward_std": 0.2756394073367119, - "rewards/embodied_reward/mean": 0.75, - "rewards/embodied_reward/std": 0.2756394073367119, - "step": 152, - "step_time": 163.33855785429478 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 192.375, - "completions/max_terminated_length": 192.375, - "completions/mean_length": 156.96875, - "completions/mean_terminated_length": 156.96875, - "completions/min_length": 109.125, - "completions/min_terminated_length": 109.125, - "entropy": 0.7870599590241909, - "epoch": 0.0512756063843157, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.990734100341797, - "learning_rate": 9.830205540661304e-07, - "loss": -2.2120773792266846e-05, - "num_tokens": 13233597.0, - "reward": 0.578125, - "reward_std": 0.32878513261675835, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.32878513261675835, - "step": 153, - "step_time": 189.93516880460083 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.125, - "completions/max_terminated_length": 218.125, - "completions/mean_length": 182.859375, - "completions/mean_terminated_length": 182.859375, - "completions/min_length": 127.0, - "completions/min_terminated_length": 127.0, - "entropy": 0.9333163574337959, - "epoch": 0.05161074106656613, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.678887367248535, - "learning_rate": 9.829088471849867e-07, - "loss": -0.02156689018011093, - "num_tokens": 13321340.0, - "reward": 0.390625, - "reward_std": 0.2198973074555397, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.2198973074555397, - "step": 154, - "step_time": 209.65106836520135 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.375, - "completions/max_terminated_length": 196.375, - "completions/mean_length": 166.984375, - "completions/mean_terminated_length": 166.984375, - "completions/min_length": 122.5, - "completions/min_terminated_length": 122.5, - "entropy": 0.8152088299393654, - "epoch": 0.05194587574881655, - "frac_reward_zero_std": 0.375, - "grad_norm": 13.69566822052002, - "learning_rate": 9.827971403038427e-07, - "loss": -0.008299238979816437, - "num_tokens": 13412731.0, - "reward": 0.515625, - "reward_std": 0.268809512257576, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.268809512257576, - "step": 155, - "step_time": 188.63190363021567 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 220.375, - "completions/max_terminated_length": 220.375, - "completions/mean_length": 186.90625, - "completions/mean_terminated_length": 186.90625, - "completions/min_length": 154.0, - "completions/min_terminated_length": 154.0, - "entropy": 0.998546127229929, - "epoch": 0.052281010431066983, - "frac_reward_zero_std": 0.5, - "grad_norm": 14.75692081451416, - "learning_rate": 9.826854334226988e-07, - "loss": -0.0022451169788837433, - "num_tokens": 13491981.0, - "reward": 0.53125, - "reward_std": 0.24511481821537018, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.24511481821537018, - "step": 156, - "step_time": 205.062283444684 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 223.75, - "completions/max_terminated_length": 223.75, - "completions/mean_length": 185.296875, - "completions/mean_terminated_length": 185.296875, - "completions/min_length": 135.375, - "completions/min_terminated_length": 135.375, - "entropy": 0.8900587484240532, - "epoch": 0.052616145113317414, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.854912757873535, - "learning_rate": 9.825737265415549e-07, - "loss": -0.05219888314604759, - "num_tokens": 13585512.0, - "reward": 0.421875, - "reward_std": 0.29826054349541664, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.29826054349541664, - "step": 157, - "step_time": 208.31380893010646 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.125, - "completions/max_terminated_length": 210.125, - "completions/mean_length": 168.359375, - "completions/mean_terminated_length": 168.359375, - "completions/min_length": 108.5, - "completions/min_terminated_length": 108.5, - "entropy": 0.8227378726005554, - "epoch": 0.052951279795567845, - "frac_reward_zero_std": 0.375, - "grad_norm": 3.6429662704467773, - "learning_rate": 9.824620196604111e-07, - "loss": -0.008812565356492996, - "num_tokens": 13668559.0, - "reward": 0.71875, - "reward_std": 0.2709311693906784, - "rewards/embodied_reward/mean": 0.71875, - "rewards/embodied_reward/std": 0.2709311693906784, - "step": 158, - "step_time": 206.1173423975706 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 171.625, - "completions/max_terminated_length": 171.625, - "completions/mean_length": 143.140625, - "completions/mean_terminated_length": 143.140625, - "completions/min_length": 97.125, - "completions/min_terminated_length": 97.125, - "entropy": 0.7035566493868828, - "epoch": 0.053286414477818275, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.646749973297119, - "learning_rate": 9.823503127792672e-07, - "loss": 0.032218966633081436, - "num_tokens": 13751176.0, - "reward": 0.671875, - "reward_std": 0.2824693024158478, - "rewards/embodied_reward/mean": 0.671875, - "rewards/embodied_reward/std": 0.2824693024158478, - "step": 159, - "step_time": 168.3530195769854 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.125, - "completions/max_terminated_length": 197.125, - "completions/mean_length": 162.609375, - "completions/mean_terminated_length": 162.609375, - "completions/min_length": 121.25, - "completions/min_terminated_length": 121.25, - "entropy": 0.8227358050644398, - "epoch": 0.053621549160068706, - "frac_reward_zero_std": 0.625, - "grad_norm": 2.9401824474334717, - "learning_rate": 9.822386058981233e-07, - "loss": -0.03797348588705063, - "num_tokens": 13836607.0, - "reward": 0.625, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 160, - "step_time": 185.32268402073532 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 181.875, - "completions/max_terminated_length": 181.875, - "completions/mean_length": 154.171875, - "completions/mean_terminated_length": 154.171875, - "completions/min_length": 125.125, - "completions/min_terminated_length": 125.125, - "entropy": 0.6938700564205647, - "epoch": 0.05395668384231913, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.0556790828704834, - "learning_rate": 9.821268990169794e-07, - "loss": -0.0019825883209705353, - "num_tokens": 13921090.0, - "reward": 0.578125, - "reward_std": 0.1804211661219597, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.1804211661219597, - "step": 161, - "step_time": 177.6889114077203 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 197.125, - "completions/max_terminated_length": 197.125, - "completions/mean_length": 170.40625, - "completions/mean_terminated_length": 170.40625, - "completions/min_length": 137.375, - "completions/min_terminated_length": 137.375, - "entropy": 0.7489240244030952, - "epoch": 0.05429181852456956, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.002825736999512, - "learning_rate": 9.820151921358354e-07, - "loss": 0.008923431858420372, - "num_tokens": 14005572.0, - "reward": 0.671875, - "reward_std": 0.22673699632287025, - "rewards/embodied_reward/mean": 0.671875, - "rewards/embodied_reward/std": 0.22673699632287025, - "step": 162, - "step_time": 190.4956602472812 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 208.75, - "completions/max_terminated_length": 208.75, - "completions/mean_length": 176.375, - "completions/mean_terminated_length": 176.375, - "completions/min_length": 134.875, - "completions/min_terminated_length": 134.875, - "entropy": 0.8725037351250648, - "epoch": 0.05462695320681999, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.0846073627471924, - "learning_rate": 9.819034852546917e-07, - "loss": 0.00010238215327262878, - "num_tokens": 14091820.0, - "reward": 0.578125, - "reward_std": 0.1804211661219597, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.1804211661219597, - "step": 163, - "step_time": 196.92556410841644 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 189.375, - "completions/max_terminated_length": 189.375, - "completions/mean_length": 150.28125, - "completions/mean_terminated_length": 150.28125, - "completions/min_length": 86.375, - "completions/min_terminated_length": 86.375, - "entropy": 0.7620778456330299, - "epoch": 0.05496208788907042, - "frac_reward_zero_std": 0.125, - "grad_norm": 7.620922088623047, - "learning_rate": 9.817917783735478e-07, - "loss": 0.04892737790942192, - "num_tokens": 14178790.0, - "reward": 0.640625, - "reward_std": 0.3934885784983635, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.3934885784983635, - "step": 164, - "step_time": 192.56079516652972 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 244.125, - "completions/max_terminated_length": 244.125, - "completions/mean_length": 196.609375, - "completions/mean_terminated_length": 196.609375, - "completions/min_length": 151.625, - "completions/min_terminated_length": 151.625, - "entropy": 1.0115062072873116, - "epoch": 0.05529722257132085, - "frac_reward_zero_std": 0.125, - "grad_norm": 5.484842777252197, - "learning_rate": 9.816800714924038e-07, - "loss": -0.031379975378513336, - "num_tokens": 14273141.0, - "reward": 0.59375, - "reward_std": 0.3230287954211235, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.3230287954211235, - "step": 165, - "step_time": 232.55000547738746 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.75, - "completions/max_terminated_length": 211.75, - "completions/mean_length": 175.171875, - "completions/mean_terminated_length": 175.171875, - "completions/min_length": 125.0, - "completions/min_terminated_length": 125.0, - "entropy": 0.807915136218071, - "epoch": 0.05563235725357128, - "frac_reward_zero_std": 0.5, - "grad_norm": 9.789375305175781, - "learning_rate": 9.815683646112601e-07, - "loss": -0.009596094489097595, - "num_tokens": 14360200.0, - "reward": 0.609375, - "reward_std": 0.2561880275607109, - "rewards/embodied_reward/mean": 0.609375, - "rewards/embodied_reward/std": 0.2561880275607109, - "step": 166, - "step_time": 196.9018118097447 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 235.0, - "completions/max_terminated_length": 235.0, - "completions/mean_length": 197.90625, - "completions/mean_terminated_length": 197.90625, - "completions/min_length": 169.0, - "completions/min_terminated_length": 169.0, - "entropy": 1.0155997946858406, - "epoch": 0.055967491935821706, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.091899871826172, - "learning_rate": 9.814566577301162e-07, - "loss": 0.008808717131614685, - "num_tokens": 14450762.0, - "reward": 0.359375, - "reward_std": 0.3492846116423607, - "rewards/embodied_reward/mean": 0.359375, - "rewards/embodied_reward/std": 0.3492846116423607, - "step": 167, - "step_time": 234.84547999175265 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 208.25, - "completions/max_terminated_length": 208.25, - "completions/mean_length": 176.625, - "completions/mean_terminated_length": 176.625, - "completions/min_length": 155.625, - "completions/min_terminated_length": 155.625, - "entropy": 0.9260853007435799, - "epoch": 0.05630262661807214, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.74759578704834, - "learning_rate": 9.813449508489723e-07, - "loss": -0.01570822298526764, - "num_tokens": 14539786.0, - "reward": 0.546875, - "reward_std": 0.29826054349541664, - "rewards/embodied_reward/mean": 0.546875, - "rewards/embodied_reward/std": 0.29826054349541664, - "step": 168, - "step_time": 199.78503389796242 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.5, - "completions/max_terminated_length": 218.5, - "completions/mean_length": 186.03125, - "completions/mean_terminated_length": 186.03125, - "completions/min_length": 159.75, - "completions/min_terminated_length": 159.75, - "entropy": 1.005273263901472, - "epoch": 0.05663776130032257, - "frac_reward_zero_std": 0.75, - "grad_norm": 2.5891878604888916, - "learning_rate": 9.812332439678283e-07, - "loss": 0.0027772746980190277, - "num_tokens": 14618300.0, - "reward": 0.375, - "reward_std": 0.1157275140285492, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.1157275140285492, - "step": 169, - "step_time": 206.34108757879585 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 186.5, - "completions/max_terminated_length": 186.5, - "completions/mean_length": 156.703125, - "completions/mean_terminated_length": 156.703125, - "completions/min_length": 114.625, - "completions/min_terminated_length": 114.625, - "entropy": 0.7837265804409981, - "epoch": 0.056972895982573, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.207291126251221, - "learning_rate": 9.811215370866846e-07, - "loss": -0.006658792495727539, - "num_tokens": 14705489.0, - "reward": 0.515625, - "reward_std": 0.2777610644698143, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.2777610644698143, - "step": 170, - "step_time": 181.35214119777083 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 206.125, - "completions/max_terminated_length": 206.125, - "completions/mean_length": 175.984375, - "completions/mean_terminated_length": 175.984375, - "completions/min_length": 142.0, - "completions/min_terminated_length": 142.0, - "entropy": 0.9218467734754086, - "epoch": 0.05730803066482343, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.243124485015869, - "learning_rate": 9.810098302055407e-07, - "loss": -0.002526424825191498, - "num_tokens": 14793072.0, - "reward": 0.65625, - "reward_std": 0.28247909620404243, - "rewards/embodied_reward/mean": 0.65625, - "rewards/embodied_reward/std": 0.28247909620404243, - "step": 171, - "step_time": 193.7562730559148 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 223.625, - "completions/max_terminated_length": 223.625, - "completions/mean_length": 190.03125, - "completions/mean_terminated_length": 190.03125, - "completions/min_length": 152.5, - "completions/min_terminated_length": 152.5, - "entropy": 1.043153379112482, - "epoch": 0.05764316534707385, - "frac_reward_zero_std": 0.25, - "grad_norm": 12.560750961303711, - "learning_rate": 9.808981233243968e-07, - "loss": -0.03616444021463394, - "num_tokens": 14877562.0, - "reward": 0.390625, - "reward_std": 0.3266732692718506, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.3266732692718506, - "step": 172, - "step_time": 207.82191449962556 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 182.375, - "completions/max_terminated_length": 182.375, - "completions/mean_length": 162.65625, - "completions/mean_terminated_length": 162.65625, - "completions/min_length": 140.125, - "completions/min_terminated_length": 140.125, - "entropy": 0.7994086109101772, - "epoch": 0.05797830002932428, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.356342315673828, - "learning_rate": 9.807864164432528e-07, - "loss": -0.00920115876942873, - "num_tokens": 14960276.0, - "reward": 0.625, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.625, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 173, - "step_time": 179.53365681599826 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 207.25, - "completions/max_terminated_length": 207.25, - "completions/mean_length": 172.078125, - "completions/mean_terminated_length": 172.078125, - "completions/min_length": 111.625, - "completions/min_terminated_length": 111.625, - "entropy": 0.9324008673429489, - "epoch": 0.05831343471157471, - "frac_reward_zero_std": 0.375, - "grad_norm": 10.460923194885254, - "learning_rate": 9.806747095621089e-07, - "loss": 0.007705314084887505, - "num_tokens": 15049265.0, - "reward": 0.328125, - "reward_std": 0.26196982339024544, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.26196982339024544, - "step": 174, - "step_time": 209.94259295240045 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 201.75, - "completions/max_terminated_length": 201.75, - "completions/mean_length": 169.171875, - "completions/mean_terminated_length": 169.171875, - "completions/min_length": 128.25, - "completions/min_terminated_length": 128.25, - "entropy": 0.888934999704361, - "epoch": 0.058648569393825144, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.482022762298584, - "learning_rate": 9.805630026809652e-07, - "loss": -0.006784997880458832, - "num_tokens": 15132388.0, - "reward": 0.578125, - "reward_std": 0.24831003323197365, - "rewards/embodied_reward/mean": 0.578125, - "rewards/embodied_reward/std": 0.24831003323197365, - "step": 175, - "step_time": 186.2720371321775 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 228.5, - "completions/max_terminated_length": 228.5, - "completions/mean_length": 195.8125, - "completions/mean_terminated_length": 195.8125, - "completions/min_length": 167.125, - "completions/min_terminated_length": 167.125, - "entropy": 1.0947145000100136, - "epoch": 0.058983704076075574, - "frac_reward_zero_std": 0.25, - "grad_norm": 7.291539669036865, - "learning_rate": 9.804512957998212e-07, - "loss": 0.008040785789489746, - "num_tokens": 15220520.0, - "reward": 0.5625, - "reward_std": 0.3424547165632248, - "rewards/embodied_reward/mean": 0.5625, - "rewards/embodied_reward/std": 0.3424547165632248, - "step": 176, - "step_time": 211.04094081697986 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 189.0, - "completions/max_terminated_length": 189.0, - "completions/mean_length": 164.15625, - "completions/mean_terminated_length": 164.15625, - "completions/min_length": 127.5, - "completions/min_terminated_length": 127.5, - "entropy": 0.8103312216699123, - "epoch": 0.059318838758326005, - "frac_reward_zero_std": 0.5, - "grad_norm": 3.5571844577789307, - "learning_rate": 9.803395889186773e-07, - "loss": -0.02935316041111946, - "num_tokens": 15306186.0, - "reward": 0.8125, - "reward_std": 0.19727617129683495, - "rewards/embodied_reward/mean": 0.8125, - "rewards/embodied_reward/std": 0.19727617129683495, - "step": 177, - "step_time": 189.32402690313756 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 222.75, - "completions/max_terminated_length": 222.75, - "completions/mean_length": 190.0625, - "completions/mean_terminated_length": 190.0625, - "completions/min_length": 166.375, - "completions/min_terminated_length": 166.375, - "entropy": 0.9991403520107269, - "epoch": 0.05965397344057643, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.888899326324463, - "learning_rate": 9.802278820375334e-07, - "loss": -0.00387641042470932, - "num_tokens": 15396726.0, - "reward": 0.5, - "reward_std": 0.1462521031498909, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.1462521031498909, - "step": 178, - "step_time": 215.43124447017908 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 230.375, - "completions/max_terminated_length": 230.375, - "completions/mean_length": 187.375, - "completions/mean_terminated_length": 187.375, - "completions/min_length": 152.0, - "completions/min_terminated_length": 152.0, - "entropy": 0.9312093257904053, - "epoch": 0.05998910812282686, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.399527072906494, - "learning_rate": 9.801161751563895e-07, - "loss": -0.029198233038187027, - "num_tokens": 15487894.0, - "reward": 0.328125, - "reward_std": 0.26196982339024544, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.26196982339024544, - "step": 179, - "step_time": 214.41843235306442 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 239.875, - "completions/max_terminated_length": 239.875, - "completions/mean_length": 207.40625, - "completions/mean_terminated_length": 207.40625, - "completions/min_length": 173.875, - "completions/min_terminated_length": 173.875, - "entropy": 1.1234982535243034, - "epoch": 0.06032424280507729, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.727447032928467, - "learning_rate": 9.800044682752457e-07, - "loss": 0.012615067884325981, - "num_tokens": 15577616.0, - "reward": 0.453125, - "reward_std": 0.2109457552433014, - "rewards/embodied_reward/mean": 0.453125, - "rewards/embodied_reward/std": 0.2109457552433014, - "step": 180, - "step_time": 227.28597903670743 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.875, - "completions/max_terminated_length": 196.875, - "completions/mean_length": 166.171875, - "completions/mean_terminated_length": 166.171875, - "completions/min_length": 129.0, - "completions/min_terminated_length": 129.0, - "entropy": 0.8302468657493591, - "epoch": 0.06065937748732772, - "frac_reward_zero_std": 0.125, - "grad_norm": 8.692622184753418, - "learning_rate": 9.798927613941018e-07, - "loss": -0.013592004776000977, - "num_tokens": 15663395.0, - "reward": 0.328125, - "reward_std": 0.38194065168499947, - "rewards/embodied_reward/mean": 0.328125, - "rewards/embodied_reward/std": 0.38194065168499947, - "step": 181, - "step_time": 183.86910033458844 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 198.75, - "completions/max_terminated_length": 198.75, - "completions/mean_length": 167.078125, - "completions/mean_terminated_length": 167.078125, - "completions/min_length": 119.375, - "completions/min_terminated_length": 119.375, - "entropy": 0.9079095385968685, - "epoch": 0.06099451216957815, - "frac_reward_zero_std": 0.25, - "grad_norm": 5.7802324295043945, - "learning_rate": 9.79781054512958e-07, - "loss": -0.027043379843235016, - "num_tokens": 15749544.0, - "reward": 0.484375, - "reward_std": 0.30617379024624825, - "rewards/embodied_reward/mean": 0.484375, - "rewards/embodied_reward/std": 0.30617379024624825, - "step": 182, - "step_time": 186.45290085859597 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 206.875, - "completions/max_terminated_length": 206.875, - "completions/mean_length": 177.359375, - "completions/mean_terminated_length": 177.359375, - "completions/min_length": 137.0, - "completions/min_terminated_length": 137.0, - "entropy": 0.8808721080422401, - "epoch": 0.06132964685182858, - "frac_reward_zero_std": 0.625, - "grad_norm": 3.8503477573394775, - "learning_rate": 9.796693476318142e-07, - "loss": 0.022838223725557327, - "num_tokens": 15833175.0, - "reward": 0.5, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.5, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 183, - "step_time": 202.6345049943775 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 214.625, - "completions/max_terminated_length": 214.625, - "completions/mean_length": 183.71875, - "completions/mean_terminated_length": 183.71875, - "completions/min_length": 156.875, - "completions/min_terminated_length": 156.875, - "entropy": 0.9803147912025452, - "epoch": 0.061664781534079005, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.12472677230835, - "learning_rate": 9.795576407506702e-07, - "loss": 0.008300341665744781, - "num_tokens": 15923349.0, - "reward": 0.765625, - "reward_std": 0.2198973074555397, - "rewards/embodied_reward/mean": 0.765625, - "rewards/embodied_reward/std": 0.2198973074555397, - "step": 184, - "step_time": 204.95670328941196 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 210.5, - "completions/max_terminated_length": 210.5, - "completions/mean_length": 175.21875, - "completions/mean_terminated_length": 175.21875, - "completions/min_length": 113.5, - "completions/min_terminated_length": 113.5, - "entropy": 0.9506698623299599, - "epoch": 0.061999916216329436, - "frac_reward_zero_std": 0.625, - "grad_norm": 5.626105308532715, - "learning_rate": 9.794459338695263e-07, - "loss": 0.01776333525776863, - "num_tokens": 16000355.0, - "reward": 0.421875, - "reward_std": 0.13258251920342445, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.13258251920342445, - "step": 185, - "step_time": 196.01582003151998 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 196.875, - "completions/max_terminated_length": 196.875, - "completions/mean_length": 162.78125, - "completions/mean_terminated_length": 162.78125, - "completions/min_length": 143.875, - "completions/min_terminated_length": 143.875, - "entropy": 0.7934233732521534, - "epoch": 0.062335050898579866, - "frac_reward_zero_std": 0.5, - "grad_norm": 4.8859148025512695, - "learning_rate": 9.793342269883824e-07, - "loss": -0.004762537777423859, - "num_tokens": 16086517.0, - "reward": 0.375, - "reward_std": 0.2041158601641655, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.2041158601641655, - "step": 186, - "step_time": 198.05003156606108 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.25, - "completions/max_terminated_length": 218.25, - "completions/mean_length": 188.40625, - "completions/mean_terminated_length": 188.40625, - "completions/min_length": 160.75, - "completions/min_terminated_length": 160.75, - "entropy": 0.9558965899050236, - "epoch": 0.06267018558083029, - "frac_reward_zero_std": 0.25, - "grad_norm": 4.799129962921143, - "learning_rate": 9.792225201072386e-07, - "loss": 0.0015453845262527466, - "num_tokens": 16166591.0, - "reward": 0.65625, - "reward_std": 0.32666347548365593, - "rewards/embodied_reward/mean": 0.65625, - "rewards/embodied_reward/std": 0.32666347548365593, - "step": 187, - "step_time": 207.85660608578473 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 219.25, - "completions/max_terminated_length": 219.25, - "completions/mean_length": 191.453125, - "completions/mean_terminated_length": 191.453125, - "completions/min_length": 165.875, - "completions/min_terminated_length": 165.875, - "entropy": 0.9930871874094009, - "epoch": 0.06300532026308073, - "frac_reward_zero_std": 0.375, - "grad_norm": 11.840558052062988, - "learning_rate": 9.791108132260947e-07, - "loss": 0.0026259254664182663, - "num_tokens": 16255860.0, - "reward": 0.609375, - "reward_std": 0.318760022521019, - "rewards/embodied_reward/mean": 0.609375, - "rewards/embodied_reward/std": 0.318760022521019, - "step": 188, - "step_time": 205.7243076218292 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 202.875, - "completions/max_terminated_length": 202.875, - "completions/mean_length": 182.0, - "completions/mean_terminated_length": 182.0, - "completions/min_length": 153.875, - "completions/min_terminated_length": 153.875, - "entropy": 0.9081077203154564, - "epoch": 0.06334045494533115, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.756415843963623, - "learning_rate": 9.789991063449508e-07, - "loss": -0.004730157554149628, - "num_tokens": 16341884.0, - "reward": 0.640625, - "reward_std": 0.2414703443646431, - "rewards/embodied_reward/mean": 0.640625, - "rewards/embodied_reward/std": 0.2414703443646431, - "step": 189, - "step_time": 197.33634724933654 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 219.125, - "completions/max_terminated_length": 219.125, - "completions/mean_length": 187.375, - "completions/mean_terminated_length": 187.375, - "completions/min_length": 160.875, - "completions/min_terminated_length": 160.875, - "entropy": 0.933253787457943, - "epoch": 0.06367558962758159, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.8498311042785645, - "learning_rate": 9.788873994638069e-07, - "loss": 0.03030911087989807, - "num_tokens": 16429164.0, - "reward": 0.59375, - "reward_std": 0.25513992831110954, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.25513992831110954, - "step": 190, - "step_time": 204.17414566222578 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 228.5, - "completions/max_terminated_length": 228.5, - "completions/mean_length": 190.0625, - "completions/mean_terminated_length": 190.0625, - "completions/min_length": 158.875, - "completions/min_terminated_length": 158.875, - "entropy": 1.0727533102035522, - "epoch": 0.06401072430983201, - "frac_reward_zero_std": 0.375, - "grad_norm": 7.631810188293457, - "learning_rate": 9.78775692582663e-07, - "loss": -0.012556495144963264, - "num_tokens": 16517120.0, - "reward": 0.296875, - "reward_std": 0.2867126166820526, - "rewards/embodied_reward/mean": 0.296875, - "rewards/embodied_reward/std": 0.2867126166820526, - "step": 191, - "step_time": 222.57813433324918 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 245.375, - "completions/max_terminated_length": 245.375, - "completions/mean_length": 212.65625, - "completions/mean_terminated_length": 212.65625, - "completions/min_length": 179.25, - "completions/min_terminated_length": 179.25, - "entropy": 1.202123023569584, - "epoch": 0.06434585899208245, - "frac_reward_zero_std": 0.625, - "grad_norm": 4.307609558105469, - "learning_rate": 9.786639857015192e-07, - "loss": 0.004330750554800034, - "num_tokens": 16603850.0, - "reward": 0.234375, - "reward_std": 0.17358147725462914, - "rewards/embodied_reward/mean": 0.234375, - "rewards/embodied_reward/std": 0.17358147725462914, - "step": 192, - "step_time": 226.26706965314224 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 211.75, - "completions/max_terminated_length": 211.75, - "completions/mean_length": 182.75, - "completions/mean_terminated_length": 182.75, - "completions/min_length": 159.0, - "completions/min_terminated_length": 159.0, - "entropy": 0.9662274792790413, - "epoch": 0.06468099367433287, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.741211891174316, - "learning_rate": 9.785522788203753e-07, - "loss": -0.008647073060274124, - "num_tokens": 16695090.0, - "reward": 0.421875, - "reward_std": 0.24831003323197365, - "rewards/embodied_reward/mean": 0.421875, - "rewards/embodied_reward/std": 0.24831003323197365, - "step": 193, - "step_time": 217.54724451340735 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 224.125, - "completions/max_terminated_length": 224.125, - "completions/mean_length": 193.390625, - "completions/mean_terminated_length": 193.390625, - "completions/min_length": 169.5, - "completions/min_terminated_length": 169.5, - "entropy": 1.011228010058403, - "epoch": 0.0650161283565833, - "frac_reward_zero_std": 0.0, - "grad_norm": 5.4721479415893555, - "learning_rate": 9.784405719392316e-07, - "loss": 0.025514502078294754, - "num_tokens": 16787467.0, - "reward": 0.515625, - "reward_std": 0.4419162720441818, - "rewards/embodied_reward/mean": 0.515625, - "rewards/embodied_reward/std": 0.4419162720441818, - "step": 194, - "step_time": 217.509397033602 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 218.375, - "completions/max_terminated_length": 218.375, - "completions/mean_length": 178.15625, - "completions/mean_terminated_length": 178.15625, - "completions/min_length": 145.875, - "completions/min_terminated_length": 145.875, - "entropy": 0.9164508953690529, - "epoch": 0.06535126303883373, - "frac_reward_zero_std": 0.375, - "grad_norm": 5.449713706970215, - "learning_rate": 9.783288650580876e-07, - "loss": 0.008525297045707703, - "num_tokens": 16874141.0, - "reward": 0.40625, - "reward_std": 0.2756394073367119, - "rewards/embodied_reward/mean": 0.40625, - "rewards/embodied_reward/std": 0.2756394073367119, - "step": 195, - "step_time": 202.45198772475123 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 215.0, - "completions/max_terminated_length": 215.0, - "completions/mean_length": 185.46875, - "completions/mean_terminated_length": 185.46875, - "completions/min_length": 165.75, - "completions/min_terminated_length": 165.75, - "entropy": 1.0198731161653996, - "epoch": 0.06568639772108416, - "frac_reward_zero_std": 0.125, - "grad_norm": 12.048229217529297, - "learning_rate": 9.782171581769437e-07, - "loss": -0.017947036772966385, - "num_tokens": 16966371.0, - "reward": 0.59375, - "reward_std": 0.3230287954211235, - "rewards/embodied_reward/mean": 0.59375, - "rewards/embodied_reward/std": 0.3230287954211235, - "step": 196, - "step_time": 198.79092814400792 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 203.5, - "completions/max_terminated_length": 203.5, - "completions/mean_length": 174.5625, - "completions/mean_terminated_length": 174.5625, - "completions/min_length": 136.875, - "completions/min_terminated_length": 136.875, - "entropy": 1.0520923435688019, - "epoch": 0.0660215324033346, - "frac_reward_zero_std": 0.375, - "grad_norm": 4.852284908294678, - "learning_rate": 9.781054512957998e-07, - "loss": 0.0012535378336906433, - "num_tokens": 17053375.0, - "reward": 0.53125, - "reward_std": 0.2845909595489502, - "rewards/embodied_reward/mean": 0.53125, - "rewards/embodied_reward/std": 0.2845909595489502, - "step": 197, - "step_time": 190.24940429115668 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 191.5, - "completions/max_terminated_length": 191.5, - "completions/mean_length": 169.6875, - "completions/mean_terminated_length": 169.6875, - "completions/min_length": 146.875, - "completions/min_terminated_length": 146.875, - "entropy": 0.8587672375142574, - "epoch": 0.06635666708558502, - "frac_reward_zero_std": 0.625, - "grad_norm": 40.64221954345703, - "learning_rate": 9.779937444146558e-07, - "loss": -0.006138060241937637, - "num_tokens": 17138795.0, - "reward": 0.375, - "reward_std": 0.16675158217549324, - "rewards/embodied_reward/mean": 0.375, - "rewards/embodied_reward/std": 0.16675158217549324, - "step": 198, - "step_time": 181.38241435447708 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 226.75, - "completions/max_terminated_length": 226.75, - "completions/mean_length": 199.15625, - "completions/mean_terminated_length": 199.15625, - "completions/min_length": 172.875, - "completions/min_terminated_length": 172.875, - "entropy": 1.0334205143153667, - "epoch": 0.06669180176783544, - "frac_reward_zero_std": 0.375, - "grad_norm": 6.913132667541504, - "learning_rate": 9.778820375335121e-07, - "loss": -0.00037422776222229004, - "num_tokens": 17226869.0, - "reward": 0.46875, - "reward_std": 0.3119301274418831, - "rewards/embodied_reward/mean": 0.46875, - "rewards/embodied_reward/std": 0.3119301274418831, - "step": 199, - "step_time": 215.54423828748986 - }, - { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "completions/clipped_ratio": 0.0, - "completions/max_length": 212.375, - "completions/max_terminated_length": 212.375, - "completions/mean_length": 184.75, - "completions/mean_terminated_length": 184.75, - "completions/min_length": 158.5, - "completions/min_terminated_length": 158.5, - "entropy": 1.0168733559548855, - "epoch": 0.06702693645008588, - "frac_reward_zero_std": 0.5, - "grad_norm": 3.899688720703125, - "learning_rate": 9.777703306523682e-07, - "loss": 0.010468535125255585, - "num_tokens": 17314693.0, - "reward": 0.390625, - "reward_std": 0.23144523426890373, - "rewards/embodied_reward/mean": 0.390625, - "rewards/embodied_reward/std": 0.23144523426890373, - "step": 200, - "step_time": 194.06805218802765 - } - ], - "logging_steps": 1, - "max_steps": 8952, - "num_input_tokens_seen": 17314693, - "num_train_epochs": 3, - "save_steps": 50, - "stateful_callbacks": { - "TrainerControl": { - "args": { - "should_epoch_stop": false, - "should_evaluate": false, - "should_log": false, - "should_save": true, - "should_training_stop": false - }, - "attributes": {} - } - }, - "total_flos": 0.0, - "train_batch_size": 2, - "trial_name": null, - "trial_params": null -}