diff --git "a/rft/trainer_state.json" "b/rft/trainer_state.json" new file mode 100644--- /dev/null +++ "b/rft/trainer_state.json" @@ -0,0 +1,29602 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 168.0, + "global_step": 1344, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 230.125, + "completions/min_length": 90.0, + "epoch": 0.002976190476190476, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.407407407407407e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1, + "step_time": 11.155032472219318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 196.625, + "completions/min_length": 170.0, + "epoch": 0.005952380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0625, + "learning_rate": 1.4814814814814814e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 2, + "step_time": 7.298196783289313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/mean_length": 120.625, + "completions/min_length": 97.0, + "epoch": 0.008928571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.222222222222222e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 3, + "step_time": 5.348140901885927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/mean_length": 199.25, + "completions/min_length": 156.0, + "epoch": 0.011904761904761904, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 2.962962962962963e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 4, + "step_time": 6.6564284772612154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/mean_length": 184.875, + "completions/min_length": 118.0, + "epoch": 0.01488095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5, + "learning_rate": 3.7037037037037036e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 5, + "step_time": 6.895135682076216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 141.75, + "completions/min_length": 116.0, + "epoch": 0.017857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.444444444444444e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 6, + "step_time": 5.988761721178889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 131.0, + "completions/min_length": 100.0, + "epoch": 0.020833333333333332, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.375, + "learning_rate": 5.1851851851851846e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 7, + "step_time": 5.98020284762606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/mean_length": 152.375, + "completions/min_length": 77.0, + "epoch": 0.023809523809523808, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.925925925925926e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 8, + "step_time": 6.3133578659035265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 294.5, + "completions/min_length": 104.0, + "epoch": 0.026785714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.75, + "learning_rate": 6.666666666666667e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 9, + "step_time": 8.308423222973943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 172.5, + "completions/min_length": 138.0, + "epoch": 0.02976190476190476, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 7.407407407407407e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 10, + "step_time": 6.211498845834285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 309.0, + "completions/mean_length": 191.125, + "completions/min_length": 127.0, + "epoch": 0.03273809523809524, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.59375, + "learning_rate": 8.148148148148149e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 11, + "step_time": 7.080944010056555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 120.0, + "completions/mean_length": 105.0, + "completions/min_length": 97.0, + "epoch": 0.03571428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 8.888888888888888e-08, + "loss": 5.21540641784668e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 12, + "step_time": 5.446210194379091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 149.875, + "completions/min_length": 107.0, + "epoch": 0.03869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.629629629629629e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 13, + "step_time": 6.300527850165963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 135.75, + "completions/min_length": 107.0, + "epoch": 0.041666666666666664, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0370370370370369e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 14, + "step_time": 5.508425169158727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 141.0, + "completions/min_length": 109.0, + "epoch": 0.044642857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 1.111111111111111e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 15, + "step_time": 6.309976649004966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/mean_length": 182.5, + "completions/min_length": 133.0, + "epoch": 0.047619047619047616, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.90625, + "learning_rate": 1.1851851851851851e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 16, + "step_time": 6.498126021120697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/mean_length": 103.25, + "completions/min_length": 79.0, + "epoch": 0.050595238095238096, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2592592592592592e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 17, + "step_time": 5.635375200305134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 142.5, + "completions/min_length": 104.0, + "epoch": 0.05357142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3333333333333334e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 18, + "step_time": 5.747607049997896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/mean_length": 292.75, + "completions/min_length": 138.0, + "epoch": 0.05654761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6875, + "learning_rate": 1.4074074074074075e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 19, + "step_time": 7.716526473872364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 134.125, + "completions/min_length": 82.0, + "epoch": 0.05952380952380952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4814814814814815e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 20, + "step_time": 5.649100444279611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 171.625, + "completions/min_length": 94.0, + "epoch": 0.0625, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.21875, + "learning_rate": 1.5555555555555556e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 21, + "step_time": 6.473167315125465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/mean_length": 169.25, + "completions/min_length": 127.0, + "epoch": 0.06547619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6296296296296298e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 22, + "step_time": 6.372152770869434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 194.0, + "completions/min_length": 142.0, + "epoch": 0.06845238095238096, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7037037037037035e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 23, + "step_time": 6.82097733579576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 377.0, + "completions/mean_length": 158.75, + "completions/min_length": 94.0, + "epoch": 0.07142857142857142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7777777777777776e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 24, + "step_time": 7.367255785968155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 186.0, + "completions/mean_length": 119.375, + "completions/min_length": 97.0, + "epoch": 0.0744047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8518518518518516e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 25, + "step_time": 5.60468562040478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 185.0, + "completions/min_length": 137.0, + "epoch": 0.07738095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9259259259259257e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 26, + "step_time": 6.348422505892813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 124.875, + "completions/min_length": 113.0, + "epoch": 0.08035714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 2e-07, + "loss": -5.21540641784668e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 27, + "step_time": 5.464688124600798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 153.125, + "completions/min_length": 128.0, + "epoch": 0.08333333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0740740740740738e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 28, + "step_time": 5.71552172396332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 154.125, + "completions/min_length": 105.0, + "epoch": 0.08630952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.875, + "learning_rate": 2.148148148148148e-07, + "loss": -4.470348358154297e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 29, + "step_time": 6.009411671664566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 128.625, + "completions/min_length": 87.0, + "epoch": 0.08928571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.222222222222222e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 30, + "step_time": 6.006711530033499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/mean_length": 109.25, + "completions/min_length": 82.0, + "epoch": 0.09226190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.296296296296296e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 31, + "step_time": 5.9712867280468345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/mean_length": 151.375, + "completions/min_length": 118.0, + "epoch": 0.09523809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.3703703703703703e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 32, + "step_time": 6.74130353005603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 161.25, + "completions/min_length": 107.0, + "epoch": 0.09821428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.4444444444444445e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 33, + "step_time": 6.16574149671942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 114.375, + "completions/min_length": 96.0, + "epoch": 0.10119047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5185185185185184e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 34, + "step_time": 5.743890190962702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/mean_length": 146.875, + "completions/min_length": 97.0, + "epoch": 0.10416666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5925925925925923e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 35, + "step_time": 6.9192711468786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 208.5, + "completions/min_length": 158.0, + "epoch": 0.10714285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.9375, + "learning_rate": 2.6666666666666667e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 36, + "step_time": 6.835666821338236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/mean_length": 212.875, + "completions/min_length": 119.0, + "epoch": 0.11011904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 2.7407407407407406e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 37, + "step_time": 6.991162579040974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 197.75, + "completions/min_length": 103.0, + "epoch": 0.1130952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.814814814814815e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 38, + "step_time": 6.613073920365423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/mean_length": 102.75, + "completions/min_length": 71.0, + "epoch": 0.11607142857142858, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.8888888888888885e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 39, + "step_time": 5.861451483797282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/mean_length": 194.125, + "completions/min_length": 121.0, + "epoch": 0.11904761904761904, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.0625, + "learning_rate": 2.962962962962963e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 40, + "step_time": 6.270247815642506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 479.0, + "completions/mean_length": 228.0, + "completions/min_length": 119.0, + "epoch": 0.12202380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 3.037037037037037e-07, + "loss": 5.21540641784668e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 41, + "step_time": 8.158147369045764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/mean_length": 110.375, + "completions/min_length": 65.0, + "epoch": 0.125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.111111111111111e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 42, + "step_time": 6.269185694865882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/mean_length": 172.875, + "completions/min_length": 104.0, + "epoch": 0.12797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.185185185185185e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 43, + "step_time": 6.352103322278708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 375.0, + "completions/mean_length": 273.75, + "completions/min_length": 169.0, + "epoch": 0.13095238095238096, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.2592592592592596e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 44, + "step_time": 7.526429612655193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/mean_length": 168.625, + "completions/min_length": 126.0, + "epoch": 0.13392857142857142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.333333333333333e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 45, + "step_time": 6.962517249863595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 135.0, + "completions/mean_length": 122.125, + "completions/min_length": 98.0, + "epoch": 0.13690476190476192, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.407407407407407e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 46, + "step_time": 5.454880472738296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.0, + "completions/mean_length": 178.125, + "completions/min_length": 102.0, + "epoch": 0.13988095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.4814814814814814e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 47, + "step_time": 7.519632881972939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 229.375, + "completions/min_length": 108.0, + "epoch": 0.14285714285714285, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.625, + "learning_rate": 3.5555555555555553e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 48, + "step_time": 7.0562366880476475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 129.125, + "completions/min_length": 98.0, + "epoch": 0.14583333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.6296296296296297e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 49, + "step_time": 5.58223051764071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 151.375, + "completions/min_length": 114.0, + "epoch": 0.1488095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.703703703703703e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 50, + "step_time": 6.403424319811165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 194.0, + "completions/mean_length": 132.5, + "completions/min_length": 97.0, + "epoch": 0.15178571428571427, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.7777777777777775e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 51, + "step_time": 7.342086322139949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 142.25, + "completions/min_length": 115.0, + "epoch": 0.15476190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.8518518518518515e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 52, + "step_time": 6.08844694821164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.0, + "completions/mean_length": 146.375, + "completions/min_length": 97.0, + "epoch": 0.15773809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 3.925925925925926e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 53, + "step_time": 7.9273724323138595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 114.125, + "completions/min_length": 77.0, + "epoch": 0.16071428571428573, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 54, + "step_time": 8.425067018251866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/mean_length": 171.625, + "completions/min_length": 114.0, + "epoch": 0.1636904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.0740740740740737e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 55, + "step_time": 8.559567152988166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 401.0, + "completions/mean_length": 286.25, + "completions/min_length": 225.0, + "epoch": 0.16666666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1481481481481476e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 56, + "step_time": 7.845788015983999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 216.0, + "completions/min_length": 118.0, + "epoch": 0.16964285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.222222222222222e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 57, + "step_time": 8.429897964932024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 131.5, + "completions/min_length": 92.0, + "epoch": 0.17261904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.296296296296296e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 58, + "step_time": 6.067105587106198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 123.0, + "completions/min_length": 98.0, + "epoch": 0.17559523809523808, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.59375, + "learning_rate": 4.3703703703703704e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 59, + "step_time": 5.743390671908855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 344.0, + "completions/mean_length": 192.0, + "completions/min_length": 118.0, + "epoch": 0.17857142857142858, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 4.444444444444444e-07, + "loss": -5.960464477539063e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 60, + "step_time": 7.602044130209833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/mean_length": 136.25, + "completions/min_length": 99.0, + "epoch": 0.18154761904761904, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5185185185185183e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 61, + "step_time": 6.997547564096749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 293.0, + "completions/mean_length": 159.875, + "completions/min_length": 115.0, + "epoch": 0.18452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.592592592592592e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 62, + "step_time": 6.897626146674156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/mean_length": 166.75, + "completions/min_length": 135.0, + "epoch": 0.1875, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.34375, + "learning_rate": 4.6666666666666666e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 63, + "step_time": 6.517527509946376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 145.625, + "completions/min_length": 104.0, + "epoch": 0.19047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.7407407407407405e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 64, + "step_time": 6.4723676959984004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 155.25, + "completions/min_length": 110.0, + "epoch": 0.19345238095238096, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.814814814814814e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 65, + "step_time": 6.555097745265812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 335.0, + "completions/mean_length": 213.875, + "completions/min_length": 148.0, + "epoch": 0.19642857142857142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.888888888888889e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 66, + "step_time": 8.327655584085733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 124.875, + "completions/min_length": 86.0, + "epoch": 0.19940476190476192, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.962962962962963e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 67, + "step_time": 7.101494054775685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 155.625, + "completions/min_length": 130.0, + "epoch": 0.20238095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.037037037037037e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 68, + "step_time": 6.129679119680077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 157.375, + "completions/min_length": 110.0, + "epoch": 0.20535714285714285, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.111111111111111e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 69, + "step_time": 6.213496702723205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 312.625, + "completions/min_length": 147.0, + "epoch": 0.20833333333333334, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.5, + "learning_rate": 5.185185185185185e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 70, + "step_time": 8.733208434656262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 165.125, + "completions/min_length": 96.0, + "epoch": 0.2113095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.259259259259258e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 71, + "step_time": 6.333869417198002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 237.125, + "completions/min_length": 117.0, + "epoch": 0.21428571428571427, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.09375, + "learning_rate": 5.333333333333333e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 72, + "step_time": 8.780140494927764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/mean_length": 180.25, + "completions/min_length": 78.0, + "epoch": 0.21726190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.28125, + "learning_rate": 5.407407407407407e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 73, + "step_time": 6.666322604753077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 112.875, + "completions/min_length": 98.0, + "epoch": 0.22023809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.481481481481481e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 74, + "step_time": 5.452395766042173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/mean_length": 182.5, + "completions/min_length": 145.0, + "epoch": 0.22321428571428573, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.84375, + "learning_rate": 5.555555555555555e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 75, + "step_time": 6.365840396378189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.0, + "completions/mean_length": 170.25, + "completions/min_length": 120.0, + "epoch": 0.2261904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 5.62962962962963e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 76, + "step_time": 6.54482208378613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 159.75, + "completions/min_length": 104.0, + "epoch": 0.22916666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.3125, + "learning_rate": 5.703703703703704e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 77, + "step_time": 6.666055994108319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 189.25, + "completions/min_length": 156.0, + "epoch": 0.23214285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.777777777777777e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 78, + "step_time": 6.9440274382941425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 129.625, + "completions/min_length": 110.0, + "epoch": 0.23511904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.851851851851851e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 79, + "step_time": 6.01745288213715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 364.0, + "completions/mean_length": 229.75, + "completions/min_length": 134.0, + "epoch": 0.23809523809523808, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.925925925925926e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 80, + "step_time": 7.221964337863028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 195.0, + "completions/mean_length": 141.25, + "completions/min_length": 112.0, + "epoch": 0.24107142857142858, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 6e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 81, + "step_time": 5.821366933640093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 166.125, + "completions/min_length": 119.0, + "epoch": 0.24404761904761904, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.074074074074074e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 82, + "step_time": 6.891093960031867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 211.0, + "completions/min_length": 103.0, + "epoch": 0.24702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.148148148148148e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 83, + "step_time": 7.190345395356417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 125.375, + "completions/min_length": 80.0, + "epoch": 0.25, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.222222222222223e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 84, + "step_time": 5.542022520210594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 307.0, + "completions/mean_length": 201.5, + "completions/min_length": 113.0, + "epoch": 0.25297619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.296296296296296e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 85, + "step_time": 7.934909557923675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 170.625, + "completions/min_length": 111.0, + "epoch": 0.25595238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.875, + "learning_rate": 6.37037037037037e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 86, + "step_time": 7.850997405592352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 143.25, + "completions/min_length": 123.0, + "epoch": 0.25892857142857145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.444444444444444e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 87, + "step_time": 7.671846440993249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/mean_length": 136.75, + "completions/min_length": 98.0, + "epoch": 0.2619047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.518518518518519e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 88, + "step_time": 6.722152492031455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 173.625, + "completions/min_length": 125.0, + "epoch": 0.2648809523809524, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.592592592592592e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 89, + "step_time": 6.481708618346602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 182.75, + "completions/min_length": 148.0, + "epoch": 0.26785714285714285, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.666666666666666e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 90, + "step_time": 6.691809656098485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 128.25, + "completions/min_length": 107.0, + "epoch": 0.2708333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.74074074074074e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 91, + "step_time": 5.980228738393635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 117.0, + "completions/mean_length": 105.75, + "completions/min_length": 95.0, + "epoch": 0.27380952380952384, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.814814814814814e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 92, + "step_time": 5.963895194698125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/mean_length": 222.375, + "completions/min_length": 98.0, + "epoch": 0.2767857142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.40625, + "learning_rate": 6.888888888888889e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 93, + "step_time": 8.847886714152992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/mean_length": 147.875, + "completions/min_length": 112.0, + "epoch": 0.27976190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.962962962962963e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 94, + "step_time": 7.658857750706375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 147.75, + "completions/min_length": 126.0, + "epoch": 0.28273809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.037037037037037e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 95, + "step_time": 7.117651046253741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 143.625, + "completions/min_length": 103.0, + "epoch": 0.2857142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.111111111111111e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 96, + "step_time": 6.783719465136528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/mean_length": 205.0, + "completions/min_length": 140.0, + "epoch": 0.28869047619047616, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.65625, + "learning_rate": 7.185185185185186e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 97, + "step_time": 6.785291670821607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 201.375, + "completions/min_length": 123.0, + "epoch": 0.2916666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.259259259259259e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 98, + "step_time": 6.93962083524093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.0, + "completions/mean_length": 170.875, + "completions/min_length": 124.0, + "epoch": 0.29464285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.333333333333332e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 99, + "step_time": 7.009510949254036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 162.25, + "completions/min_length": 99.0, + "epoch": 0.2976190476190476, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.0, + "learning_rate": 7.407407407407406e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 100, + "step_time": 6.742004980798811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 119.75, + "completions/min_length": 91.0, + "epoch": 0.3005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.481481481481481e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 101, + "step_time": 6.861165119335055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/mean_length": 107.875, + "completions/min_length": 87.0, + "epoch": 0.30357142857142855, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.555555555555555e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 102, + "step_time": 6.033797680400312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 135.875, + "completions/min_length": 73.0, + "epoch": 0.30654761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.3125, + "learning_rate": 7.629629629629629e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 103, + "step_time": 6.042028442956507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 131.0, + "completions/mean_length": 109.75, + "completions/min_length": 95.0, + "epoch": 0.30952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.703703703703703e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 104, + "step_time": 6.36736325873062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/mean_length": 169.5, + "completions/min_length": 117.0, + "epoch": 0.3125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.777777777777778e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 105, + "step_time": 7.2259732582606375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/mean_length": 122.5, + "completions/min_length": 88.0, + "epoch": 0.31547619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.851851851851852e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 106, + "step_time": 5.732897481881082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 118.0, + "completions/min_length": 84.0, + "epoch": 0.31845238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 7.925925925925926e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 107, + "step_time": 6.120217599906027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 125.875, + "completions/min_length": 110.0, + "epoch": 0.32142857142857145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 108, + "step_time": 5.909699882846326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 325.0, + "completions/mean_length": 191.375, + "completions/min_length": 114.0, + "epoch": 0.3244047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.074074074074075e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 109, + "step_time": 7.457380433101207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 177.0, + "completions/min_length": 129.0, + "epoch": 0.3273809523809524, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.148148148148147e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 110, + "step_time": 7.220073699951172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 166.625, + "completions/min_length": 130.0, + "epoch": 0.33035714285714285, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.222222222222221e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 111, + "step_time": 6.665845420677215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 169.5, + "completions/min_length": 92.0, + "epoch": 0.3333333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.296296296296295e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 112, + "step_time": 7.324518795125186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 150.875, + "completions/min_length": 102.0, + "epoch": 0.33630952380952384, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.37037037037037e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 113, + "step_time": 6.367212758865207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 174.5, + "completions/min_length": 107.0, + "epoch": 0.3392857142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.71875, + "learning_rate": 8.444444444444444e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 114, + "step_time": 6.48159848805517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 140.125, + "completions/min_length": 110.0, + "epoch": 0.34226190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.518518518518518e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 115, + "step_time": 6.339078231714666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 334.0, + "completions/mean_length": 194.875, + "completions/min_length": 136.0, + "epoch": 0.34523809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.592592592592592e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 116, + "step_time": 7.166483615990728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 275.25, + "completions/min_length": 120.0, + "epoch": 0.3482142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.0, + "learning_rate": 8.666666666666667e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 117, + "step_time": 9.409736136440188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 281.0, + "completions/mean_length": 186.25, + "completions/min_length": 81.0, + "epoch": 0.35119047619047616, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.740740740740741e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 118, + "step_time": 6.79190028924495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 130.375, + "completions/min_length": 78.0, + "epoch": 0.3541666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.814814814814815e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 119, + "step_time": 6.120216542854905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 145.625, + "completions/min_length": 92.0, + "epoch": 0.35714285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.888888888888888e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 120, + "step_time": 5.902018727269024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 142.25, + "completions/min_length": 87.0, + "epoch": 0.3601190476190476, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.962962962962963e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 121, + "step_time": 6.4657844211906195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 347.0, + "completions/mean_length": 180.75, + "completions/min_length": 127.0, + "epoch": 0.3630952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.03125, + "learning_rate": 9.037037037037037e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 122, + "step_time": 7.923760036937892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 272.0, + "completions/mean_length": 160.125, + "completions/min_length": 118.0, + "epoch": 0.36607142857142855, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 9.11111111111111e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 123, + "step_time": 7.4519112459383905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 225.75, + "completions/min_length": 128.0, + "epoch": 0.36904761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 9.185185185185184e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 124, + "step_time": 8.948895571753383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 115.25, + "completions/min_length": 89.0, + "epoch": 0.37202380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.259259259259259e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 125, + "step_time": 5.656541469972581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/mean_length": 160.0, + "completions/min_length": 92.0, + "epoch": 0.375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.333333333333333e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 126, + "step_time": 6.7728057112544775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 170.75, + "completions/min_length": 98.0, + "epoch": 0.37797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.407407407407407e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 127, + "step_time": 7.006603910122067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 149.0, + "completions/min_length": 103.0, + "epoch": 0.38095238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 9.481481481481481e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 128, + "step_time": 6.286806297954172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/mean_length": 136.5, + "completions/min_length": 104.0, + "epoch": 0.38392857142857145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.555555555555556e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 129, + "step_time": 5.753967259079218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/mean_length": 149.125, + "completions/min_length": 103.0, + "epoch": 0.3869047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.34375, + "learning_rate": 9.629629629629628e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 130, + "step_time": 6.574743086937815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 192.0, + "completions/mean_length": 139.5, + "completions/min_length": 114.0, + "epoch": 0.3898809523809524, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.703703703703704e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 131, + "step_time": 6.414534179959446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 174.25, + "completions/min_length": 127.0, + "epoch": 0.39285714285714285, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.1875, + "learning_rate": 9.777777777777778e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 132, + "step_time": 6.675969321280718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 137.0, + "completions/min_length": 113.0, + "epoch": 0.3958333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 9.851851851851852e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 133, + "step_time": 6.758370861876756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 223.75, + "completions/min_length": 121.0, + "epoch": 0.39880952380952384, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.925925925925926e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 134, + "step_time": 9.420055339112878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 391.0, + "completions/mean_length": 172.375, + "completions/min_length": 114.0, + "epoch": 0.4017857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 135, + "step_time": 7.9604815151542425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 139.0, + "completions/mean_length": 115.75, + "completions/min_length": 94.0, + "epoch": 0.40476190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.999983119437743e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 136, + "step_time": 5.542306653223932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 116.0, + "completions/mean_length": 101.75, + "completions/min_length": 93.0, + "epoch": 0.40773809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.4375, + "learning_rate": 9.999932477864956e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 137, + "step_time": 5.673644294030964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/mean_length": 184.0, + "completions/min_length": 111.0, + "epoch": 0.4107142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.999848075623583e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 138, + "step_time": 7.370975101366639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 159.5, + "completions/min_length": 99.0, + "epoch": 0.41369047619047616, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.999729913283522e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 139, + "step_time": 7.567347541451454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 179.125, + "completions/min_length": 137.0, + "epoch": 0.4166666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.99957799164264e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 140, + "step_time": 6.607319022063166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/mean_length": 164.25, + "completions/min_length": 94.0, + "epoch": 0.41964285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.999392311726737e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 141, + "step_time": 6.464730001054704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 157.5, + "completions/min_length": 107.0, + "epoch": 0.4226190476190476, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 9.999172874789572e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 142, + "step_time": 5.871565772220492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 151.25, + "completions/min_length": 112.0, + "epoch": 0.4255952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.998919682312828e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 143, + "step_time": 6.4860388790257275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 141.5, + "completions/min_length": 109.0, + "epoch": 0.42857142857142855, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.998632736006122e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 144, + "step_time": 6.618686438072473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 248.375, + "completions/min_length": 145.0, + "epoch": 0.43154761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.46875, + "learning_rate": 9.998312037806977e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 145, + "step_time": 8.813428091350943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/mean_length": 250.5, + "completions/min_length": 129.0, + "epoch": 0.43452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.997957589880822e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 146, + "step_time": 6.910566996783018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 168.5, + "completions/min_length": 100.0, + "epoch": 0.4375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.997569394620964e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 147, + "step_time": 6.765647196676582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 162.125, + "completions/min_length": 105.0, + "epoch": 0.44047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.997147454648588e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 148, + "step_time": 6.258278019726276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 158.625, + "completions/min_length": 133.0, + "epoch": 0.44345238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.65625, + "learning_rate": 9.99669177281273e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 149, + "step_time": 8.548544520046562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 110.0, + "completions/mean_length": 93.375, + "completions/min_length": 75.0, + "epoch": 0.44642857142857145, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.996202352190255e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 150, + "step_time": 5.905823368113488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/mean_length": 163.125, + "completions/min_length": 100.0, + "epoch": 0.4494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.995679196085838e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 151, + "step_time": 7.424603764433414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 194.0, + "completions/mean_length": 148.5, + "completions/min_length": 107.0, + "epoch": 0.4523809523809524, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.03125, + "learning_rate": 9.99512230803195e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 152, + "step_time": 6.642257389612496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 121.0, + "completions/min_length": 85.0, + "epoch": 0.45535714285714285, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.994531691788821e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 153, + "step_time": 6.065606968011707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/mean_length": 142.375, + "completions/min_length": 108.0, + "epoch": 0.4583333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.993907351344426e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 154, + "step_time": 7.849403732921928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 141.0, + "completions/min_length": 101.0, + "epoch": 0.46130952380952384, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.993249290914455e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 155, + "step_time": 6.424862145911902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 158.875, + "completions/min_length": 106.0, + "epoch": 0.4642857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.992557514942276e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 156, + "step_time": 7.187588380184025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 159.75, + "completions/min_length": 121.0, + "epoch": 0.46726190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.03125, + "learning_rate": 9.991832028098922e-07, + "loss": -5.21540641784668e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 157, + "step_time": 6.103514443151653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 183.625, + "completions/min_length": 146.0, + "epoch": 0.47023809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.625, + "learning_rate": 9.991072835283035e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 158, + "step_time": 6.33417296782136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/mean_length": 175.75, + "completions/min_length": 122.0, + "epoch": 0.4732142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.59375, + "learning_rate": 9.99027994162086e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 159, + "step_time": 8.08957302896306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 151.125, + "completions/min_length": 109.0, + "epoch": 0.47619047619047616, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 9.989453352466195e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 160, + "step_time": 6.395296220201999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/mean_length": 177.625, + "completions/min_length": 136.0, + "epoch": 0.4791666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.988593073400354e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 161, + "step_time": 6.537385892122984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 111.875, + "completions/min_length": 79.0, + "epoch": 0.48214285714285715, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.987699110232132e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 162, + "step_time": 6.079313917085528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 356.0, + "completions/mean_length": 199.0, + "completions/min_length": 122.0, + "epoch": 0.4851190476190476, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.986771468997774e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 163, + "step_time": 7.410659202840179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 144.75, + "completions/min_length": 107.0, + "epoch": 0.4880952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.985810155960921e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 164, + "step_time": 6.694489839952439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.0, + "completions/mean_length": 163.625, + "completions/min_length": 122.0, + "epoch": 0.49107142857142855, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.984815177612573e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 165, + "step_time": 7.920315752271563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 124.5, + "completions/min_length": 100.0, + "epoch": 0.49404761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.98378654067105e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 166, + "step_time": 5.992400540038943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 179.875, + "completions/min_length": 140.0, + "epoch": 0.49702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.982724252081938e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 167, + "step_time": 6.4797025779262185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 127.0, + "completions/min_length": 102.0, + "epoch": 0.5, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.981628319018048e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 168, + "step_time": 6.251780834048986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/mean_length": 192.75, + "completions/min_length": 103.0, + "epoch": 0.5029761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5625, + "learning_rate": 9.98049874887937e-07, + "loss": 7.450580596923828e-09, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 169, + "step_time": 7.348913388326764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/mean_length": 101.625, + "completions/min_length": 73.0, + "epoch": 0.5059523809523809, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.97933554929301e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 170, + "step_time": 5.854466297198087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 147.75, + "completions/min_length": 103.0, + "epoch": 0.5089285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.96875, + "learning_rate": 9.978138728113158e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 171, + "step_time": 6.279781615827233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 145.25, + "completions/min_length": 123.0, + "epoch": 0.5119047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.97690829342102e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 172, + "step_time": 7.159720789175481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 139.25, + "completions/min_length": 111.0, + "epoch": 0.5148809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 9.975644253524765e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 173, + "step_time": 7.073706584982574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/mean_length": 136.125, + "completions/min_length": 86.0, + "epoch": 0.5178571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.974346616959475e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 174, + "step_time": 6.58434765599668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 185.75, + "completions/min_length": 112.0, + "epoch": 0.5208333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.973015392487085e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 175, + "step_time": 6.719707268290222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 367.0, + "completions/mean_length": 212.75, + "completions/min_length": 134.0, + "epoch": 0.5238095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.971650589096323e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 176, + "step_time": 7.489886619150639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 400.0, + "completions/mean_length": 183.0, + "completions/min_length": 111.0, + "epoch": 0.5267857142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 9.970252216002646e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 177, + "step_time": 8.571962567046285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/mean_length": 140.75, + "completions/min_length": 94.0, + "epoch": 0.5297619047619048, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.4375, + "learning_rate": 9.968820282648185e-07, + "loss": 4.470348358154297e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 178, + "step_time": 7.820800366345793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 163.75, + "completions/min_length": 111.0, + "epoch": 0.5327380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 9.967354798701675e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 179, + "step_time": 8.329285824671388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.0, + "completions/mean_length": 203.0, + "completions/min_length": 92.0, + "epoch": 0.5357142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.965855774058393e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 180, + "step_time": 7.054183530155569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 144.875, + "completions/min_length": 112.0, + "epoch": 0.5386904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.964323218840095e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 181, + "step_time": 6.553702551871538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 148.625, + "completions/min_length": 123.0, + "epoch": 0.5416666666666666, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 9.962757143394932e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 182, + "step_time": 6.491319551132619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/mean_length": 166.875, + "completions/min_length": 86.0, + "epoch": 0.5446428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.961157558297403e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 183, + "step_time": 6.770999962929636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 404.0, + "completions/mean_length": 205.875, + "completions/min_length": 128.0, + "epoch": 0.5476190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.959524474348261e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 184, + "step_time": 9.207066718954593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 147.0, + "completions/min_length": 101.0, + "epoch": 0.5505952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.53125, + "learning_rate": 9.957857902574463e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 185, + "step_time": 7.624723447021097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/mean_length": 161.25, + "completions/min_length": 104.0, + "epoch": 0.5535714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.95615785422907e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 186, + "step_time": 6.978752207010984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 170.125, + "completions/min_length": 117.0, + "epoch": 0.5565476190476191, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 9.954424340791195e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 187, + "step_time": 6.461363906040788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 191.0, + "completions/min_length": 111.0, + "epoch": 0.5595238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.952657373965907e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 188, + "step_time": 6.979501858353615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 118.5, + "completions/min_length": 89.0, + "epoch": 0.5625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.950856965684165e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 189, + "step_time": 5.839645578991622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 274.0, + "completions/mean_length": 206.375, + "completions/min_length": 161.0, + "epoch": 0.5654761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.949023128102733e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 190, + "step_time": 7.713810011744499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/mean_length": 170.75, + "completions/min_length": 133.0, + "epoch": 0.5684523809523809, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 9.947155873604092e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 191, + "step_time": 6.5413825158029795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.0, + "completions/mean_length": 215.875, + "completions/min_length": 121.0, + "epoch": 0.5714285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75, + "learning_rate": 9.945255214796363e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 192, + "step_time": 7.5773515230976045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 447.0, + "completions/mean_length": 216.125, + "completions/min_length": 123.0, + "epoch": 0.5744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 9.943321164513229e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 193, + "step_time": 8.473063574638218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 130.5, + "completions/min_length": 104.0, + "epoch": 0.5773809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.941353735813823e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 194, + "step_time": 6.258966329973191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 271.0, + "completions/mean_length": 212.5, + "completions/min_length": 152.0, + "epoch": 0.5803571428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0625, + "learning_rate": 9.93935294198267e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 195, + "step_time": 7.776881145779043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 185.75, + "completions/min_length": 120.0, + "epoch": 0.5833333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.93731879652958e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 196, + "step_time": 7.169058647938073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 178.75, + "completions/min_length": 99.0, + "epoch": 0.5863095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.935251313189563e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 197, + "step_time": 6.970610441174358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 129.75, + "completions/min_length": 98.0, + "epoch": 0.5892857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.93315050592273e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 198, + "step_time": 6.645945317111909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 159.375, + "completions/min_length": 113.0, + "epoch": 0.5922619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.9310163889142e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 199, + "step_time": 7.225684280041605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 186.0, + "completions/mean_length": 128.375, + "completions/min_length": 83.0, + "epoch": 0.5952380952380952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.928848976574018e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 200, + "step_time": 6.470069575123489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 116.125, + "completions/min_length": 91.0, + "epoch": 0.5982142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.926648283537035e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 201, + "step_time": 6.0648820898495615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 169.375, + "completions/min_length": 136.0, + "epoch": 0.6011904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.6875, + "learning_rate": 9.924414324662827e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 202, + "step_time": 6.869843796826899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/mean_length": 111.375, + "completions/min_length": 88.0, + "epoch": 0.6041666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.922147115035584e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 203, + "step_time": 5.572331936098635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 123.5, + "completions/min_length": 109.0, + "epoch": 0.6071428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.91984666996402e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 204, + "step_time": 5.951608886942267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 116.625, + "completions/min_length": 102.0, + "epoch": 0.6101190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.917513004981252e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 205, + "step_time": 5.959681106731296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 131.75, + "completions/min_length": 78.0, + "epoch": 0.6130952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.915146135844717e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 206, + "step_time": 6.2949144071899354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 166.875, + "completions/min_length": 99.0, + "epoch": 0.6160714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.912746078536043e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 207, + "step_time": 6.61795231513679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 139.25, + "completions/min_length": 103.0, + "epoch": 0.6190476190476191, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.910312849260957e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 208, + "step_time": 5.864823654294014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 99.0, + "completions/min_length": 74.0, + "epoch": 0.6220238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.907846464449173e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 209, + "step_time": 6.357253000605851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 167.625, + "completions/min_length": 117.0, + "epoch": 0.625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.905346940754274e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 210, + "step_time": 6.458803258836269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 206.0, + "completions/min_length": 114.0, + "epoch": 0.6279761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.6875, + "learning_rate": 9.902814295053605e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 211, + "step_time": 8.513880347833037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 145.0, + "completions/min_length": 109.0, + "epoch": 0.6309523809523809, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.900248544448164e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 212, + "step_time": 6.577255669049919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 148.25, + "completions/min_length": 109.0, + "epoch": 0.6339285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.897649706262473e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 213, + "step_time": 5.964396294672042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 332.25, + "completions/min_length": 242.0, + "epoch": 0.6369047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.703125, + "learning_rate": 9.895017798044469e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 214, + "step_time": 8.727145112119615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 118.0, + "completions/mean_length": 101.875, + "completions/min_length": 84.0, + "epoch": 0.6398809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.892352837565395e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 215, + "step_time": 5.867095009889454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 155.5, + "completions/min_length": 96.0, + "epoch": 0.6428571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.889654842819658e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 216, + "step_time": 6.873281337786466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 108.625, + "completions/min_length": 88.0, + "epoch": 0.6458333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.886923832024726e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 217, + "step_time": 6.0254300511442125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 113.25, + "completions/min_length": 96.0, + "epoch": 0.6488095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.884159823621e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 218, + "step_time": 5.76990609196946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 180.75, + "completions/min_length": 122.0, + "epoch": 0.6517857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.881362836271685e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 219, + "step_time": 6.841893550939858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/mean_length": 183.25, + "completions/min_length": 96.0, + "epoch": 0.6547619047619048, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 9.87853288886267e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 220, + "step_time": 7.303980407770723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 138.625, + "completions/min_length": 119.0, + "epoch": 0.6577380952380952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.875670000502394e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 221, + "step_time": 6.492957917973399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/mean_length": 169.0, + "completions/min_length": 115.0, + "epoch": 0.6607142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.872774190521726e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 222, + "step_time": 7.229721094015986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 120.125, + "completions/min_length": 102.0, + "epoch": 0.6636904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.869845478473823e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 223, + "step_time": 6.076435730326921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 195.125, + "completions/min_length": 125.0, + "epoch": 0.6666666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.866883884134012e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 224, + "step_time": 6.951615005731583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 339.0, + "completions/mean_length": 227.875, + "completions/min_length": 135.0, + "epoch": 0.6696428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.86388942749964e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 225, + "step_time": 7.339146425016224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 122.375, + "completions/min_length": 75.0, + "epoch": 0.6726190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.860862128789952e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 226, + "step_time": 6.45499960007146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 124.25, + "completions/min_length": 97.0, + "epoch": 0.6755952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.857802008445953e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 227, + "step_time": 5.819004646036774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 161.0, + "completions/min_length": 132.0, + "epoch": 0.6785714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.5, + "learning_rate": 9.85470908713026e-07, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 228, + "step_time": 6.1652161460369825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 135.25, + "completions/min_length": 110.0, + "epoch": 0.6815476190476191, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.851583385726976e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 229, + "step_time": 5.977071820292622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 180.0, + "completions/min_length": 124.0, + "epoch": 0.6845238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.848424925341537e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 230, + "step_time": 6.975077678915113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 174.875, + "completions/min_length": 99.0, + "epoch": 0.6875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.845233727300578e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 231, + "step_time": 6.9611867386847734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 283.0, + "completions/mean_length": 187.375, + "completions/min_length": 104.0, + "epoch": 0.6904761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 9.842009813151792e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 232, + "step_time": 7.175272705964744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/mean_length": 128.0, + "completions/min_length": 115.0, + "epoch": 0.6934523809523809, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.838753204663766e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 233, + "step_time": 6.071777472738177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 185.75, + "completions/min_length": 142.0, + "epoch": 0.6964285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.835463923825852e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 234, + "step_time": 6.817284746095538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/mean_length": 196.75, + "completions/min_length": 143.0, + "epoch": 0.6994047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.8125, + "learning_rate": 9.83214199284802e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 235, + "step_time": 6.447141325101256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/mean_length": 151.375, + "completions/min_length": 106.0, + "epoch": 0.7023809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.828787434160692e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 236, + "step_time": 6.698061613831669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 155.625, + "completions/min_length": 95.0, + "epoch": 0.7053571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.8254002704146e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 237, + "step_time": 6.930398681201041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/mean_length": 183.25, + "completions/min_length": 153.0, + "epoch": 0.7083333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.821980524480641e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 238, + "step_time": 6.945659636985511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 145.375, + "completions/min_length": 86.0, + "epoch": 0.7113095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.818528219449704e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 239, + "step_time": 6.406262260861695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 165.0, + "completions/min_length": 104.0, + "epoch": 0.7142857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.81504337863253e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 240, + "step_time": 6.601762888021767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 173.0, + "completions/min_length": 88.0, + "epoch": 0.7172619047619048, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.78125, + "learning_rate": 9.81152602555955e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 241, + "step_time": 7.055416677612811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 174.625, + "completions/min_length": 114.0, + "epoch": 0.7202380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 9.80797618398072e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 242, + "step_time": 6.928461780305952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 172.125, + "completions/min_length": 138.0, + "epoch": 0.7232142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 9.80439387786537e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 243, + "step_time": 6.728572931140661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/mean_length": 171.0, + "completions/min_length": 129.0, + "epoch": 0.7261904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.80077913140204e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 244, + "step_time": 6.995509340893477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/mean_length": 179.875, + "completions/min_length": 120.0, + "epoch": 0.7291666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.797131968998306e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 245, + "step_time": 7.155452862847596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 425.0, + "completions/mean_length": 250.25, + "completions/min_length": 164.0, + "epoch": 0.7321428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 9.793452415280628e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 246, + "step_time": 7.814091041218489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 176.875, + "completions/min_length": 137.0, + "epoch": 0.7351190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.789740495094186e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 247, + "step_time": 6.497635883279145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 159.75, + "completions/min_length": 123.0, + "epoch": 0.7380952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.15625, + "learning_rate": 9.785996233502696e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 248, + "step_time": 6.556327135767788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 274.0, + "completions/mean_length": 133.625, + "completions/min_length": 59.0, + "epoch": 0.7410714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.782219655788254e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 249, + "step_time": 6.893348393030465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 199.5, + "completions/min_length": 143.0, + "epoch": 0.7440476190476191, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.778410787451166e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 250, + "step_time": 6.511718889698386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 114.0, + "completions/min_length": 91.0, + "epoch": 0.7470238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.774569654209762e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 251, + "step_time": 5.536877145990729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/mean_length": 113.25, + "completions/min_length": 95.0, + "epoch": 0.75, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.770696282000244e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 252, + "step_time": 6.037399540189654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 230.125, + "completions/min_length": 136.0, + "epoch": 0.7529761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.90625, + "learning_rate": 9.766790696976486e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 253, + "step_time": 8.527744207996875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 138.25, + "completions/min_length": 67.0, + "epoch": 0.7559523809523809, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.76285292550988e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 254, + "step_time": 6.290543572045863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 157.0, + "completions/min_length": 122.0, + "epoch": 0.7589285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.758882994189144e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 255, + "step_time": 6.517410790082067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 303.0, + "completions/mean_length": 161.5, + "completions/min_length": 102.0, + "epoch": 0.7619047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5, + "learning_rate": 9.754880929820147e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 256, + "step_time": 7.256885950919241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 142.875, + "completions/min_length": 126.0, + "epoch": 0.7648809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.53125, + "learning_rate": 9.750846759425727e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 257, + "step_time": 6.092566562816501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 112.625, + "completions/min_length": 87.0, + "epoch": 0.7678571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.74678051024551e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 258, + "step_time": 6.092220501974225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/mean_length": 161.875, + "completions/min_length": 108.0, + "epoch": 0.7708333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.742682209735727e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 259, + "step_time": 6.169968211092055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 125.25, + "completions/min_length": 97.0, + "epoch": 0.7738095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.738551885569021e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 260, + "step_time": 6.28986530797556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/mean_length": 152.5, + "completions/min_length": 124.0, + "epoch": 0.7767857142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.4375, + "learning_rate": 9.734389565634274e-07, + "loss": 4.470348358154297e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 261, + "step_time": 5.987504028249532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/mean_length": 123.125, + "completions/min_length": 91.0, + "epoch": 0.7797619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.730195278036403e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 262, + "step_time": 5.87759335199371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/mean_length": 163.625, + "completions/min_length": 107.0, + "epoch": 0.7827380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 9.725969051096182e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 263, + "step_time": 6.33027234300971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 244.125, + "completions/min_length": 138.0, + "epoch": 0.7857142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.721710913350047e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 264, + "step_time": 8.839448262937367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 123.25, + "completions/min_length": 99.0, + "epoch": 0.7886904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.7174208935499e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 265, + "step_time": 6.0088799968361855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 157.625, + "completions/min_length": 114.0, + "epoch": 0.7916666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.713099020662922e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 266, + "step_time": 6.481920012738556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 223.625, + "completions/min_length": 156.0, + "epoch": 0.7946428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.65625, + "learning_rate": 9.708745323871369e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 267, + "step_time": 7.07361699687317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/mean_length": 167.875, + "completions/min_length": 137.0, + "epoch": 0.7976190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.704359832572378e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 268, + "step_time": 6.320436167065054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 134.0, + "completions/mean_length": 102.625, + "completions/min_length": 78.0, + "epoch": 0.8005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.699942576377777e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 269, + "step_time": 5.898031254298985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 154.5, + "completions/min_length": 112.0, + "epoch": 0.8035714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.695493585113871e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 270, + "step_time": 6.4317721370607615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/mean_length": 131.125, + "completions/min_length": 101.0, + "epoch": 0.8065476190476191, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 9.691012888821253e-07, + "loss": 5.960464477539063e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 271, + "step_time": 6.140955029055476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 125.375, + "completions/min_length": 109.0, + "epoch": 0.8095238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 9.686500517754588e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 272, + "step_time": 5.974210328888148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 130.0, + "completions/mean_length": 105.875, + "completions/min_length": 87.0, + "epoch": 0.8125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.681956502382422e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 273, + "step_time": 5.370294685009867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 371.0, + "completions/mean_length": 180.375, + "completions/min_length": 81.0, + "epoch": 0.8154761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.677380873386966e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 274, + "step_time": 7.214732462074608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 184.5, + "completions/min_length": 118.0, + "epoch": 0.8184523809523809, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.672773661663902e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 275, + "step_time": 7.02124657901004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 162.5, + "completions/min_length": 112.0, + "epoch": 0.8214285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.875, + "learning_rate": 9.668134898322155e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 276, + "step_time": 6.917223046068102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 157.375, + "completions/min_length": 116.0, + "epoch": 0.8244047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.6634646146837e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 277, + "step_time": 6.606817539315671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 136.5, + "completions/min_length": 82.0, + "epoch": 0.8273809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.658762842283341e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 278, + "step_time": 6.14553879480809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.0, + "completions/mean_length": 149.75, + "completions/min_length": 96.0, + "epoch": 0.8303571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.654029612868506e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 279, + "step_time": 7.171280166134238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/mean_length": 198.125, + "completions/min_length": 151.0, + "epoch": 0.8333333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.649264958399021e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 280, + "step_time": 7.078219813294709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/mean_length": 156.875, + "completions/min_length": 104.0, + "epoch": 0.8363095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.644468911046905e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 281, + "step_time": 6.399759707041085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 121.75, + "completions/min_length": 91.0, + "epoch": 0.8392857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.639641503196152e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 282, + "step_time": 5.774837303441018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 124.0, + "completions/mean_length": 114.375, + "completions/min_length": 103.0, + "epoch": 0.8422619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.634782767442499e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 283, + "step_time": 5.805520494934171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 135.625, + "completions/min_length": 93.0, + "epoch": 0.8452380952380952, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.629892736593231e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 284, + "step_time": 6.432885671965778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/mean_length": 275.375, + "completions/min_length": 207.0, + "epoch": 0.8482142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.09375, + "learning_rate": 9.624971443666928e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 285, + "step_time": 8.310513755306602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 317.0, + "completions/mean_length": 217.0, + "completions/min_length": 140.0, + "epoch": 0.8511904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 9.62001892189327e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 286, + "step_time": 7.100426178891212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 365.0, + "completions/mean_length": 159.875, + "completions/min_length": 99.0, + "epoch": 0.8541666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.6150352047128e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 287, + "step_time": 7.390601910185069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 177.25, + "completions/min_length": 125.0, + "epoch": 0.8571428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.09375, + "learning_rate": 9.610020325776692e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 288, + "step_time": 6.826282006222755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 148.0, + "completions/min_length": 121.0, + "epoch": 0.8601190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.604974318946543e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 289, + "step_time": 5.887035163119435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 400.0, + "completions/mean_length": 212.25, + "completions/min_length": 71.0, + "epoch": 0.8630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.599897218294121e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 290, + "step_time": 7.661485492717475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 123.125, + "completions/min_length": 86.0, + "epoch": 0.8660714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.594789058101153e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 291, + "step_time": 6.231751970946789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/mean_length": 121.375, + "completions/min_length": 80.0, + "epoch": 0.8690476190476191, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.589649872859086e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 292, + "step_time": 5.638906482141465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/mean_length": 234.5, + "completions/min_length": 173.0, + "epoch": 0.8720238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 9.584479697268852e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 293, + "step_time": 7.229902531951666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 184.0, + "completions/min_length": 133.0, + "epoch": 0.875, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.375, + "learning_rate": 9.579278566240644e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 294, + "step_time": 7.090372166596353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 177.25, + "completions/min_length": 104.0, + "epoch": 0.8779761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.15625, + "learning_rate": 9.574046514893665e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 295, + "step_time": 6.766362108755857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 130.375, + "completions/min_length": 96.0, + "epoch": 0.8809523809523809, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 9.568783578555904e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 296, + "step_time": 6.479617989156395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 142.75, + "completions/min_length": 105.0, + "epoch": 0.8839285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.4375, + "learning_rate": 9.563489792763887e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 297, + "step_time": 6.403363138902932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 168.25, + "completions/min_length": 96.0, + "epoch": 0.8869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.558165193262452e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 298, + "step_time": 6.756994131021202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 162.0, + "completions/mean_length": 132.875, + "completions/min_length": 119.0, + "epoch": 0.8898809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.55280981600449e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 299, + "step_time": 6.167910833843052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 158.0, + "completions/min_length": 119.0, + "epoch": 0.8928571428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 9.547423697150711e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 300, + "step_time": 6.5145053206942976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 161.5, + "completions/min_length": 127.0, + "epoch": 0.8958333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.542006873069403e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 301, + "step_time": 6.406883844174445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 146.375, + "completions/min_length": 107.0, + "epoch": 0.8988095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.536559380336181e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 302, + "step_time": 6.575038954149932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/mean_length": 206.5, + "completions/min_length": 107.0, + "epoch": 0.9017857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.531081255733738e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 303, + "step_time": 7.67408792860806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 170.0, + "completions/mean_length": 121.75, + "completions/min_length": 104.0, + "epoch": 0.9047619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.525572536251605e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 304, + "step_time": 6.287868345156312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 153.5, + "completions/min_length": 105.0, + "epoch": 0.9077380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 9.520033259085895e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 305, + "step_time": 6.622633547056466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 189.375, + "completions/min_length": 110.0, + "epoch": 0.9107142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.75, + "learning_rate": 9.514463461639055e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 306, + "step_time": 7.1706836479716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 140.5, + "completions/min_length": 107.0, + "epoch": 0.9136904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.96875, + "learning_rate": 9.508863181519607e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 307, + "step_time": 6.4929118901491165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 128.0, + "completions/mean_length": 106.375, + "completions/min_length": 87.0, + "epoch": 0.9166666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.503232456541903e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 308, + "step_time": 5.825671032071114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/mean_length": 139.125, + "completions/min_length": 88.0, + "epoch": 0.9196428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.497571324725864e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 309, + "step_time": 6.146680134814233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.0, + "completions/mean_length": 197.75, + "completions/min_length": 105.0, + "epoch": 0.9226190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.491879824296727e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 310, + "step_time": 7.534808059222996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 281.0, + "completions/mean_length": 199.375, + "completions/min_length": 141.0, + "epoch": 0.9255952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.96875, + "learning_rate": 9.48615799368478e-07, + "loss": 0.0, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 311, + "step_time": 7.097760064993054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/mean_length": 164.0, + "completions/min_length": 128.0, + "epoch": 0.9285714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 9.480405871525113e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 312, + "step_time": 6.938767283223569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 152.875, + "completions/min_length": 131.0, + "epoch": 0.9315476190476191, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.6875, + "learning_rate": 9.474623496657346e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 313, + "step_time": 6.672648090869188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 239.0, + "completions/min_length": 122.0, + "epoch": 0.9345238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.468810908125377e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 314, + "step_time": 8.915829243138433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 157.375, + "completions/min_length": 126.0, + "epoch": 0.9375, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.96875, + "learning_rate": 9.46296814517711e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 315, + "step_time": 6.616387244779617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 213.0, + "completions/mean_length": 162.0, + "completions/min_length": 123.0, + "epoch": 0.9404761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 9.457095247264195e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 316, + "step_time": 6.369384010788053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 184.75, + "completions/min_length": 83.0, + "epoch": 0.9434523809523809, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 9.451192254041758e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 317, + "step_time": 6.6970265242271125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 389.0, + "completions/mean_length": 166.5, + "completions/min_length": 102.0, + "epoch": 0.9464285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.445259205368137e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 318, + "step_time": 7.874004915822297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/mean_length": 138.375, + "completions/min_length": 107.0, + "epoch": 0.9494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.439296141304613e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 319, + "step_time": 6.132075788918883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/mean_length": 152.625, + "completions/min_length": 109.0, + "epoch": 0.9523809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.433303102115134e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 320, + "step_time": 5.968500940594822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 251.75, + "completions/min_length": 99.0, + "epoch": 0.9553571428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 9.427280128266049e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 321, + "step_time": 8.50381827307865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 125.625, + "completions/min_length": 95.0, + "epoch": 0.9583333333333334, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.421227260425832e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 322, + "step_time": 6.531526532024145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 329.0, + "completions/mean_length": 225.75, + "completions/min_length": 133.0, + "epoch": 0.9613095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 9.415144539464808e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 323, + "step_time": 7.062642334960401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 163.125, + "completions/min_length": 109.0, + "epoch": 0.9642857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.409032006454875e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 324, + "step_time": 6.67801956133917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 248.75, + "completions/min_length": 94.0, + "epoch": 0.9672619047619048, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.402889702669234e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 325, + "step_time": 8.57711410196498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 258.25, + "completions/min_length": 177.0, + "epoch": 0.9702380952380952, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.28125, + "learning_rate": 9.396717669582102e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 326, + "step_time": 7.227719044312835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.0, + "completions/mean_length": 229.125, + "completions/min_length": 147.0, + "epoch": 0.9732142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.78125, + "learning_rate": 9.39051594886843e-07, + "loss": 5.960464477539063e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 327, + "step_time": 7.1010369951836765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.0, + "completions/mean_length": 157.875, + "completions/min_length": 127.0, + "epoch": 0.9761904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.384284582403635e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 328, + "step_time": 6.698163327760994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 156.75, + "completions/min_length": 112.0, + "epoch": 0.9791666666666666, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.378023612263302e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 329, + "step_time": 6.690596415661275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 149.25, + "completions/min_length": 101.0, + "epoch": 0.9821428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.37173308072291e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 330, + "step_time": 6.036483069881797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 199.5, + "completions/min_length": 138.0, + "epoch": 0.9851190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 9.365413030257545e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 331, + "step_time": 6.939983612857759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/mean_length": 177.75, + "completions/min_length": 120.0, + "epoch": 0.9880952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 9.359063503541607e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 332, + "step_time": 6.406989202834666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 148.875, + "completions/min_length": 108.0, + "epoch": 0.9910714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 9.35268454344853e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 333, + "step_time": 5.85466769291088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 143.125, + "completions/min_length": 108.0, + "epoch": 0.9940476190476191, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.346276193050486e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 334, + "step_time": 5.896954426076263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.0, + "completions/mean_length": 202.25, + "completions/min_length": 147.0, + "epoch": 0.9970238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 9.339838495618099e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 335, + "step_time": 7.490660207811743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 144.0, + "completions/min_length": 115.0, + "epoch": 1.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.333371494620148e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 336, + "step_time": 6.337250125128776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 132.25, + "completions/min_length": 100.0, + "epoch": 1.0029761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.326875233723282e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 337, + "step_time": 6.448735137004405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 130.5, + "completions/min_length": 99.0, + "epoch": 1.005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.32034975679171e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 338, + "step_time": 6.609668989200145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 115.0, + "completions/mean_length": 103.75, + "completions/min_length": 86.0, + "epoch": 1.0089285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.313795107886925e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 339, + "step_time": 5.718191069085151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 139.125, + "completions/min_length": 117.0, + "epoch": 1.0119047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.307211331267388e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 340, + "step_time": 6.016182879917324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 133.625, + "completions/min_length": 107.0, + "epoch": 1.0148809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.300598471388239e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 341, + "step_time": 6.102423216216266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 160.125, + "completions/min_length": 106.0, + "epoch": 1.0178571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.293956572900999e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 342, + "step_time": 6.531535160262138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 177.875, + "completions/min_length": 123.0, + "epoch": 1.0208333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.287285680653254e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 343, + "step_time": 6.772773632314056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 275.0, + "completions/mean_length": 162.0, + "completions/min_length": 120.0, + "epoch": 1.0238095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0625, + "learning_rate": 9.280585839688375e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 344, + "step_time": 6.7476000119932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 114.375, + "completions/min_length": 83.0, + "epoch": 1.0267857142857142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.273857095245191e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 345, + "step_time": 5.96030747005716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 114.5, + "completions/min_length": 86.0, + "epoch": 1.0297619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.267099492757699e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 346, + "step_time": 6.0284962030127645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.0, + "completions/mean_length": 156.125, + "completions/min_length": 112.0, + "epoch": 1.0327380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0, + "learning_rate": 9.260313077854751e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 347, + "step_time": 6.855810145847499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 121.875, + "completions/min_length": 91.0, + "epoch": 1.0357142857142858, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.253497896359747e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 348, + "step_time": 5.598614999093115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/mean_length": 169.375, + "completions/min_length": 89.0, + "epoch": 1.0386904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.246653994290325e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 349, + "step_time": 7.614593775942922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 143.625, + "completions/min_length": 133.0, + "epoch": 1.0416666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.23978141785805e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 350, + "step_time": 6.06354014435783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 139.5, + "completions/min_length": 85.0, + "epoch": 1.0446428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.232880213468105e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 351, + "step_time": 5.87412748625502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 138.625, + "completions/min_length": 97.0, + "epoch": 1.0476190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.225950427718974e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 352, + "step_time": 6.58128809928894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 136.875, + "completions/min_length": 112.0, + "epoch": 1.0505952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.218992107402128e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 353, + "step_time": 6.965596412774175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 165.625, + "completions/min_length": 120.0, + "epoch": 1.0535714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.21200529950171e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 354, + "step_time": 7.532678697258234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.0, + "completions/mean_length": 167.625, + "completions/min_length": 96.0, + "epoch": 1.056547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.204990051194221e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 355, + "step_time": 6.454752212390304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 208.125, + "completions/min_length": 139.0, + "epoch": 1.0595238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.25, + "learning_rate": 9.197946409848194e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 356, + "step_time": 7.1438658237457275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 199.5, + "completions/min_length": 126.0, + "epoch": 1.0625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.190874423023879e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 357, + "step_time": 7.23655775282532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 118.125, + "completions/min_length": 80.0, + "epoch": 1.0654761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.183774138472921e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 358, + "step_time": 6.197784414049238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 153.625, + "completions/min_length": 124.0, + "epoch": 1.068452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.176645604138041e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 359, + "step_time": 7.268629282712936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 139.125, + "completions/min_length": 98.0, + "epoch": 1.0714285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 9.169488868152703e-07, + "loss": 7.450580596923828e-09, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 360, + "step_time": 6.8814732641913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 233.125, + "completions/min_length": 128.0, + "epoch": 1.0744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 9.1623039788408e-07, + "loss": -7.450580596923828e-09, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 361, + "step_time": 8.613777433056384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/mean_length": 179.75, + "completions/min_length": 114.0, + "epoch": 1.0773809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.78125, + "learning_rate": 9.155090984716318e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 362, + "step_time": 6.465327315032482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 194.0, + "completions/mean_length": 143.625, + "completions/min_length": 92.0, + "epoch": 1.0803571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.147849934483018e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 363, + "step_time": 6.4278118261136115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 171.875, + "completions/min_length": 106.0, + "epoch": 1.0833333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.140580877034099e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 364, + "step_time": 6.846489664632827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 132.5, + "completions/min_length": 85.0, + "epoch": 1.0863095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.13328386145187e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 365, + "step_time": 7.985271935351193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 117.0, + "completions/mean_length": 96.75, + "completions/min_length": 85.0, + "epoch": 1.0892857142857142, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.25, + "learning_rate": 9.125958937007425e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 366, + "step_time": 5.760137461125851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 176.25, + "completions/min_length": 138.0, + "epoch": 1.0922619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.09375, + "learning_rate": 9.118606153160299e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 367, + "step_time": 6.764860205817968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 188.0, + "completions/min_length": 146.0, + "epoch": 1.0952380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 9.111225559558142e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 368, + "step_time": 6.770627622958273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 151.125, + "completions/min_length": 112.0, + "epoch": 1.0982142857142858, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.103817206036382e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 369, + "step_time": 6.272682177834213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 209.875, + "completions/min_length": 146.0, + "epoch": 1.1011904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.096381142617888e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 370, + "step_time": 6.961912147235125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 132.5, + "completions/min_length": 107.0, + "epoch": 1.1041666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.088917419512633e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 371, + "step_time": 5.878374587278813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 138.875, + "completions/min_length": 95.0, + "epoch": 1.1071428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.081426087117354e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 372, + "step_time": 6.487149108201265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 163.875, + "completions/min_length": 125.0, + "epoch": 1.1101190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.073907196015214e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 373, + "step_time": 6.435839913319796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/mean_length": 224.125, + "completions/min_length": 133.0, + "epoch": 1.1130952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.84375, + "learning_rate": 9.066360796975452e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 374, + "step_time": 8.441857651807368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 176.75, + "completions/min_length": 120.0, + "epoch": 1.1160714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 9.058786940953056e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 375, + "step_time": 7.02809982933104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 129.75, + "completions/min_length": 96.0, + "epoch": 1.119047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.051185679088404e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 376, + "step_time": 6.481721176765859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 138.25, + "completions/min_length": 107.0, + "epoch": 1.1220238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.043557062706924e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 377, + "step_time": 6.157064237631857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 145.5, + "completions/min_length": 89.0, + "epoch": 1.125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.035901143318752e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 378, + "step_time": 6.015296334866434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/mean_length": 225.25, + "completions/min_length": 127.0, + "epoch": 1.1279761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.96875, + "learning_rate": 9.028217972618375e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 379, + "step_time": 7.853160131257027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 142.0, + "completions/min_length": 101.0, + "epoch": 1.130952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.020507602484293e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 380, + "step_time": 6.279550564941019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.0, + "completions/mean_length": 181.875, + "completions/min_length": 124.0, + "epoch": 1.1339285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.71875, + "learning_rate": 9.012770084978656e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 381, + "step_time": 6.576640552841127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 128.0, + "completions/mean_length": 105.625, + "completions/min_length": 78.0, + "epoch": 1.1369047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.005005472346923e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 382, + "step_time": 5.803129341918975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 139.0, + "completions/mean_length": 111.5, + "completions/min_length": 93.0, + "epoch": 1.1398809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.997213817017506e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 383, + "step_time": 5.912731540389359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 170.0, + "completions/mean_length": 138.5, + "completions/min_length": 110.0, + "epoch": 1.1428571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.989395171601413e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 384, + "step_time": 5.756022270768881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 122.875, + "completions/min_length": 96.0, + "epoch": 1.1458333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.981549588891897e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 385, + "step_time": 6.902101381216198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/mean_length": 162.75, + "completions/min_length": 119.0, + "epoch": 1.1488095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.973677121864096e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 386, + "step_time": 6.7775628552772105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 125.0, + "completions/mean_length": 102.375, + "completions/min_length": 85.0, + "epoch": 1.1517857142857142, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.625, + "learning_rate": 8.965777823674679e-07, + "loss": 5.960464477539063e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 387, + "step_time": 5.759946807287633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 166.25, + "completions/min_length": 131.0, + "epoch": 1.1547619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.875, + "learning_rate": 8.957851747661483e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 388, + "step_time": 6.6158316931687295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 302.0, + "completions/mean_length": 189.75, + "completions/min_length": 130.0, + "epoch": 1.1577380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 8.949898947343156e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 389, + "step_time": 6.797453346196562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 213.0, + "completions/mean_length": 157.75, + "completions/min_length": 112.0, + "epoch": 1.1607142857142858, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.941919476418795e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 390, + "step_time": 6.3714064680971205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/mean_length": 196.5, + "completions/min_length": 124.0, + "epoch": 1.1636904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.933913388767583e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 391, + "step_time": 6.3582132291048765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 502.0, + "completions/mean_length": 279.0, + "completions/min_length": 124.0, + "epoch": 1.1666666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 8.925880738448424e-07, + "loss": 7.450580596923828e-09, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 392, + "step_time": 8.307688753120601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 172.0, + "completions/min_length": 112.0, + "epoch": 1.1696428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.917821579699577e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 393, + "step_time": 6.414707330986857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 178.5, + "completions/min_length": 129.0, + "epoch": 1.1726190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 8.909735966938296e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 394, + "step_time": 6.486548306886107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 129.125, + "completions/min_length": 87.0, + "epoch": 1.1755952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.901623954760459e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 395, + "step_time": 6.493811053223908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 124.0, + "completions/min_length": 87.0, + "epoch": 1.1785714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.893485597940193e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 396, + "step_time": 6.116458509117365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 198.5, + "completions/min_length": 87.0, + "epoch": 1.181547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.885320951429516e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 397, + "step_time": 6.899109044112265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 256.0, + "completions/mean_length": 139.25, + "completions/min_length": 104.0, + "epoch": 1.1845238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.877130070357958e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 398, + "step_time": 6.788813997991383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 272.25, + "completions/min_length": 182.0, + "epoch": 1.1875, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.65625, + "learning_rate": 8.868913010032188e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 399, + "step_time": 8.633411695249379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/mean_length": 147.125, + "completions/min_length": 90.0, + "epoch": 1.1904761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 8.860669825935646e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 400, + "step_time": 7.072733739856631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 150.875, + "completions/min_length": 125.0, + "epoch": 1.193452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.852400573728165e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 401, + "step_time": 6.189604802057147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 169.0, + "completions/min_length": 146.0, + "epoch": 1.1964285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.844105309245598e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 402, + "step_time": 6.252419047057629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 158.125, + "completions/min_length": 93.0, + "epoch": 1.1994047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.835784088499432e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 403, + "step_time": 5.975698145106435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 350.0, + "completions/mean_length": 220.25, + "completions/min_length": 80.0, + "epoch": 1.2023809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.827436967676425e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 404, + "step_time": 7.111581088043749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/mean_length": 216.5, + "completions/min_length": 130.0, + "epoch": 1.2053571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.81906400313821e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 405, + "step_time": 6.958982706069946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 137.75, + "completions/min_length": 107.0, + "epoch": 1.2083333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.81066525142093e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 406, + "step_time": 6.587918737903237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 314.375, + "completions/min_length": 135.0, + "epoch": 1.2113095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.802240769234846e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 407, + "step_time": 10.614725422114134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 129.0, + "completions/mean_length": 106.375, + "completions/min_length": 89.0, + "epoch": 1.2142857142857142, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.793790613463954e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 408, + "step_time": 5.732173650991172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 173.0, + "completions/mean_length": 129.75, + "completions/min_length": 97.0, + "epoch": 1.2172619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.785314841165607e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 409, + "step_time": 5.894926798995584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 139.0, + "completions/min_length": 115.0, + "epoch": 1.2202380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.46875, + "learning_rate": 8.776813509570126e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 410, + "step_time": 6.364635951351374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 341.75, + "completions/min_length": 151.0, + "epoch": 1.2232142857142858, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.0625, + "learning_rate": 8.768286676080415e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 411, + "step_time": 8.942533804103732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 264.0, + "completions/mean_length": 195.875, + "completions/min_length": 139.0, + "epoch": 1.2261904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 8.75973439827157e-07, + "loss": 6.705522537231445e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 412, + "step_time": 6.57636861782521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 131.5, + "completions/min_length": 98.0, + "epoch": 1.2291666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.751156733890492e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 413, + "step_time": 6.4062091605737805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 375.0, + "completions/mean_length": 196.625, + "completions/min_length": 86.0, + "epoch": 1.2321428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.742553740855505e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 414, + "step_time": 7.298365281894803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 120.0, + "completions/mean_length": 99.375, + "completions/min_length": 81.0, + "epoch": 1.2351190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 12.0, + "learning_rate": 8.73392547725595e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 415, + "step_time": 7.151811821851879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 232.375, + "completions/min_length": 93.0, + "epoch": 1.2380952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.125, + "learning_rate": 8.725272001351803e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 416, + "step_time": 8.518492832779884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 423.0, + "completions/mean_length": 194.0, + "completions/min_length": 124.0, + "epoch": 1.2410714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.03125, + "learning_rate": 8.716593371573279e-07, + "loss": 7.450580596923828e-09, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 417, + "step_time": 7.874222898390144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 213.0, + "completions/mean_length": 131.5, + "completions/min_length": 97.0, + "epoch": 1.244047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.707889646520442e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 418, + "step_time": 5.977882527280599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 380.0, + "completions/mean_length": 228.75, + "completions/min_length": 154.0, + "epoch": 1.2470238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.75, + "learning_rate": 8.699160884962797e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 419, + "step_time": 7.7271082028746605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/mean_length": 203.625, + "completions/min_length": 162.0, + "epoch": 1.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 8.690407145838907e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 420, + "step_time": 8.08047993015498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 143.25, + "completions/min_length": 106.0, + "epoch": 1.2529761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.46875, + "learning_rate": 8.681628488255986e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 421, + "step_time": 8.015746274963021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 124.0, + "completions/mean_length": 111.0, + "completions/min_length": 88.0, + "epoch": 1.255952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.672824971489505e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 422, + "step_time": 6.176702105905861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 192.0, + "completions/mean_length": 116.375, + "completions/min_length": 90.0, + "epoch": 1.2589285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.66399665498279e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 423, + "step_time": 6.2943086377345026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/mean_length": 259.5, + "completions/min_length": 171.0, + "epoch": 1.2619047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.655143598346619e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 424, + "step_time": 7.142252197954804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 204.0, + "completions/min_length": 139.0, + "epoch": 1.2648809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.646265861358821e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 425, + "step_time": 6.5905273500829935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 172.875, + "completions/min_length": 115.0, + "epoch": 1.2678571428571428, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.71875, + "learning_rate": 8.637363503963873e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 426, + "step_time": 6.400784926023334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 344.0, + "completions/mean_length": 156.875, + "completions/min_length": 102.0, + "epoch": 1.2708333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.628436586272494e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 427, + "step_time": 7.169776183087379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 317.0, + "completions/mean_length": 225.625, + "completions/min_length": 149.0, + "epoch": 1.2738095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 8.61948516856124e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 428, + "step_time": 7.361971506848931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/mean_length": 191.875, + "completions/min_length": 105.0, + "epoch": 1.2767857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.610509311272097e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 429, + "step_time": 6.9739120760932565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 233.25, + "completions/min_length": 143.0, + "epoch": 1.2797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.601509075012072e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 430, + "step_time": 8.574857041239738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 108.625, + "completions/min_length": 86.0, + "epoch": 1.2827380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.0625, + "learning_rate": 8.592484520552785e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 431, + "step_time": 6.029499302152544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 164.5, + "completions/min_length": 137.0, + "epoch": 1.2857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.583435708830057e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 432, + "step_time": 6.67109924601391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 157.5, + "completions/min_length": 117.0, + "epoch": 1.2886904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.4375, + "learning_rate": 8.574362700943499e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 433, + "step_time": 6.5806815712712705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 269.0, + "completions/mean_length": 158.0, + "completions/min_length": 107.0, + "epoch": 1.2916666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.6875, + "learning_rate": 8.565265558156101e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 434, + "step_time": 6.9081932860426605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.0, + "completions/mean_length": 124.75, + "completions/min_length": 78.0, + "epoch": 1.2946428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.556144341893818e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 435, + "step_time": 6.27675871597603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 147.625, + "completions/min_length": 105.0, + "epoch": 1.2976190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.375, + "learning_rate": 8.546999113745152e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 436, + "step_time": 6.708735820837319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/mean_length": 115.5, + "completions/min_length": 88.0, + "epoch": 1.3005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.537829935460743e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 437, + "step_time": 7.121197847183794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.0, + "completions/mean_length": 171.125, + "completions/min_length": 114.0, + "epoch": 1.3035714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 8.528636868952943e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 438, + "step_time": 6.601202291902155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 151.875, + "completions/min_length": 109.0, + "epoch": 1.306547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.519419976295403e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 439, + "step_time": 6.046424815431237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 138.875, + "completions/min_length": 105.0, + "epoch": 1.3095238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.510179319722659e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 440, + "step_time": 6.159470907878131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/mean_length": 156.0, + "completions/min_length": 100.0, + "epoch": 1.3125, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.40625, + "learning_rate": 8.5009149616297e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 441, + "step_time": 6.294852530118078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.0, + "completions/mean_length": 199.375, + "completions/min_length": 127.0, + "epoch": 1.3154761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.491626964571555e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 442, + "step_time": 7.725416432134807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/mean_length": 180.75, + "completions/min_length": 113.0, + "epoch": 1.318452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.48231539126287e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 443, + "step_time": 8.705979803111404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 107.125, + "completions/min_length": 62.0, + "epoch": 1.3214285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.472980304577483e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 444, + "step_time": 6.175657639279962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 356.0, + "completions/mean_length": 235.375, + "completions/min_length": 122.0, + "epoch": 1.3244047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.463621767547997e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 445, + "step_time": 8.52614117693156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 213.0, + "completions/mean_length": 160.75, + "completions/min_length": 130.0, + "epoch": 1.3273809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.6875, + "learning_rate": 8.45423984336536e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 446, + "step_time": 6.573741817846894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 146.75, + "completions/min_length": 87.0, + "epoch": 1.3303571428571428, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 8.444834595378433e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 447, + "step_time": 6.8087981953285635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 293.0, + "completions/mean_length": 167.75, + "completions/min_length": 106.0, + "epoch": 1.3333333333333333, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.8125, + "learning_rate": 8.435406087093566e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 448, + "step_time": 6.780178240966052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 392.0, + "completions/mean_length": 239.0, + "completions/min_length": 116.0, + "epoch": 1.3363095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.875, + "learning_rate": 8.425954382174169e-07, + "loss": -4.470348358154297e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 449, + "step_time": 7.587192682083696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 168.0, + "completions/min_length": 89.0, + "epoch": 1.3392857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.375, + "learning_rate": 8.416479544440277e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 450, + "step_time": 6.76289453310892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 163.375, + "completions/min_length": 128.0, + "epoch": 1.3422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.406981637868127e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 451, + "step_time": 6.648111061193049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 172.25, + "completions/min_length": 145.0, + "epoch": 1.3452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.39746072658972e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 452, + "step_time": 6.581657867413014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 113.75, + "completions/min_length": 92.0, + "epoch": 1.3482142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.387916874892389e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 453, + "step_time": 5.954212611075491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/mean_length": 158.75, + "completions/min_length": 107.0, + "epoch": 1.3511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.378350147218368e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 454, + "step_time": 6.447872373275459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 179.875, + "completions/min_length": 129.0, + "epoch": 1.3541666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.78125, + "learning_rate": 8.368760608164354e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 455, + "step_time": 6.6168092912994325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 133.5, + "completions/min_length": 98.0, + "epoch": 1.3571428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.359148322481072e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 456, + "step_time": 6.246366605628282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 163.5, + "completions/min_length": 106.0, + "epoch": 1.3601190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 8.349513355072835e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 457, + "step_time": 6.37343035498634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/mean_length": 111.5, + "completions/min_length": 97.0, + "epoch": 1.3630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.339855770997112e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 458, + "step_time": 6.912858576979488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 168.75, + "completions/min_length": 106.0, + "epoch": 1.3660714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.78125, + "learning_rate": 8.33017563546408e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 459, + "step_time": 7.733357693068683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 143.625, + "completions/min_length": 89.0, + "epoch": 1.369047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 8.320473013836195e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 460, + "step_time": 6.408474809024483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 291.75, + "completions/min_length": 152.0, + "epoch": 1.3720238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.03125, + "learning_rate": 8.310747971627736e-07, + "loss": -4.470348358154297e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 461, + "step_time": 8.741665267851204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 136.25, + "completions/min_length": 86.0, + "epoch": 1.375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.301000574504378e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 462, + "step_time": 6.0388893517665565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 206.5, + "completions/min_length": 127.0, + "epoch": 1.3779761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.291230888282738e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 463, + "step_time": 8.483216572087258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 143.125, + "completions/min_length": 126.0, + "epoch": 1.380952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.281438978929935e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 464, + "step_time": 6.240995227359235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 132.75, + "completions/min_length": 84.0, + "epoch": 1.3839285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0, + "learning_rate": 8.271624912563142e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 465, + "step_time": 5.8389262510463595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 125.0, + "completions/mean_length": 99.75, + "completions/min_length": 80.0, + "epoch": 1.3869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.261788755449143e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 466, + "step_time": 5.845433293841779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 287.0, + "completions/mean_length": 202.625, + "completions/min_length": 111.0, + "epoch": 1.3898809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.251930574003885e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 467, + "step_time": 7.510769555810839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 271.125, + "completions/min_length": 221.0, + "epoch": 1.3928571428571428, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.25, + "learning_rate": 8.242050434792022e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 468, + "step_time": 7.541041410993785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 138.875, + "completions/min_length": 111.0, + "epoch": 1.3958333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.232148404526479e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 469, + "step_time": 6.223003047052771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 136.625, + "completions/min_length": 96.0, + "epoch": 1.3988095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.222224550067989e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 470, + "step_time": 6.07132978271693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/mean_length": 201.875, + "completions/min_length": 118.0, + "epoch": 1.4017857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.125, + "learning_rate": 8.212278938424653e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 471, + "step_time": 6.7516699600964785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 203.75, + "completions/min_length": 96.0, + "epoch": 1.4047619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.09375, + "learning_rate": 8.202311636751475e-07, + "loss": 5.960464477539063e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 472, + "step_time": 6.992376019246876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 173.0, + "completions/mean_length": 142.25, + "completions/min_length": 111.0, + "epoch": 1.4077380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0, + "learning_rate": 8.192322712349917e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 473, + "step_time": 6.9990641279146075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 133.75, + "completions/min_length": 93.0, + "epoch": 1.4107142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.182312232667446e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 474, + "step_time": 5.803789249621332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 199.25, + "completions/min_length": 81.0, + "epoch": 1.4136904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.71875, + "learning_rate": 8.172280265297068e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 475, + "step_time": 6.958233657293022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 131.25, + "completions/min_length": 107.0, + "epoch": 1.4166666666666667, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 8.162226877976886e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 476, + "step_time": 6.173502924852073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 456.0, + "completions/mean_length": 228.0, + "completions/min_length": 147.0, + "epoch": 1.4196428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.152152138589632e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 477, + "step_time": 8.319427941925824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 136.25, + "completions/min_length": 108.0, + "epoch": 1.4226190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.14205611516221e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 478, + "step_time": 7.208865305874497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 211.5, + "completions/min_length": 127.0, + "epoch": 1.4255952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.131938875865244e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 479, + "step_time": 6.652871783822775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 350.0, + "completions/mean_length": 182.5, + "completions/min_length": 106.0, + "epoch": 1.4285714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.121800489012607e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 480, + "step_time": 7.5827672141604125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 108.25, + "completions/min_length": 88.0, + "epoch": 1.431547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.111641023060969e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 481, + "step_time": 6.1811378402635455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/mean_length": 189.5, + "completions/min_length": 132.0, + "epoch": 1.4345238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.65625, + "learning_rate": 8.101460546609326e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 482, + "step_time": 6.577762414701283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/mean_length": 140.375, + "completions/min_length": 97.0, + "epoch": 1.4375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.091259128398546e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 483, + "step_time": 7.6867076102644205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 186.0, + "completions/mean_length": 136.625, + "completions/min_length": 112.0, + "epoch": 1.4404761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.6875, + "learning_rate": 8.0810368373109e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 484, + "step_time": 5.98314843326807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 114.5, + "completions/min_length": 86.0, + "epoch": 1.443452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.070793742369595e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 485, + "step_time": 5.572043111082166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 206.625, + "completions/min_length": 117.0, + "epoch": 1.4464285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.09375, + "learning_rate": 8.060529912738314e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 486, + "step_time": 8.680437766946852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 397.0, + "completions/mean_length": 204.375, + "completions/min_length": 108.0, + "epoch": 1.4494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.05024541772074e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 487, + "step_time": 7.898303939029574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 161.625, + "completions/min_length": 134.0, + "epoch": 1.4523809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 8.0399403267601e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 488, + "step_time": 6.401889564003795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/mean_length": 119.0, + "completions/min_length": 72.0, + "epoch": 1.4553571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.029614709438684e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 489, + "step_time": 5.976210700813681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 164.375, + "completions/min_length": 106.0, + "epoch": 1.4583333333333333, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.01926863547738e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 490, + "step_time": 6.252845821902156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/mean_length": 188.125, + "completions/min_length": 107.0, + "epoch": 1.4613095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.008902174735209e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 491, + "step_time": 8.6849489021115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 233.625, + "completions/min_length": 129.0, + "epoch": 1.4642857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.998515397208845e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 492, + "step_time": 10.544240893796086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/mean_length": 227.25, + "completions/min_length": 140.0, + "epoch": 1.4672619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.59375, + "learning_rate": 7.988108373032145e-07, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 493, + "step_time": 6.913428429048508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 145.0, + "completions/min_length": 124.0, + "epoch": 1.4702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.977681172475679e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 494, + "step_time": 6.900671160779893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 120.5, + "completions/min_length": 96.0, + "epoch": 1.4732142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.967233865946248e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 495, + "step_time": 5.839576021768153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 156.0, + "completions/mean_length": 116.625, + "completions/min_length": 95.0, + "epoch": 1.4761904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.956766523986415e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 496, + "step_time": 5.980532057117671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 128.5, + "completions/min_length": 91.0, + "epoch": 1.4791666666666667, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.94627921727403e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 497, + "step_time": 6.379812290892005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 190.625, + "completions/min_length": 137.0, + "epoch": 1.4821428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.935772016621743e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 498, + "step_time": 6.669577561318874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 184.0, + "completions/mean_length": 121.75, + "completions/min_length": 81.0, + "epoch": 1.4851190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.925244992976537e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 499, + "step_time": 5.885437068063766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 144.625, + "completions/min_length": 129.0, + "epoch": 1.4880952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.914698217419246e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 500, + "step_time": 5.913400365971029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 184.875, + "completions/min_length": 149.0, + "epoch": 1.4910714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.15625, + "learning_rate": 7.904131761164067e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 501, + "step_time": 8.005456978920847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 152.875, + "completions/min_length": 112.0, + "epoch": 1.494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.89354569555809e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 502, + "step_time": 6.312075585126877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 253.75, + "completions/min_length": 142.0, + "epoch": 1.4970238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.882940092080812e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 503, + "step_time": 8.555384458974004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 148.75, + "completions/min_length": 113.0, + "epoch": 1.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 7.872315022343652e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 504, + "step_time": 6.303891957271844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.0, + "completions/mean_length": 170.25, + "completions/min_length": 124.0, + "epoch": 1.5029761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 7.861670558089471e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 505, + "step_time": 6.926020727027208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 178.5, + "completions/min_length": 111.0, + "epoch": 1.505952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5, + "learning_rate": 7.851006771192083e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 506, + "step_time": 6.618862411007285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/mean_length": 208.875, + "completions/min_length": 133.0, + "epoch": 1.5089285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.840323733655778e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 507, + "step_time": 7.006933955941349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/mean_length": 172.875, + "completions/min_length": 118.0, + "epoch": 1.5119047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.875, + "learning_rate": 7.829621517614828e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 508, + "step_time": 6.925103840883821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/mean_length": 124.5, + "completions/min_length": 92.0, + "epoch": 1.5148809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.818900195333005e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 509, + "step_time": 5.9801436848938465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 142.0, + "completions/min_length": 100.0, + "epoch": 1.5178571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.808159839203084e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 510, + "step_time": 5.9960891250520945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 422.0, + "completions/mean_length": 197.125, + "completions/min_length": 113.0, + "epoch": 1.5208333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.797400521746364e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 511, + "step_time": 7.709240256343037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 129.875, + "completions/min_length": 91.0, + "epoch": 1.5238095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.0625, + "learning_rate": 7.786622315612181e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 512, + "step_time": 6.228770628105849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 112.75, + "completions/min_length": 78.0, + "epoch": 1.5267857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.775825293577406e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 513, + "step_time": 5.90455813286826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.0, + "completions/mean_length": 214.0, + "completions/min_length": 158.0, + "epoch": 1.5297619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.765009528545959e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 514, + "step_time": 7.338764548767358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 164.75, + "completions/min_length": 105.0, + "epoch": 1.5327380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.754175093548315e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 515, + "step_time": 6.523057112004608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 339.0, + "completions/mean_length": 263.5, + "completions/min_length": 177.0, + "epoch": 1.5357142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.1875, + "learning_rate": 7.743322061741022e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 516, + "step_time": 7.422533689066768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 310.0, + "completions/mean_length": 189.125, + "completions/min_length": 125.0, + "epoch": 1.5386904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.90625, + "learning_rate": 7.732450506406187e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 517, + "step_time": 6.787024392746389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 160.0, + "completions/mean_length": 116.875, + "completions/min_length": 85.0, + "epoch": 1.5416666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.721560500950997e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 518, + "step_time": 5.932739180047065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 105.0, + "completions/min_length": 62.0, + "epoch": 1.5446428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.71065211890722e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 519, + "step_time": 6.607396883890033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/mean_length": 198.875, + "completions/min_length": 126.0, + "epoch": 1.5476190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.699725433930704e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 520, + "step_time": 6.760377326980233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 144.875, + "completions/min_length": 72.0, + "epoch": 1.5505952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 7.688780519800881e-07, + "loss": -5.21540641784668e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 521, + "step_time": 7.047319132834673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 241.875, + "completions/min_length": 116.0, + "epoch": 1.5535714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.677817450420279e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 522, + "step_time": 8.342853073962033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 169.75, + "completions/min_length": 130.0, + "epoch": 1.556547619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 7.666836299814002e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 523, + "step_time": 6.330201772972941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 207.625, + "completions/min_length": 138.0, + "epoch": 1.5595238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.655837142129252e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 524, + "step_time": 6.880217887926847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 274.0, + "completions/mean_length": 167.25, + "completions/min_length": 121.0, + "epoch": 1.5625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.644820051634812e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 525, + "step_time": 6.933703545015305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 160.625, + "completions/min_length": 120.0, + "epoch": 1.5654761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.633785102720558e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 526, + "step_time": 6.961121235974133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 109.25, + "completions/min_length": 76.0, + "epoch": 1.568452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.622732369896945e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 527, + "step_time": 5.9073141207918525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 138.375, + "completions/min_length": 110.0, + "epoch": 1.5714285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.611661927794512e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 528, + "step_time": 5.975446774624288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 296.0, + "completions/mean_length": 251.0, + "completions/min_length": 216.0, + "epoch": 1.5744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.8125, + "learning_rate": 7.600573851163375e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 529, + "step_time": 6.722910589072853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 140.625, + "completions/min_length": 91.0, + "epoch": 1.5773809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.589468214872719e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 530, + "step_time": 6.219283614773303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 172.5, + "completions/min_length": 108.0, + "epoch": 1.5803571428571428, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 7.578345093910297e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 531, + "step_time": 6.815860792063177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 275.125, + "completions/min_length": 112.0, + "epoch": 1.5833333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.567204563381926e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 532, + "step_time": 8.754033819772303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 161.5, + "completions/min_length": 95.0, + "epoch": 1.5863095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.556046698510971e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 533, + "step_time": 6.4783472758717835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 214.625, + "completions/min_length": 123.0, + "epoch": 1.5892857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.544871574637849e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 534, + "step_time": 8.551423822063953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 145.625, + "completions/min_length": 123.0, + "epoch": 1.5922619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.533679267219506e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 535, + "step_time": 6.138499652966857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 368.0, + "completions/mean_length": 177.5, + "completions/min_length": 104.0, + "epoch": 1.5952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.522469851828918e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 536, + "step_time": 7.198045615572482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 178.125, + "completions/min_length": 123.0, + "epoch": 1.5982142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.59375, + "learning_rate": 7.511243404154584e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 537, + "step_time": 6.538632450159639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 143.625, + "completions/min_length": 93.0, + "epoch": 1.6011904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.1875, + "learning_rate": 7.5e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 538, + "step_time": 6.011594552081078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 196.125, + "completions/min_length": 151.0, + "epoch": 1.6041666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.488739715283159e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 539, + "step_time": 6.422552461270243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 192.0, + "completions/mean_length": 123.25, + "completions/min_length": 71.0, + "epoch": 1.6071428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.477462626036038e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 540, + "step_time": 6.76392219401896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 318.5, + "completions/min_length": 170.0, + "epoch": 1.6101190476190477, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.875, + "learning_rate": 7.466168808404077e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 541, + "step_time": 8.593425875063986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.0, + "completions/mean_length": 233.5, + "completions/min_length": 148.0, + "epoch": 1.6130952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.454858338645674e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 542, + "step_time": 7.455569449812174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 170.0, + "completions/mean_length": 150.0, + "completions/min_length": 115.0, + "epoch": 1.6160714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.443531293131666e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 543, + "step_time": 6.090559555683285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 214.125, + "completions/min_length": 86.0, + "epoch": 1.619047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.432187748344811e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 544, + "step_time": 6.56460028514266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 237.125, + "completions/min_length": 122.0, + "epoch": 1.6220238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.09375, + "learning_rate": 7.420827780879275e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 545, + "step_time": 8.366578328888863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 155.0, + "completions/min_length": 109.0, + "epoch": 1.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 7.40945146744011e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 546, + "step_time": 6.074601424392313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 161.5, + "completions/min_length": 108.0, + "epoch": 1.6279761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.398058884842747e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 547, + "step_time": 6.1026353649795055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/mean_length": 116.375, + "completions/min_length": 99.0, + "epoch": 1.630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.386650110012465e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 548, + "step_time": 5.429215318057686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 131.375, + "completions/min_length": 100.0, + "epoch": 1.6339285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.375225219983875e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 549, + "step_time": 5.851514117792249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 160.625, + "completions/min_length": 126.0, + "epoch": 1.6369047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5625, + "learning_rate": 7.363784291900406e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 550, + "step_time": 6.058104171883315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 134.0, + "completions/mean_length": 107.125, + "completions/min_length": 94.0, + "epoch": 1.6398809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.352327403013779e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 551, + "step_time": 5.803600009065121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 144.0, + "completions/mean_length": 107.625, + "completions/min_length": 95.0, + "epoch": 1.6428571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.34085463068348e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 552, + "step_time": 5.514120173640549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/mean_length": 174.0, + "completions/min_length": 123.0, + "epoch": 1.6458333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.329366052376253e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 553, + "step_time": 6.489137799944729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/mean_length": 240.625, + "completions/min_length": 93.0, + "epoch": 1.6488095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.375, + "learning_rate": 7.317861745665559e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 554, + "step_time": 6.7692892868071795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/mean_length": 254.375, + "completions/min_length": 157.0, + "epoch": 1.6517857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.306341788231066e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 555, + "step_time": 7.277938550338149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 184.5, + "completions/min_length": 136.0, + "epoch": 1.6547619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.294806257858117e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 556, + "step_time": 6.91539316996932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 120.75, + "completions/min_length": 102.0, + "epoch": 1.6577380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.283255232437206e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 557, + "step_time": 5.747305345721543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 151.375, + "completions/min_length": 125.0, + "epoch": 1.6607142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.4375, + "learning_rate": 7.271688789963457e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 558, + "step_time": 6.107146724127233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 208.0, + "completions/min_length": 145.0, + "epoch": 1.6636904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 7.260107008536092e-07, + "loss": 4.470348358154297e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 559, + "step_time": 7.227717824745923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 128.5, + "completions/min_length": 104.0, + "epoch": 1.6666666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.2485099663579e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 560, + "step_time": 5.512426191940904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/mean_length": 231.125, + "completions/min_length": 162.0, + "epoch": 1.6696428571428572, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.21875, + "learning_rate": 7.236897741734719e-07, + "loss": -8.195638656616211e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 561, + "step_time": 7.312939410097897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 171.375, + "completions/min_length": 126.0, + "epoch": 1.6726190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.225270413074903e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 562, + "step_time": 6.326357896905392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 296.0, + "completions/mean_length": 222.25, + "completions/min_length": 99.0, + "epoch": 1.6755952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 7.213628058888789e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 563, + "step_time": 6.7317884848453104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 302.0, + "completions/mean_length": 187.625, + "completions/min_length": 127.0, + "epoch": 1.6785714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.201970757788171e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 564, + "step_time": 7.1644079252146184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/mean_length": 115.5, + "completions/min_length": 94.0, + "epoch": 1.681547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.190298588485768e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 565, + "step_time": 5.517556677106768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 154.25, + "completions/min_length": 123.0, + "epoch": 1.6845238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.178611629794692e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 566, + "step_time": 5.9719433868303895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 115.375, + "completions/min_length": 90.0, + "epoch": 1.6875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.166909960627917e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 567, + "step_time": 5.986145778093487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 168.0, + "completions/min_length": 114.0, + "epoch": 1.6904761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.875, + "learning_rate": 7.155193659997745e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 568, + "step_time": 6.787625494878739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 336.0, + "completions/mean_length": 162.0, + "completions/min_length": 116.0, + "epoch": 1.693452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.14346280701527e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 569, + "step_time": 7.342197087127715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 197.75, + "completions/min_length": 127.0, + "epoch": 1.6964285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 7.131717480889854e-07, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 570, + "step_time": 7.0584047213196754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 137.0, + "completions/min_length": 100.0, + "epoch": 1.6994047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.119957760928578e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 571, + "step_time": 5.985019750427455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 154.75, + "completions/min_length": 114.0, + "epoch": 1.7023809523809523, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 7.108183726535717e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 572, + "step_time": 6.575057719834149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 120.0, + "completions/min_length": 78.0, + "epoch": 1.7053571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.096395457212199e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 573, + "step_time": 6.4757821573875844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 256.0, + "completions/mean_length": 197.75, + "completions/min_length": 153.0, + "epoch": 1.7083333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.08459303255507e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 574, + "step_time": 6.7163466569036245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/mean_length": 146.875, + "completions/min_length": 108.0, + "epoch": 1.7113095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.072776532256953e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 575, + "step_time": 6.788003339897841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 173.125, + "completions/min_length": 129.0, + "epoch": 1.7142857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.06094603610552e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 576, + "step_time": 6.599445888772607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 168.625, + "completions/min_length": 103.0, + "epoch": 1.7172619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.46875, + "learning_rate": 7.049101623982937e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 577, + "step_time": 6.562542331870645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 202.625, + "completions/min_length": 114.0, + "epoch": 1.7202380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 7.037243375865342e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 578, + "step_time": 8.728659316897392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.0, + "completions/mean_length": 195.75, + "completions/min_length": 128.0, + "epoch": 1.7232142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.78125, + "learning_rate": 7.025371371822293e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 579, + "step_time": 7.071826498024166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 184.25, + "completions/min_length": 95.0, + "epoch": 1.7261904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.013485692016231e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 580, + "step_time": 7.142271446995437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 361.0, + "completions/mean_length": 167.5, + "completions/min_length": 88.0, + "epoch": 1.7291666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.001586416701937e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 581, + "step_time": 7.619657818228006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 241.5, + "completions/min_length": 204.0, + "epoch": 1.7321428571428572, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 6.989673626225996e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 582, + "step_time": 7.101847340818495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/mean_length": 140.75, + "completions/min_length": 100.0, + "epoch": 1.7351190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.977747401026248e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 583, + "step_time": 6.17890197224915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 132.875, + "completions/min_length": 100.0, + "epoch": 1.7380952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.96580782163125e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 584, + "step_time": 6.526731953956187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 196.375, + "completions/min_length": 138.0, + "epoch": 1.7410714285714286, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.21875, + "learning_rate": 6.953854968659724e-07, + "loss": 5.960464477539063e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 585, + "step_time": 6.907890782225877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/mean_length": 172.25, + "completions/min_length": 100.0, + "epoch": 1.744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.375, + "learning_rate": 6.941888922820023e-07, + "loss": 4.470348358154297e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 586, + "step_time": 6.319535842165351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 155.25, + "completions/min_length": 99.0, + "epoch": 1.7470238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.625, + "learning_rate": 6.92990976490958e-07, + "loss": 4.470348358154297e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 587, + "step_time": 6.601616515312344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 151.75, + "completions/min_length": 86.0, + "epoch": 1.75, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.917917575814363e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 588, + "step_time": 6.584796139039099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/mean_length": 172.125, + "completions/min_length": 113.0, + "epoch": 1.7529761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.90591243650833e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 589, + "step_time": 6.973037629853934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 124.75, + "completions/min_length": 103.0, + "epoch": 1.755952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.89389442805288e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 590, + "step_time": 5.729716795962304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 95.875, + "completions/min_length": 63.0, + "epoch": 1.7589285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.881863631596312e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 591, + "step_time": 5.992018199991435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 308.5, + "completions/min_length": 153.0, + "epoch": 1.7619047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 6.869820128373266e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 592, + "step_time": 8.87639362923801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/mean_length": 127.0, + "completions/min_length": 93.0, + "epoch": 1.7648809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.857763999704187e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 593, + "step_time": 5.8647761177271605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/mean_length": 117.375, + "completions/min_length": 87.0, + "epoch": 1.7678571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.845695326994767e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 594, + "step_time": 6.0003216760233045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 404.0, + "completions/mean_length": 243.625, + "completions/min_length": 119.0, + "epoch": 1.7708333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.833614191735397e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 595, + "step_time": 8.019402171950787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 119.5, + "completions/min_length": 98.0, + "epoch": 1.7738095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.82152067550062e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 596, + "step_time": 5.832992972806096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 310.0, + "completions/mean_length": 176.375, + "completions/min_length": 95.0, + "epoch": 1.7767857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.809414859948578e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 597, + "step_time": 6.876212568022311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 152.875, + "completions/min_length": 92.0, + "epoch": 1.7797619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.4375, + "learning_rate": 6.797296826820459e-07, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 598, + "step_time": 6.5691957250237465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 405.0, + "completions/mean_length": 234.5, + "completions/min_length": 126.0, + "epoch": 1.7827380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.78516665793995e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 599, + "step_time": 7.684455535840243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 241.0, + "completions/mean_length": 142.375, + "completions/min_length": 116.0, + "epoch": 1.7857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.773024435212677e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 600, + "step_time": 6.693681109230965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 145.0, + "completions/min_length": 107.0, + "epoch": 1.7886904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.760870240625662e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 601, + "step_time": 6.349499138072133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 105.5, + "completions/min_length": 91.0, + "epoch": 1.7916666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.748704156246758e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 602, + "step_time": 5.5258915279991925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 170.25, + "completions/min_length": 119.0, + "epoch": 1.7946428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.7365262642241e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 603, + "step_time": 6.153776994906366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/mean_length": 161.5, + "completions/min_length": 117.0, + "epoch": 1.7976190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.72433664678556e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 604, + "step_time": 6.156213109847158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 429.0, + "completions/mean_length": 252.5, + "completions/min_length": 186.0, + "epoch": 1.8005952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.96875, + "learning_rate": 6.712135386238173e-07, + "loss": 5.960464477539063e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 605, + "step_time": 7.94815813517198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/mean_length": 110.625, + "completions/min_length": 94.0, + "epoch": 1.8035714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.699922564967595e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 606, + "step_time": 5.482465222943574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.0, + "completions/mean_length": 115.375, + "completions/min_length": 80.0, + "epoch": 1.806547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.687698265437542e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 607, + "step_time": 6.522500257007778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 173.0, + "completions/min_length": 133.0, + "epoch": 1.8095238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.675462570189232e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 608, + "step_time": 6.619840792845935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/mean_length": 234.25, + "completions/min_length": 148.0, + "epoch": 1.8125, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8125, + "learning_rate": 6.663215561840833e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 609, + "step_time": 6.843543654773384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 316.0, + "completions/mean_length": 163.25, + "completions/min_length": 115.0, + "epoch": 1.8154761904761905, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.25, + "learning_rate": 6.650957323086899e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 610, + "step_time": 7.631115891970694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 135.375, + "completions/min_length": 122.0, + "epoch": 1.818452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.59375, + "learning_rate": 6.638687936697816e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 611, + "step_time": 6.035264085046947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 317.0, + "completions/mean_length": 222.0, + "completions/min_length": 149.0, + "epoch": 1.8214285714285714, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 6.626407485519237e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 612, + "step_time": 7.109886204823852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/mean_length": 217.25, + "completions/min_length": 145.0, + "epoch": 1.8244047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.46875, + "learning_rate": 6.614116052471535e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 613, + "step_time": 7.292308156378567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.0, + "completions/mean_length": 232.75, + "completions/min_length": 123.0, + "epoch": 1.8273809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.601813720549227e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 614, + "step_time": 7.102331404108554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 126.5, + "completions/min_length": 109.0, + "epoch": 1.8303571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.589500572820427e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 615, + "step_time": 5.756903262343258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 140.125, + "completions/min_length": 86.0, + "epoch": 1.8333333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.577176692426278e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 616, + "step_time": 6.843285126145929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 227.125, + "completions/min_length": 143.0, + "epoch": 1.8363095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.56484216258039e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 617, + "step_time": 7.183432898018509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/mean_length": 188.25, + "completions/min_length": 107.0, + "epoch": 1.8392857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.552497066568281e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 618, + "step_time": 6.449534005951136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/mean_length": 140.375, + "completions/min_length": 105.0, + "epoch": 1.8422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.54014148774682e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 619, + "step_time": 6.441839111037552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 162.875, + "completions/min_length": 122.0, + "epoch": 1.8452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.527775509543653e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 620, + "step_time": 6.438742856960744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 123.0, + "completions/mean_length": 104.25, + "completions/min_length": 91.0, + "epoch": 1.8482142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.515399215456643e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 621, + "step_time": 5.335171150043607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 289.0, + "completions/mean_length": 254.75, + "completions/min_length": 123.0, + "epoch": 1.8511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.503012689053316e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 622, + "step_time": 6.731420794967562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 204.75, + "completions/min_length": 134.0, + "epoch": 1.8541666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.03125, + "learning_rate": 6.490616013970281e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 623, + "step_time": 6.755814132280648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 176.125, + "completions/min_length": 117.0, + "epoch": 1.8571428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.478209273912673e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 624, + "step_time": 6.593219838105142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 143.25, + "completions/min_length": 87.0, + "epoch": 1.8601190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.465792552653599e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 625, + "step_time": 6.192937118001282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 369.0, + "completions/mean_length": 203.625, + "completions/min_length": 135.0, + "epoch": 1.8630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.453365934033547e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 626, + "step_time": 7.094990411307663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/mean_length": 209.0, + "completions/min_length": 132.0, + "epoch": 1.8660714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.440929501959843e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 627, + "step_time": 6.746242946945131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 195.0, + "completions/min_length": 127.0, + "epoch": 1.869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.428483340406073e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 628, + "step_time": 6.658585336059332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 265.0, + "completions/mean_length": 168.375, + "completions/min_length": 128.0, + "epoch": 1.8720238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.416027533411519e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 629, + "step_time": 6.506838753353804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 180.75, + "completions/min_length": 117.0, + "epoch": 1.875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.403562165080594e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 630, + "step_time": 6.489547713659704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 212.875, + "completions/min_length": 157.0, + "epoch": 1.8779761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.391087319582263e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 631, + "step_time": 6.439553169067949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 188.625, + "completions/min_length": 130.0, + "epoch": 1.880952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.78125, + "learning_rate": 6.378603081149493e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 632, + "step_time": 6.275468077976257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 145.875, + "completions/min_length": 117.0, + "epoch": 1.8839285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.366109534078666e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 633, + "step_time": 5.674604550935328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/mean_length": 130.25, + "completions/min_length": 107.0, + "epoch": 1.8869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.353606762729024e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 634, + "step_time": 5.838054716121405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 156.0, + "completions/mean_length": 105.875, + "completions/min_length": 84.0, + "epoch": 1.8898809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.341094851522092e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 635, + "step_time": 5.879489349201322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 139.5, + "completions/min_length": 89.0, + "epoch": 1.8928571428571428, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.328573884941107e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 636, + "step_time": 5.983377147931606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/mean_length": 154.0, + "completions/min_length": 121.0, + "epoch": 1.8958333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.0625, + "learning_rate": 6.316043947530451e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 637, + "step_time": 6.475651933811605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 123.625, + "completions/min_length": 83.0, + "epoch": 1.8988095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.303505123895079e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 638, + "step_time": 5.513528850860894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 306.625, + "completions/min_length": 132.0, + "epoch": 1.9017857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.290957498699948e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 639, + "step_time": 8.670461870264262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 128.0, + "completions/mean_length": 105.125, + "completions/min_length": 79.0, + "epoch": 1.9047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.278401156669448e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 640, + "step_time": 5.470680030062795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 140.375, + "completions/min_length": 120.0, + "epoch": 1.9077380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 6.265836182586822e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 641, + "step_time": 6.079779532272369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 477.0, + "completions/mean_length": 216.5, + "completions/min_length": 119.0, + "epoch": 1.9107142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.253262661293602e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 642, + "step_time": 7.9615997658111155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 156.375, + "completions/min_length": 111.0, + "epoch": 1.9136904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 6.240680677689032e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 643, + "step_time": 6.227998401038349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 131.875, + "completions/min_length": 117.0, + "epoch": 1.9166666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.228090316729493e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 644, + "step_time": 5.946632222272456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 179.25, + "completions/min_length": 135.0, + "epoch": 1.9196428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.215491663427935e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 645, + "step_time": 6.963252102956176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 147.875, + "completions/min_length": 87.0, + "epoch": 1.9226190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.202884802853299e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 646, + "step_time": 6.390824407339096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 125.125, + "completions/min_length": 89.0, + "epoch": 1.9255952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.190269820129942e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 647, + "step_time": 5.361865345854312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 188.25, + "completions/min_length": 120.0, + "epoch": 1.9285714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.177646800437065e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 648, + "step_time": 6.6949479738250375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 130.25, + "completions/min_length": 115.0, + "epoch": 1.931547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.165015829008136e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 649, + "step_time": 6.325125282630324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/mean_length": 241.5, + "completions/min_length": 164.0, + "epoch": 1.9345238095238095, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 6.152376991130313e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 650, + "step_time": 7.508390899281949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 178.25, + "completions/min_length": 131.0, + "epoch": 1.9375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.139730372143876e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 651, + "step_time": 6.869192547164857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 162.375, + "completions/min_length": 95.0, + "epoch": 1.9404761904761905, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.127076057441637e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 652, + "step_time": 5.845087867230177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 129.625, + "completions/min_length": 94.0, + "epoch": 1.943452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.114414132468376e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 653, + "step_time": 6.131076022051275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 175.875, + "completions/min_length": 97.0, + "epoch": 1.9464285714285714, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.101744682720259e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 654, + "step_time": 7.100650453940034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 436.0, + "completions/mean_length": 263.625, + "completions/min_length": 111.0, + "epoch": 1.9494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.089067793744257e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 655, + "step_time": 7.784284115303308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 201.25, + "completions/min_length": 112.0, + "epoch": 1.9523809523809523, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.076383551137579e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 656, + "step_time": 6.314402482006699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 317.0, + "completions/mean_length": 237.125, + "completions/min_length": 139.0, + "epoch": 1.9553571428571428, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.59375, + "learning_rate": 6.063692040547083e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 657, + "step_time": 7.543929960113019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 324.0, + "completions/mean_length": 191.5, + "completions/min_length": 116.0, + "epoch": 1.9583333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.050993347668702e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 658, + "step_time": 7.187816263176501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 142.5, + "completions/min_length": 106.0, + "epoch": 1.9613095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.75, + "learning_rate": 6.038287558246867e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 659, + "step_time": 5.758392373099923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 169.25, + "completions/min_length": 105.0, + "epoch": 1.9642857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 6.025574758073925e-07, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 660, + "step_time": 6.201454643160105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 170.0, + "completions/min_length": 94.0, + "epoch": 1.9672619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.012855032989561e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 661, + "step_time": 6.400634373072535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 143.625, + "completions/min_length": 111.0, + "epoch": 1.9702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.000128468880222e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 662, + "step_time": 6.082203985191882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/mean_length": 179.875, + "completions/min_length": 143.0, + "epoch": 1.9732142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.987395151678529e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 663, + "step_time": 7.119799257256091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 140.125, + "completions/min_length": 109.0, + "epoch": 1.9761904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 5.974655167362705e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 664, + "step_time": 6.321734772995114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 176.125, + "completions/min_length": 129.0, + "epoch": 1.9791666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 5.96190860195599e-07, + "loss": 0.0, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 665, + "step_time": 6.116869555786252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 160.75, + "completions/min_length": 101.0, + "epoch": 1.9821428571428572, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.949155541526056e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 666, + "step_time": 6.640152124222368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/mean_length": 122.875, + "completions/min_length": 79.0, + "epoch": 1.9851190476190477, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.93639607218444e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 667, + "step_time": 6.113728017080575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 166.5, + "completions/min_length": 70.0, + "epoch": 1.9880952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.923630280085947e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 668, + "step_time": 6.488211444113404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 152.0, + "completions/mean_length": 119.75, + "completions/min_length": 95.0, + "epoch": 1.9910714285714286, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.910858251428076e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 669, + "step_time": 5.403573614079505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 309.0, + "completions/mean_length": 176.25, + "completions/min_length": 86.0, + "epoch": 1.994047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.898080072450436e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 670, + "step_time": 7.533651103731245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 171.875, + "completions/min_length": 128.0, + "epoch": 1.9970238095238095, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.885295829434167e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 671, + "step_time": 6.416091809980571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/mean_length": 96.25, + "completions/min_length": 77.0, + "epoch": 2.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.872505608701353e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 672, + "step_time": 5.884717517066747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 177.0, + "completions/min_length": 137.0, + "epoch": 2.0029761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.859709496614442e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 673, + "step_time": 7.041010749060661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/mean_length": 160.125, + "completions/min_length": 96.0, + "epoch": 2.005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.846907579575656e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 674, + "step_time": 6.410118679050356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 136.75, + "completions/min_length": 120.0, + "epoch": 2.0089285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.834099944026421e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 675, + "step_time": 6.197507317177951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 110.0, + "completions/mean_length": 97.875, + "completions/min_length": 74.0, + "epoch": 2.011904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.821286676446775e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 676, + "step_time": 5.711070434190333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 198.0, + "completions/mean_length": 144.0, + "completions/min_length": 109.0, + "epoch": 2.0148809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.80846786335478e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 677, + "step_time": 6.0944580947980285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/mean_length": 164.625, + "completions/min_length": 139.0, + "epoch": 2.017857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.795643591305944e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 678, + "step_time": 6.481434735935181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 256.0, + "completions/mean_length": 176.25, + "completions/min_length": 79.0, + "epoch": 2.0208333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.782813946892638e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 679, + "step_time": 6.3882345971651375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 192.75, + "completions/min_length": 134.0, + "epoch": 2.0238095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.769979016743507e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 680, + "step_time": 6.770508501678705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 195.0, + "completions/mean_length": 158.5, + "completions/min_length": 119.0, + "epoch": 2.0267857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5625, + "learning_rate": 5.757138887522883e-07, + "loss": 0.0, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 681, + "step_time": 6.352492064237595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 174.75, + "completions/min_length": 122.0, + "epoch": 2.0297619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 5.74429364593021e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 682, + "step_time": 6.890115365386009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/mean_length": 262.0, + "completions/min_length": 131.0, + "epoch": 2.0327380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.731443378699444e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 683, + "step_time": 8.886257438920438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 157.0, + "completions/min_length": 125.0, + "epoch": 2.0357142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.718588172598483e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 684, + "step_time": 6.388382066972554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 154.875, + "completions/min_length": 114.0, + "epoch": 2.0386904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.705728114428567e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 685, + "step_time": 6.015388038009405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 161.75, + "completions/min_length": 93.0, + "epoch": 2.0416666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.6875, + "learning_rate": 5.692863291023702e-07, + "loss": -5.960464477539063e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 686, + "step_time": 6.384378856047988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 211.375, + "completions/min_length": 112.0, + "epoch": 2.044642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.75, + "learning_rate": 5.679993789250074e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 687, + "step_time": 7.221760802902281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 168.875, + "completions/min_length": 122.0, + "epoch": 2.0476190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.667119696005445e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 688, + "step_time": 6.70683808485046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 177.5, + "completions/min_length": 119.0, + "epoch": 2.050595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.654241098218593e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 689, + "step_time": 6.587403789162636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 174.25, + "completions/min_length": 135.0, + "epoch": 2.0535714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 5.641358082848704e-07, + "loss": -5.960464477539063e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 690, + "step_time": 6.661484209354967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 124.0, + "completions/mean_length": 102.5, + "completions/min_length": 86.0, + "epoch": 2.056547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.628470736884796e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 691, + "step_time": 5.779400139115751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/mean_length": 133.5, + "completions/min_length": 95.0, + "epoch": 2.0595238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.615579147345129e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 692, + "step_time": 6.552673947997391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 159.5, + "completions/min_length": 132.0, + "epoch": 2.0625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.602683401276614e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 693, + "step_time": 6.305466436780989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 155.75, + "completions/min_length": 121.0, + "epoch": 2.0654761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.589783585754231e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 694, + "step_time": 5.8484010649845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 134.0, + "completions/mean_length": 104.25, + "completions/min_length": 78.0, + "epoch": 2.068452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.576879787880431e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 695, + "step_time": 5.59524996811524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 140.625, + "completions/min_length": 122.0, + "epoch": 2.0714285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.56397209478456e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 696, + "step_time": 5.86249227123335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 359.0, + "completions/mean_length": 186.875, + "completions/min_length": 97.0, + "epoch": 2.074404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.551060593622269e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 697, + "step_time": 7.843713358044624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 179.375, + "completions/min_length": 140.0, + "epoch": 2.0773809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.538145371574913e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 698, + "step_time": 6.811115520074964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/mean_length": 116.875, + "completions/min_length": 90.0, + "epoch": 2.080357142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 5.525226515848979e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 699, + "step_time": 6.057850224897265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/mean_length": 205.0, + "completions/min_length": 79.0, + "epoch": 2.0833333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.512304113675486e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 700, + "step_time": 7.062918398994952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 124.375, + "completions/min_length": 92.0, + "epoch": 2.0863095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.499378252309396e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 701, + "step_time": 6.416020652279258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 99.0, + "completions/mean_length": 87.75, + "completions/min_length": 73.0, + "epoch": 2.0892857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.486449019029038e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 702, + "step_time": 5.377194762695581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 305.125, + "completions/min_length": 172.0, + "epoch": 2.0922619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.21875, + "learning_rate": 5.4735165011355e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 703, + "step_time": 8.618447397369891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 154.5, + "completions/min_length": 92.0, + "epoch": 2.0952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.46058078595205e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 704, + "step_time": 6.98280752915889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 129.0, + "completions/min_length": 97.0, + "epoch": 2.0982142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0625, + "learning_rate": 5.447641960823548e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 705, + "step_time": 6.466237045824528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/mean_length": 168.625, + "completions/min_length": 122.0, + "epoch": 2.1011904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.59375, + "learning_rate": 5.434700113115851e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 706, + "step_time": 6.298460976686329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 237.875, + "completions/min_length": 128.0, + "epoch": 2.1041666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.921875, + "learning_rate": 5.421755330215223e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 707, + "step_time": 8.355033890344203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 193.375, + "completions/min_length": 107.0, + "epoch": 2.107142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.84375, + "learning_rate": 5.408807699527756e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 708, + "step_time": 7.400161024183035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 421.0, + "completions/mean_length": 212.125, + "completions/min_length": 139.0, + "epoch": 2.1101190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.46875, + "learning_rate": 5.395857308478756e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 709, + "step_time": 7.677660576067865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.0, + "completions/mean_length": 178.0, + "completions/min_length": 86.0, + "epoch": 2.113095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.382904244512181e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 710, + "step_time": 7.577966247219592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 188.875, + "completions/min_length": 130.0, + "epoch": 2.1160714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 5.369948595090033e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 711, + "step_time": 6.80570021411404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 143.125, + "completions/min_length": 116.0, + "epoch": 2.119047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.625, + "learning_rate": 5.356990447691765e-07, + "loss": 4.470348358154297e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 712, + "step_time": 6.3663267102092505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 177.875, + "completions/min_length": 135.0, + "epoch": 2.1220238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.344029889813708e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 713, + "step_time": 6.827689485624433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 128.625, + "completions/min_length": 91.0, + "epoch": 2.125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.331067008968462e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 714, + "step_time": 6.021626187954098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 128.125, + "completions/min_length": 102.0, + "epoch": 2.1279761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.125, + "learning_rate": 5.318101892684315e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 715, + "step_time": 5.676037495024502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/mean_length": 125.875, + "completions/min_length": 71.0, + "epoch": 2.130952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.305134628504643e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 716, + "step_time": 6.009786933660507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 168.375, + "completions/min_length": 88.0, + "epoch": 2.1339285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.292165303987336e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 717, + "step_time": 6.391666901297867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 171.75, + "completions/min_length": 122.0, + "epoch": 2.136904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.279194006704189e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 718, + "step_time": 6.46942078974098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 134.125, + "completions/min_length": 101.0, + "epoch": 2.1398809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.266220824240315e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 719, + "step_time": 6.35199364181608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/mean_length": 167.25, + "completions/min_length": 103.0, + "epoch": 2.142857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.253245844193564e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 720, + "step_time": 7.283409458119422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/mean_length": 150.25, + "completions/min_length": 115.0, + "epoch": 2.1458333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.240269154173917e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 721, + "step_time": 6.537633273750544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/mean_length": 117.875, + "completions/min_length": 99.0, + "epoch": 2.1488095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.227290841802903e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 722, + "step_time": 5.804867316968739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/mean_length": 186.75, + "completions/min_length": 127.0, + "epoch": 2.1517857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.214310994713007e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 723, + "step_time": 6.981330434791744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 104.5, + "completions/min_length": 78.0, + "epoch": 2.1547619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.201329700547076e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 724, + "step_time": 5.433821700979024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/mean_length": 124.875, + "completions/min_length": 116.0, + "epoch": 2.1577380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.188347046957728e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 725, + "step_time": 5.739325533155352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 244.0, + "completions/mean_length": 175.875, + "completions/min_length": 120.0, + "epoch": 2.1607142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.46875, + "learning_rate": 5.175363121606758e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 726, + "step_time": 6.380639856215566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 215.875, + "completions/min_length": 154.0, + "epoch": 2.1636904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.162378012164551e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 727, + "step_time": 6.978704777080566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 179.625, + "completions/min_length": 127.0, + "epoch": 2.1666666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.149391806309488e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 728, + "step_time": 6.829311961308122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 148.125, + "completions/min_length": 109.0, + "epoch": 2.169642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.40625, + "learning_rate": 5.136404591727351e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 729, + "step_time": 5.986106976866722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/mean_length": 249.375, + "completions/min_length": 143.0, + "epoch": 2.1726190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.123416456110731e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 730, + "step_time": 7.472349966876209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 182.25, + "completions/min_length": 122.0, + "epoch": 2.175595238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 5.110427487158443e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 731, + "step_time": 8.634675314184278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 158.25, + "completions/min_length": 120.0, + "epoch": 2.1785714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 5.097437772574927e-07, + "loss": -5.960464477539063e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 732, + "step_time": 6.263532727025449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 180.0, + "completions/min_length": 135.0, + "epoch": 2.181547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.084447400069654e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 733, + "step_time": 6.650523297023028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 127.875, + "completions/min_length": 110.0, + "epoch": 2.1845238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.071456457356546e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 734, + "step_time": 5.760835746303201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 157.625, + "completions/min_length": 117.0, + "epoch": 2.1875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.058465032153368e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 735, + "step_time": 6.254221213981509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 274.75, + "completions/min_length": 171.0, + "epoch": 2.1904761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.40625, + "learning_rate": 5.045473212181144e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 736, + "step_time": 8.391267244704068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 204.0, + "completions/min_length": 128.0, + "epoch": 2.193452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 5.032481085163561e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 737, + "step_time": 6.684891051147133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/mean_length": 164.875, + "completions/min_length": 101.0, + "epoch": 2.1964285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.019488738826389e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 738, + "step_time": 7.481817507185042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 152.0, + "completions/min_length": 105.0, + "epoch": 2.199404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.006496260896868e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 739, + "step_time": 6.186661561019719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 142.25, + "completions/min_length": 122.0, + "epoch": 2.2023809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.993503739103134e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 740, + "step_time": 6.159967323765159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 142.75, + "completions/min_length": 106.0, + "epoch": 2.205357142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5, + "learning_rate": 4.980511261173613e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 741, + "step_time": 6.259495310019702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 143.375, + "completions/min_length": 119.0, + "epoch": 2.2083333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.967518914836439e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 742, + "step_time": 6.3205044232308865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 170.25, + "completions/min_length": 132.0, + "epoch": 2.2113095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 4.954526787818858e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 743, + "step_time": 6.259214375168085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 212.25, + "completions/min_length": 116.0, + "epoch": 2.2142857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.941534967846633e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 744, + "step_time": 6.996774678118527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 224.0, + "completions/min_length": 112.0, + "epoch": 2.2172619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.928543542643453e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 745, + "step_time": 6.887477465905249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/mean_length": 118.625, + "completions/min_length": 90.0, + "epoch": 2.2202380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.915552599930345e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 746, + "step_time": 6.4126962958835065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/mean_length": 187.375, + "completions/min_length": 121.0, + "epoch": 2.2232142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.902562227425074e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 747, + "step_time": 7.233188767917454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/mean_length": 185.625, + "completions/min_length": 110.0, + "epoch": 2.2261904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.96875, + "learning_rate": 4.889572512841557e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 748, + "step_time": 7.317946159280837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 271.0, + "completions/mean_length": 198.625, + "completions/min_length": 132.0, + "epoch": 2.2291666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.96875, + "learning_rate": 4.87658354388927e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 749, + "step_time": 6.67089664703235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 170.875, + "completions/min_length": 126.0, + "epoch": 2.232142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.86359540827265e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 750, + "step_time": 6.299531725700945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.0, + "completions/mean_length": 140.625, + "completions/min_length": 68.0, + "epoch": 2.2351190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.850608193690512e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 751, + "step_time": 6.565877535846084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 194.125, + "completions/min_length": 111.0, + "epoch": 2.238095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.837621987835449e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 752, + "step_time": 8.873563462868333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 126.0, + "completions/mean_length": 110.5, + "completions/min_length": 97.0, + "epoch": 2.2410714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.824636878393242e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 753, + "step_time": 5.733749276958406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 334.0, + "completions/mean_length": 220.375, + "completions/min_length": 107.0, + "epoch": 2.244047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.9375, + "learning_rate": 4.811652953042274e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 754, + "step_time": 7.485757729038596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/mean_length": 112.5, + "completions/min_length": 83.0, + "epoch": 2.2470238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.798670299452926e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 755, + "step_time": 5.833051780238748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 192.0, + "completions/min_length": 103.0, + "epoch": 2.25, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.125, + "learning_rate": 4.785689005286994e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 756, + "step_time": 7.175878953188658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 143.875, + "completions/min_length": 79.0, + "epoch": 2.2529761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 10.125, + "learning_rate": 4.772709158197097e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 757, + "step_time": 6.38432688312605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 296.0, + "completions/mean_length": 253.125, + "completions/min_length": 153.0, + "epoch": 2.255952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.375, + "learning_rate": 4.759730845826084e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 758, + "step_time": 6.875125227030367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 110.0, + "completions/mean_length": 96.875, + "completions/min_length": 84.0, + "epoch": 2.2589285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5, + "learning_rate": 4.746754155806437e-07, + "loss": 6.705522537231445e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 759, + "step_time": 5.718329794239253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 167.0, + "completions/mean_length": 144.375, + "completions/min_length": 127.0, + "epoch": 2.261904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.7337791757596846e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 760, + "step_time": 5.806675442028791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 224.625, + "completions/min_length": 105.0, + "epoch": 2.2648809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 4.720805993295812e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 761, + "step_time": 8.52117916662246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 137.75, + "completions/min_length": 108.0, + "epoch": 2.267857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.7078346960126636e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 762, + "step_time": 6.244819052983075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/mean_length": 261.875, + "completions/min_length": 167.0, + "epoch": 2.2708333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.25, + "learning_rate": 4.6948653714953566e-07, + "loss": -8.940696716308594e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 763, + "step_time": 7.533643594011664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 130.0, + "completions/min_length": 86.0, + "epoch": 2.2738095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.681898107315686e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 764, + "step_time": 6.014381917193532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/mean_length": 144.125, + "completions/min_length": 101.0, + "epoch": 2.2767857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.668932991031537e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 765, + "step_time": 6.338336239103228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 163.0, + "completions/min_length": 120.0, + "epoch": 2.2797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.6559701101862916e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 766, + "step_time": 6.436910600867122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 130.625, + "completions/min_length": 98.0, + "epoch": 2.2827380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.6430095523082343e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 767, + "step_time": 5.618523363955319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/mean_length": 167.0, + "completions/min_length": 118.0, + "epoch": 2.2857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.630051404909969e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 768, + "step_time": 6.4882751177065074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 417.0, + "completions/mean_length": 209.75, + "completions/min_length": 149.0, + "epoch": 2.2886904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 4.617095755487819e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 769, + "step_time": 7.871695147361606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.0, + "completions/mean_length": 168.125, + "completions/min_length": 134.0, + "epoch": 2.2916666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.604142691521244e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 770, + "step_time": 6.671145047061145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 151.25, + "completions/min_length": 112.0, + "epoch": 2.294642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5911923004722464e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 771, + "step_time": 6.3942066370509565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 130.75, + "completions/min_length": 105.0, + "epoch": 2.2976190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5782446697847764e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 772, + "step_time": 5.747997802682221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/mean_length": 132.5, + "completions/min_length": 110.0, + "epoch": 2.300595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.565299886884149e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 773, + "step_time": 6.346112809609622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 469.0, + "completions/mean_length": 299.0, + "completions/min_length": 154.0, + "epoch": 2.3035714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 4.5523580391764523e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 774, + "step_time": 8.65570131316781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 121.125, + "completions/min_length": 96.0, + "epoch": 2.306547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5394192140479496e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 775, + "step_time": 5.743464483879507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 128.0, + "completions/mean_length": 92.0, + "completions/min_length": 58.0, + "epoch": 2.3095238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5264834988645006e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 776, + "step_time": 5.7379839909262955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 155.875, + "completions/min_length": 123.0, + "epoch": 2.3125, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.34375, + "learning_rate": 4.5135509809709617e-07, + "loss": 4.470348358154297e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 777, + "step_time": 6.671502362005413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/mean_length": 155.5, + "completions/min_length": 117.0, + "epoch": 2.3154761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.500621747690603e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 778, + "step_time": 6.052963929250836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 409.0, + "completions/mean_length": 225.75, + "completions/min_length": 118.0, + "epoch": 2.318452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.5625, + "learning_rate": 4.487695886324514e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 779, + "step_time": 7.618228960316628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 168.625, + "completions/min_length": 112.0, + "epoch": 2.3214285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.47477348415102e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 780, + "step_time": 6.255455554928631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 131.375, + "completions/min_length": 98.0, + "epoch": 2.324404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.461854628425086e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 781, + "step_time": 6.1597903817892075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 132.5, + "completions/min_length": 103.0, + "epoch": 2.3273809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.448939406377731e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 782, + "step_time": 5.774885207880288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 108.0, + "completions/mean_length": 89.875, + "completions/min_length": 76.0, + "epoch": 2.330357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.43602790521544e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 783, + "step_time": 5.544263531919569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 137.0, + "completions/min_length": 94.0, + "epoch": 2.3333333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.4231202121195705e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 784, + "step_time": 5.964478526264429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 269.125, + "completions/min_length": 109.0, + "epoch": 2.3363095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.6875, + "learning_rate": 4.4102164142457705e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 785, + "step_time": 8.346817818004638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 121.0, + "completions/mean_length": 96.125, + "completions/min_length": 83.0, + "epoch": 2.3392857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.397316598723385e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 786, + "step_time": 5.578028466086835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 276.0, + "completions/mean_length": 217.625, + "completions/min_length": 131.0, + "epoch": 2.3422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.38442085265487e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 787, + "step_time": 6.593969538342208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 346.0, + "completions/mean_length": 218.125, + "completions/min_length": 126.0, + "epoch": 2.3452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.371529263115204e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 788, + "step_time": 7.006427683867514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 129.875, + "completions/min_length": 95.0, + "epoch": 2.3482142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.3586419171512964e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 789, + "step_time": 5.810539122205228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 133.0, + "completions/min_length": 115.0, + "epoch": 2.3511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.3457589017814075e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 790, + "step_time": 5.7114599659107625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/mean_length": 180.75, + "completions/min_length": 130.0, + "epoch": 2.3541666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.332880303994555e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 791, + "step_time": 6.706970923114568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 407.0, + "completions/mean_length": 271.875, + "completions/min_length": 161.0, + "epoch": 2.357142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.320006210749927e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 792, + "step_time": 7.569053364917636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 181.625, + "completions/min_length": 89.0, + "epoch": 2.3601190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.46875, + "learning_rate": 4.3071367089762956e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 793, + "step_time": 6.956480543594807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 133.75, + "completions/min_length": 109.0, + "epoch": 2.363095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.294271885571432e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 794, + "step_time": 5.483034247998148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 384.5, + "completions/min_length": 231.0, + "epoch": 2.3660714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2814118274015165e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 795, + "step_time": 8.573509463109076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 115.875, + "completions/min_length": 83.0, + "epoch": 2.369047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2685566213005554e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 796, + "step_time": 5.5363488919101655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/mean_length": 120.125, + "completions/min_length": 86.0, + "epoch": 2.3720238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2557063540697926e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 797, + "step_time": 5.733833072241396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 272.0, + "completions/mean_length": 172.25, + "completions/min_length": 126.0, + "epoch": 2.375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.242861112477118e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 798, + "step_time": 6.852669500745833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 262.0, + "completions/mean_length": 157.0, + "completions/min_length": 98.0, + "epoch": 2.3779761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.230020983256494e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 799, + "step_time": 6.804686116054654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 153.875, + "completions/min_length": 97.0, + "epoch": 2.380952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2171860531073617e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 800, + "step_time": 6.468709263019264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 162.0, + "completions/mean_length": 129.125, + "completions/min_length": 108.0, + "epoch": 2.3839285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2043564086940556e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 801, + "step_time": 5.625169885344803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 154.875, + "completions/min_length": 134.0, + "epoch": 2.386904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1915321366452207e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 802, + "step_time": 6.1816018228419125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/mean_length": 135.25, + "completions/min_length": 85.0, + "epoch": 2.3898809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 4.178713323553225e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 803, + "step_time": 6.429743368178606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 182.375, + "completions/min_length": 133.0, + "epoch": 2.392857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1659000559735783e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 804, + "step_time": 6.465400363784283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/mean_length": 231.75, + "completions/min_length": 84.0, + "epoch": 2.3958333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1530924204243437e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 805, + "step_time": 6.684331562835723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 127.875, + "completions/min_length": 67.0, + "epoch": 2.3988095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1402905033855595e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 806, + "step_time": 6.601730263326317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 139.125, + "completions/min_length": 107.0, + "epoch": 2.4017857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1274943912986467e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 807, + "step_time": 6.1045990460552275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.0, + "completions/mean_length": 160.125, + "completions/min_length": 125.0, + "epoch": 2.4047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.114704170565833e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 808, + "step_time": 5.980292248073965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 301.0, + "completions/mean_length": 229.0, + "completions/min_length": 154.0, + "epoch": 2.4077380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.1019199275495636e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 809, + "step_time": 6.784026526845992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 118.75, + "completions/min_length": 90.0, + "epoch": 2.4107142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.089141748571925e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 810, + "step_time": 5.809976485092193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 115.0, + "completions/min_length": 84.0, + "epoch": 2.4136904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.0763697199140546e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 811, + "step_time": 5.835543476976454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 139.0, + "completions/min_length": 99.0, + "epoch": 2.4166666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.1875, + "learning_rate": 4.0636039278155607e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 812, + "step_time": 6.137678309343755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 410.0, + "completions/mean_length": 238.0, + "completions/min_length": 84.0, + "epoch": 2.419642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 4.050844458473945e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 813, + "step_time": 7.6554298610426486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/mean_length": 170.25, + "completions/min_length": 122.0, + "epoch": 2.4226190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.84375, + "learning_rate": 4.038091398044011e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 814, + "step_time": 6.717086726799607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 386.0, + "completions/mean_length": 169.875, + "completions/min_length": 107.0, + "epoch": 2.425595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.025344832637295e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 815, + "step_time": 7.6846282300539315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 186.0, + "completions/mean_length": 134.625, + "completions/min_length": 76.0, + "epoch": 2.4285714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.01260484832147e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 816, + "step_time": 5.682578143198043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/mean_length": 184.125, + "completions/min_length": 113.0, + "epoch": 2.431547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.9998715311197783e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 817, + "step_time": 6.532231421675533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/mean_length": 226.625, + "completions/min_length": 115.0, + "epoch": 2.4345238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 3.987144967010439e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 818, + "step_time": 7.221632113214582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 182.875, + "completions/min_length": 138.0, + "epoch": 2.4375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.974425241926076e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 819, + "step_time": 6.532968726940453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 147.375, + "completions/min_length": 114.0, + "epoch": 2.4404761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.9617124417531326e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 820, + "step_time": 6.269656743854284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 150.375, + "completions/min_length": 127.0, + "epoch": 2.443452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.949006652331297e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 821, + "step_time": 6.111786238849163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 150.5, + "completions/min_length": 122.0, + "epoch": 2.4464285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.9375, + "learning_rate": 3.9363079594529156e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 822, + "step_time": 6.613645658828318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 135.375, + "completions/min_length": 98.0, + "epoch": 2.449404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.92361644886242e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 823, + "step_time": 5.7579033127985895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 272.0, + "completions/mean_length": 221.0, + "completions/min_length": 150.0, + "epoch": 2.4523809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 3.910932206255742e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 824, + "step_time": 6.590191193856299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/mean_length": 226.125, + "completions/min_length": 111.0, + "epoch": 2.455357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.898255317279743e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 825, + "step_time": 7.055122679099441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 150.625, + "completions/min_length": 124.0, + "epoch": 2.4583333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.885585867531625e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 826, + "step_time": 5.860781039111316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 325.0, + "completions/mean_length": 198.0, + "completions/min_length": 139.0, + "epoch": 2.4613095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.8729239425583637e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 827, + "step_time": 6.710695268586278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/mean_length": 212.125, + "completions/min_length": 112.0, + "epoch": 2.4642857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.40625, + "learning_rate": 3.8602696278561254e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 828, + "step_time": 7.167350459843874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.0, + "completions/mean_length": 159.125, + "completions/min_length": 118.0, + "epoch": 2.4672619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.847623008869687e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 829, + "step_time": 6.511444945819676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 180.875, + "completions/min_length": 106.0, + "epoch": 2.4702380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.375, + "learning_rate": 3.8349841709918643e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 830, + "step_time": 7.022844004910439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 310.0, + "completions/mean_length": 151.75, + "completions/min_length": 104.0, + "epoch": 2.4732142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.8223531995629355e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 831, + "step_time": 6.822341015096754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 224.0, + "completions/mean_length": 143.625, + "completions/min_length": 101.0, + "epoch": 2.4761904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.375, + "learning_rate": 3.809730179870058e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 832, + "step_time": 6.252119068987668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 156.5, + "completions/min_length": 94.0, + "epoch": 2.4791666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.75, + "learning_rate": 3.7971151971467013e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 833, + "step_time": 7.289531113114208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 143.625, + "completions/min_length": 118.0, + "epoch": 2.482142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.784508336572065e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 834, + "step_time": 6.159796697087586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/mean_length": 116.375, + "completions/min_length": 86.0, + "epoch": 2.4851190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.771909683270507e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 835, + "step_time": 5.952516556717455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 233.25, + "completions/min_length": 128.0, + "epoch": 2.488095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.7593193223109677e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 836, + "step_time": 6.91282150009647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/mean_length": 163.625, + "completions/min_length": 110.0, + "epoch": 2.4910714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 3.7467373387063964e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 837, + "step_time": 7.330040080007166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 217.875, + "completions/min_length": 119.0, + "epoch": 2.494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.7341638174131764e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 838, + "step_time": 8.777691081631929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 384.0, + "completions/mean_length": 233.625, + "completions/min_length": 155.0, + "epoch": 2.4970238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 3.721598843330551e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 839, + "step_time": 7.894990711007267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 138.25, + "completions/min_length": 106.0, + "epoch": 2.5, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.7090425013000514e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 840, + "step_time": 6.119384054094553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 168.375, + "completions/min_length": 105.0, + "epoch": 2.5029761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.696494876104922e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 841, + "step_time": 6.89551093429327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 150.125, + "completions/min_length": 111.0, + "epoch": 2.505952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.683956052469551e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 842, + "step_time": 6.330699578858912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 510.0, + "completions/mean_length": 191.0, + "completions/min_length": 114.0, + "epoch": 2.508928571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.671426115058894e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 843, + "step_time": 8.642364783678204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 154.75, + "completions/min_length": 109.0, + "epoch": 2.511904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.658905148477909e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 844, + "step_time": 6.342370501253754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 465.0, + "completions/mean_length": 249.625, + "completions/min_length": 122.0, + "epoch": 2.5148809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.78125, + "learning_rate": 3.6463932372709763e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 845, + "step_time": 8.55380662670359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 162.75, + "completions/min_length": 128.0, + "epoch": 2.517857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.633890465921333e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 846, + "step_time": 6.583058323245496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 355.0, + "completions/mean_length": 220.625, + "completions/min_length": 151.0, + "epoch": 2.5208333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.40625, + "learning_rate": 3.621396918850508e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 847, + "step_time": 7.443197014275938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/mean_length": 139.25, + "completions/min_length": 84.0, + "epoch": 2.5238095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5, + "learning_rate": 3.6089126804177364e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 848, + "step_time": 5.881586753297597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/mean_length": 139.875, + "completions/min_length": 90.0, + "epoch": 2.5267857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.5964378349194066e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 849, + "step_time": 6.223151735961437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 213.0, + "completions/mean_length": 158.875, + "completions/min_length": 100.0, + "epoch": 2.5297619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.5839724665884795e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 850, + "step_time": 6.221734056249261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 129.5, + "completions/min_length": 87.0, + "epoch": 2.5327380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.571516659593926e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 851, + "step_time": 5.69882610719651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 141.0, + "completions/min_length": 118.0, + "epoch": 2.5357142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.559070498040156e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 852, + "step_time": 6.059832233935595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 363.0, + "completions/mean_length": 187.125, + "completions/min_length": 114.0, + "epoch": 2.5386904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 3.5466340659664524e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 853, + "step_time": 7.67543530697003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 176.625, + "completions/min_length": 119.0, + "epoch": 2.5416666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.875, + "learning_rate": 3.5342074473464025e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 854, + "step_time": 6.545692037791014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 150.0, + "completions/min_length": 113.0, + "epoch": 2.544642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.71875, + "learning_rate": 3.521790726087326e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 855, + "step_time": 6.884178454987705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 138.75, + "completions/min_length": 90.0, + "epoch": 2.5476190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.5093839860297205e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 856, + "step_time": 6.2058063563890755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 427.0, + "completions/mean_length": 276.125, + "completions/min_length": 181.0, + "epoch": 2.550595238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0625, + "learning_rate": 3.496987310946684e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 857, + "step_time": 8.103002517018467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 180.0, + "completions/min_length": 104.0, + "epoch": 2.553571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.484600784543356e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 858, + "step_time": 8.579990246798843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 271.0, + "completions/mean_length": 249.625, + "completions/min_length": 217.0, + "epoch": 2.556547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.4722244904563475e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 859, + "step_time": 6.636859627906233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 139.0, + "completions/min_length": 110.0, + "epoch": 2.5595238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.45985851225318e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 860, + "step_time": 5.888571377843618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/mean_length": 190.5, + "completions/min_length": 107.0, + "epoch": 2.5625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.447502933431719e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 861, + "step_time": 7.234678755048662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/mean_length": 236.0, + "completions/min_length": 110.0, + "epoch": 2.5654761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.5, + "learning_rate": 3.435157837419611e-07, + "loss": -7.450580596923828e-09, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 862, + "step_time": 7.550787798129022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/mean_length": 119.875, + "completions/min_length": 86.0, + "epoch": 2.568452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.4228233075737223e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 863, + "step_time": 5.9359823181293905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 310.375, + "completions/min_length": 192.0, + "epoch": 2.571428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.410499427179572e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 864, + "step_time": 8.723399898968637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 263.0, + "completions/mean_length": 167.5, + "completions/min_length": 115.0, + "epoch": 2.574404761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.40625, + "learning_rate": 3.398186279450772e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 865, + "step_time": 6.140366756822914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/mean_length": 158.75, + "completions/min_length": 121.0, + "epoch": 2.5773809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.3858839475284644e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 866, + "step_time": 5.810107024386525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 115.25, + "completions/min_length": 87.0, + "epoch": 2.580357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.373592514480762e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 867, + "step_time": 5.6664845258928835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 146.875, + "completions/min_length": 114.0, + "epoch": 2.5833333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.9375, + "learning_rate": 3.361312063302186e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 868, + "step_time": 5.865370790939778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 193.0, + "completions/mean_length": 143.625, + "completions/min_length": 106.0, + "epoch": 2.5863095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 3.349042676913103e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 869, + "step_time": 6.350162573158741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 440.0, + "completions/mean_length": 192.375, + "completions/min_length": 117.0, + "epoch": 2.5892857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.336784438159169e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 870, + "step_time": 8.258691359311342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 183.0, + "completions/min_length": 119.0, + "epoch": 2.5922619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.324537429810769e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 871, + "step_time": 6.698338464833796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 124.625, + "completions/min_length": 94.0, + "epoch": 2.5952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.312301734562459e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 872, + "step_time": 6.055445511359721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 202.25, + "completions/min_length": 108.0, + "epoch": 2.5982142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.03125, + "learning_rate": 3.300077435032406e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 873, + "step_time": 6.547923247329891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 155.25, + "completions/min_length": 103.0, + "epoch": 2.6011904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.125, + "learning_rate": 3.2878646137618273e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 874, + "step_time": 6.370487804990262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 419.0, + "completions/mean_length": 233.0, + "completions/min_length": 102.0, + "epoch": 2.6041666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.21875, + "learning_rate": 3.2756633532144393e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 875, + "step_time": 8.01524333562702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 152.0, + "completions/min_length": 125.0, + "epoch": 2.607142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.625, + "learning_rate": 3.263473735775899e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 876, + "step_time": 6.101528807077557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 506.0, + "completions/mean_length": 255.125, + "completions/min_length": 114.0, + "epoch": 2.6101190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.84375, + "learning_rate": 3.2512958437532424e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 877, + "step_time": 8.20470567420125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 181.25, + "completions/min_length": 119.0, + "epoch": 2.613095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 3.2391297593743374e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 878, + "step_time": 6.634721023961902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 136.125, + "completions/min_length": 102.0, + "epoch": 2.616071428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.1875, + "learning_rate": 3.2269755647873214e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 879, + "step_time": 6.513652910944074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 202.25, + "completions/min_length": 104.0, + "epoch": 2.619047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.6875, + "learning_rate": 3.2148333420600494e-07, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 880, + "step_time": 6.589510113932192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.0, + "completions/mean_length": 190.75, + "completions/min_length": 144.0, + "epoch": 2.6220238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 3.20270317317954e-07, + "loss": 4.470348358154297e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 881, + "step_time": 7.021402987651527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 127.25, + "completions/min_length": 83.0, + "epoch": 2.625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.190585140051423e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 882, + "step_time": 6.115568044129759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 140.0, + "completions/mean_length": 119.625, + "completions/min_length": 88.0, + "epoch": 2.6279761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.178479324499381e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 883, + "step_time": 5.624518316239119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 121.0, + "completions/mean_length": 100.125, + "completions/min_length": 79.0, + "epoch": 2.630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.1663858082646047e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 884, + "step_time": 5.1852946961298585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/mean_length": 163.25, + "completions/min_length": 109.0, + "epoch": 2.633928571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.59375, + "learning_rate": 3.154304673005235e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 885, + "step_time": 6.827136036939919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 350.0, + "completions/mean_length": 219.75, + "completions/min_length": 129.0, + "epoch": 2.636904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.25, + "learning_rate": 3.1422360002958136e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 886, + "step_time": 7.5724431169219315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/mean_length": 139.25, + "completions/min_length": 100.0, + "epoch": 2.6398809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.1301798716267335e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 887, + "step_time": 5.9632318103685975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.0, + "completions/mean_length": 130.0, + "completions/min_length": 76.0, + "epoch": 2.642857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.1181363684036885e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 888, + "step_time": 6.7362300041131675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 188.0, + "completions/min_length": 141.0, + "epoch": 2.6458333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.1061055719471197e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 889, + "step_time": 6.641169091686606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 122.375, + "completions/min_length": 85.0, + "epoch": 2.6488095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.125, + "learning_rate": 3.094087563491671e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 890, + "step_time": 6.006326277274638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 115.75, + "completions/min_length": 90.0, + "epoch": 2.6517857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.082082424185637e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 891, + "step_time": 5.511950637213886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 157.5, + "completions/min_length": 117.0, + "epoch": 2.6547619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.65625, + "learning_rate": 3.07009023509042e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 892, + "step_time": 6.631321289110929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 165.625, + "completions/min_length": 143.0, + "epoch": 2.6577380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.0581110771799754e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 893, + "step_time": 6.2990402723662555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 309.0, + "completions/mean_length": 186.5, + "completions/min_length": 124.0, + "epoch": 2.6607142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.0461450313402745e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 894, + "step_time": 6.919253030791879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 224.625, + "completions/min_length": 151.0, + "epoch": 2.6636904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.5, + "learning_rate": 3.03419217836875e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 895, + "step_time": 8.466134398244321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 141.0, + "completions/mean_length": 107.875, + "completions/min_length": 93.0, + "epoch": 2.6666666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.022252598973751e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 896, + "step_time": 5.585940327960998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 130.0, + "completions/mean_length": 113.125, + "completions/min_length": 93.0, + "epoch": 2.669642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.0103263737740037e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 897, + "step_time": 5.653248894959688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 164.75, + "completions/min_length": 112.0, + "epoch": 2.6726190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.998413583298064e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 898, + "step_time": 6.473078179638833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 170.0, + "completions/mean_length": 124.25, + "completions/min_length": 106.0, + "epoch": 2.675595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.986514307983771e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 899, + "step_time": 6.043903537094593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 156.0, + "completions/mean_length": 143.5, + "completions/min_length": 116.0, + "epoch": 2.678571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.9746286281777074e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 900, + "step_time": 5.572084366809577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 141.75, + "completions/min_length": 85.0, + "epoch": 2.681547619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.75, + "learning_rate": 2.962756624134658e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 901, + "step_time": 6.395345248747617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 389.0, + "completions/mean_length": 240.5, + "completions/min_length": 106.0, + "epoch": 2.6845238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.9508983760170634e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 902, + "step_time": 7.327485790010542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 136.875, + "completions/min_length": 107.0, + "epoch": 2.6875, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 2.9390539638944806e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 903, + "step_time": 5.868123146705329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 164.25, + "completions/min_length": 120.0, + "epoch": 2.6904761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.34375, + "learning_rate": 2.927223467743046e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 904, + "step_time": 6.270551833324134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 153.0, + "completions/mean_length": 124.5, + "completions/min_length": 91.0, + "epoch": 2.693452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.375, + "learning_rate": 2.915406967444932e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 905, + "step_time": 6.096761311870068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 161.0, + "completions/mean_length": 126.75, + "completions/min_length": 107.0, + "epoch": 2.696428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.75, + "learning_rate": 2.9036045427877995e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 906, + "step_time": 6.080880808178335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 300.0, + "completions/mean_length": 226.375, + "completions/min_length": 107.0, + "epoch": 2.699404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.8918162734642814e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 907, + "step_time": 6.731692139059305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 130.5, + "completions/min_length": 112.0, + "epoch": 2.7023809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.8800422390714206e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 908, + "step_time": 5.782589654903859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 151.0, + "completions/min_length": 87.0, + "epoch": 2.705357142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 2.8682825191101447e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 909, + "step_time": 6.572397619020194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/mean_length": 250.375, + "completions/min_length": 167.0, + "epoch": 2.7083333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.856537192984728e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 910, + "step_time": 8.169840735848993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 243.625, + "completions/min_length": 139.0, + "epoch": 2.7113095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.844806340002257e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 911, + "step_time": 8.756737115327269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/mean_length": 105.25, + "completions/min_length": 86.0, + "epoch": 2.7142857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.833090039372084e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 912, + "step_time": 5.5728119551204145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 190.0, + "completions/mean_length": 129.875, + "completions/min_length": 87.0, + "epoch": 2.7172619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.821388370205309e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 913, + "step_time": 5.950592671055347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/mean_length": 193.5, + "completions/min_length": 96.0, + "epoch": 2.7202380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 2.809701411514233e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 914, + "step_time": 6.588152241893113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.0, + "completions/mean_length": 192.5, + "completions/min_length": 137.0, + "epoch": 2.7232142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.7980292422118277e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 915, + "step_time": 7.416121243964881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/mean_length": 125.625, + "completions/min_length": 79.0, + "epoch": 2.7261904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.7863719411112103e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 916, + "step_time": 6.171976554207504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 281.0, + "completions/mean_length": 191.25, + "completions/min_length": 110.0, + "epoch": 2.7291666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.774729586925096e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 917, + "step_time": 7.099046261049807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 144.0, + "completions/mean_length": 123.75, + "completions/min_length": 100.0, + "epoch": 2.732142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.76310225826528e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 918, + "step_time": 5.890287369955331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 145.75, + "completions/min_length": 93.0, + "epoch": 2.7351190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.7514900336421e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 919, + "step_time": 6.160637252964079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 126.25, + "completions/min_length": 105.0, + "epoch": 2.738095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.739892991463908e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 920, + "step_time": 6.31655561318621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 459.0, + "completions/mean_length": 215.875, + "completions/min_length": 126.0, + "epoch": 2.741071428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.71875, + "learning_rate": 2.7283112100365413e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 921, + "step_time": 8.426037176977843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 121.0, + "completions/mean_length": 105.875, + "completions/min_length": 96.0, + "epoch": 2.744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.6875, + "learning_rate": 2.716744767562793e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 922, + "step_time": 5.826742818113416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 162.0, + "completions/mean_length": 129.375, + "completions/min_length": 91.0, + "epoch": 2.7470238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.705193742141883e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 923, + "step_time": 5.850444837938994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 173.875, + "completions/min_length": 145.0, + "epoch": 2.75, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.4375, + "learning_rate": 2.693658211768934e-07, + "loss": -7.450580596923828e-09, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 924, + "step_time": 6.494241142179817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 449.0, + "completions/mean_length": 168.0, + "completions/min_length": 101.0, + "epoch": 2.7529761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.71875, + "learning_rate": 2.682138254334441e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 925, + "step_time": 8.325831182766706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.0, + "completions/mean_length": 169.75, + "completions/min_length": 127.0, + "epoch": 2.755952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.670633947623748e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 926, + "step_time": 6.436518779024482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/mean_length": 160.5, + "completions/min_length": 126.0, + "epoch": 2.758928571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 2.65914536931652e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 927, + "step_time": 6.239301606081426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 165.375, + "completions/min_length": 126.0, + "epoch": 2.761904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.6476725969862226e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 928, + "step_time": 6.82735794596374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 150.375, + "completions/min_length": 110.0, + "epoch": 2.7648809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.636215708099594e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 929, + "step_time": 6.4384326627478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 166.0, + "completions/mean_length": 142.75, + "completions/min_length": 122.0, + "epoch": 2.767857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.6247747800161255e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 930, + "step_time": 5.867767903953791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 138.625, + "completions/min_length": 94.0, + "epoch": 2.7708333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.613349889987536e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 931, + "step_time": 6.140529266092926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 102.875, + "completions/min_length": 66.0, + "epoch": 2.7738095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.6019411151572537e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 932, + "step_time": 6.126621526200324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 144.0, + "completions/min_length": 122.0, + "epoch": 2.7767857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.59054853255989e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 933, + "step_time": 6.1267966250889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 249.75, + "completions/min_length": 154.0, + "epoch": 2.7797619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.9375, + "learning_rate": 2.5791722191207264e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 934, + "step_time": 7.0532582476735115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 121.0, + "completions/min_length": 85.0, + "epoch": 2.7827380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.567812251655189e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 935, + "step_time": 5.705879226326942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 407.0, + "completions/mean_length": 250.0, + "completions/min_length": 152.0, + "epoch": 2.7857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.556468706868333e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 936, + "step_time": 7.488189974334091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 111.0, + "completions/mean_length": 95.75, + "completions/min_length": 75.0, + "epoch": 2.7886904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5451416613543235e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 937, + "step_time": 5.472176753915846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 167.5, + "completions/min_length": 118.0, + "epoch": 2.7916666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.75, + "learning_rate": 2.533831191595922e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 938, + "step_time": 7.100880043581128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 226.0, + "completions/mean_length": 150.625, + "completions/min_length": 118.0, + "epoch": 2.794642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5225373739639633e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 939, + "step_time": 6.33347631432116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 114.125, + "completions/min_length": 96.0, + "epoch": 2.7976190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5112602847168416e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 940, + "step_time": 5.793334336951375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 112.0, + "completions/mean_length": 97.0, + "completions/min_length": 70.0, + "epoch": 2.800595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.500000000000001e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 941, + "step_time": 5.480075028724968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 278.0, + "completions/mean_length": 160.75, + "completions/min_length": 105.0, + "epoch": 2.803571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.4887565958454164e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 942, + "step_time": 6.901666908990592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 216.375, + "completions/min_length": 129.0, + "epoch": 2.806547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.477530148171081e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 943, + "step_time": 6.599302829243243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 147.0, + "completions/mean_length": 121.875, + "completions/min_length": 98.0, + "epoch": 2.8095238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.466320732780495e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 944, + "step_time": 5.561659432947636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 267.0, + "completions/mean_length": 142.375, + "completions/min_length": 93.0, + "epoch": 2.8125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.4551284253621523e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 945, + "step_time": 6.48388172313571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 337.0, + "completions/mean_length": 181.125, + "completions/min_length": 107.0, + "epoch": 2.8154761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.443953301489029e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 946, + "step_time": 7.170768681913614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 119.0, + "completions/mean_length": 94.625, + "completions/min_length": 74.0, + "epoch": 2.818452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 11.75, + "learning_rate": 2.4327954366180736e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 947, + "step_time": 5.6466736807487905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 205.625, + "completions/min_length": 109.0, + "epoch": 2.821428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.421654906089702e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 948, + "step_time": 6.772928972262889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 169.125, + "completions/min_length": 117.0, + "epoch": 2.824404761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 2.410531785127281e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 949, + "step_time": 6.400646863039583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 120.875, + "completions/min_length": 90.0, + "epoch": 2.8273809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.399426148836625e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 950, + "step_time": 6.007944454904646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 154.375, + "completions/min_length": 99.0, + "epoch": 2.830357142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 2.388338072205486e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 951, + "step_time": 6.251455712597817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 280.0, + "completions/mean_length": 187.875, + "completions/min_length": 147.0, + "epoch": 2.8333333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 2.3772676301030536e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 952, + "step_time": 7.003179209772497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 174.75, + "completions/min_length": 87.0, + "epoch": 2.8363095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.4375, + "learning_rate": 2.3662148972794434e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 953, + "step_time": 6.747127127368003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 152.75, + "completions/min_length": 115.0, + "epoch": 2.8392857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.355179948365189e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 954, + "step_time": 6.603515713009983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 114.625, + "completions/min_length": 100.0, + "epoch": 2.8422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.34416285787075e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 955, + "step_time": 5.706555046141148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 203.25, + "completions/min_length": 119.0, + "epoch": 2.8452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.3331637001859972e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 956, + "step_time": 6.99360666424036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 135.0, + "completions/min_length": 111.0, + "epoch": 2.8482142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 16.0, + "learning_rate": 2.322182549579721e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 957, + "step_time": 6.1186978640034795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/mean_length": 163.625, + "completions/min_length": 102.0, + "epoch": 2.8511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.311219480199117e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 958, + "step_time": 7.136003129184246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 266.125, + "completions/min_length": 111.0, + "epoch": 2.8541666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 2.3002745660692964e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 959, + "step_time": 8.436994772870094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 154.75, + "completions/min_length": 120.0, + "epoch": 2.857142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2893478810927797e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 960, + "step_time": 6.390151392202824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.0, + "completions/mean_length": 148.125, + "completions/min_length": 104.0, + "epoch": 2.8601190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.278439499049003e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 961, + "step_time": 6.0966017679311335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/mean_length": 191.875, + "completions/min_length": 126.0, + "epoch": 2.863095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 2.2675494935938138e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 962, + "step_time": 7.3620660090819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 175.125, + "completions/min_length": 114.0, + "epoch": 2.866071428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.40625, + "learning_rate": 2.2566779382589786e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 963, + "step_time": 6.601333106867969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 196.0, + "completions/mean_length": 148.75, + "completions/min_length": 96.0, + "epoch": 2.869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2458249064516842e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 964, + "step_time": 6.181686053052545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 192.0, + "completions/mean_length": 136.375, + "completions/min_length": 100.0, + "epoch": 2.8720238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 2.2349904714540424e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 965, + "step_time": 5.997990619856864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 423.0, + "completions/mean_length": 191.125, + "completions/min_length": 124.0, + "epoch": 2.875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.224174706422594e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 966, + "step_time": 7.8558966419659555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/mean_length": 177.5, + "completions/min_length": 133.0, + "epoch": 2.8779761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2133776843878183e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 967, + "step_time": 7.1009211130440235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 185.125, + "completions/min_length": 146.0, + "epoch": 2.880952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2025994782536355e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 968, + "step_time": 6.242553662043065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 275.5, + "completions/min_length": 136.0, + "epoch": 2.883928571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.191840160796918e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 969, + "step_time": 8.745815380010754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 211.875, + "completions/min_length": 138.0, + "epoch": 2.886904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1810998046669958e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 970, + "step_time": 8.378336769063026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 183.625, + "completions/min_length": 154.0, + "epoch": 2.8898809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.125, + "learning_rate": 2.170378482385171e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 971, + "step_time": 6.391270495019853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 290.0, + "completions/mean_length": 213.375, + "completions/min_length": 116.0, + "epoch": 2.892857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1596762663442213e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 972, + "step_time": 6.823626541066915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 154.5, + "completions/min_length": 113.0, + "epoch": 2.8958333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.96875, + "learning_rate": 2.1489932288079176e-07, + "loss": 6.705522537231445e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 973, + "step_time": 6.460515002254397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 149.25, + "completions/min_length": 106.0, + "epoch": 2.8988095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 2.138329441910531e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 974, + "step_time": 6.080578871071339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 130.0, + "completions/mean_length": 98.25, + "completions/min_length": 83.0, + "epoch": 2.9017857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1276849776563487e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 975, + "step_time": 5.825200038030744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/mean_length": 219.125, + "completions/min_length": 156.0, + "epoch": 2.9047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1170599079191886e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 976, + "step_time": 7.056264812592417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 162.75, + "completions/min_length": 105.0, + "epoch": 2.9077380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1064543044419103e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 977, + "step_time": 6.722529632039368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 344.0, + "completions/mean_length": 184.25, + "completions/min_length": 120.0, + "epoch": 2.9107142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5625, + "learning_rate": 2.0958682388359317e-07, + "loss": -3.725290298461914e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 978, + "step_time": 7.154385246802121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 162.125, + "completions/min_length": 117.0, + "epoch": 2.9136904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.71875, + "learning_rate": 2.0853017825807528e-07, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 979, + "step_time": 7.002933328971267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/mean_length": 161.125, + "completions/min_length": 125.0, + "epoch": 2.9166666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 2.074755007023461e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 980, + "step_time": 6.537544555030763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 362.0, + "completions/mean_length": 215.875, + "completions/min_length": 122.0, + "epoch": 2.919642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.5625, + "learning_rate": 2.0642279833782562e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 981, + "step_time": 7.279564531054348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 388.0, + "completions/mean_length": 184.5, + "completions/min_length": 117.0, + "epoch": 2.9226190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0537207827259716e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 982, + "step_time": 7.847245968878269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/mean_length": 243.375, + "completions/min_length": 147.0, + "epoch": 2.925595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0432334760135854e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 983, + "step_time": 6.858618059195578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/mean_length": 109.125, + "completions/min_length": 65.0, + "epoch": 2.928571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0327661340537533e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 984, + "step_time": 5.993539543822408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 117.5, + "completions/min_length": 89.0, + "epoch": 2.931547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0223188275243226e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 985, + "step_time": 5.684340734966099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 130.875, + "completions/min_length": 98.0, + "epoch": 2.9345238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0118916269678555e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 986, + "step_time": 5.822816350031644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 178.25, + "completions/min_length": 139.0, + "epoch": 2.9375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0014846027911565e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 987, + "step_time": 6.158100032247603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 192.0, + "completions/mean_length": 119.0, + "completions/min_length": 98.0, + "epoch": 2.9404761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.991097825264791e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 988, + "step_time": 5.923535203561187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 356.0, + "completions/mean_length": 215.5, + "completions/min_length": 140.0, + "epoch": 2.943452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9807313645226199e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 989, + "step_time": 7.560786743182689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/mean_length": 144.25, + "completions/min_length": 104.0, + "epoch": 2.946428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9703852905613166e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 990, + "step_time": 7.1315190233290195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 500.0, + "completions/mean_length": 198.5, + "completions/min_length": 111.0, + "epoch": 2.949404761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 1.9600596732398994e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 991, + "step_time": 8.12673129979521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/mean_length": 196.625, + "completions/min_length": 114.0, + "epoch": 2.9523809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.5625, + "learning_rate": 1.9497545822792582e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 992, + "step_time": 7.291526440996677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 160.125, + "completions/min_length": 122.0, + "epoch": 2.955357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9394700872616853e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 993, + "step_time": 6.435176691040397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 479.0, + "completions/mean_length": 237.5, + "completions/min_length": 152.0, + "epoch": 2.9583333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.96875, + "learning_rate": 1.9292062576304042e-07, + "loss": 0.0, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 994, + "step_time": 8.642545452807099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 158.875, + "completions/min_length": 120.0, + "epoch": 2.9613095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9189631626891e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 995, + "step_time": 6.418474182952195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 291.0, + "completions/mean_length": 226.5, + "completions/min_length": 111.0, + "epoch": 2.9642857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9087408716014557e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 996, + "step_time": 6.652342380024493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 124.5, + "completions/min_length": 89.0, + "epoch": 2.9672619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8985394533906746e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 997, + "step_time": 6.13274723989889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 310.0, + "completions/mean_length": 187.75, + "completions/min_length": 144.0, + "epoch": 2.9702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.888358976939032e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 998, + "step_time": 6.9235658356919885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/mean_length": 151.5, + "completions/min_length": 115.0, + "epoch": 2.9732142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8781995109873927e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 999, + "step_time": 6.9301703199744225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 150.5, + "completions/min_length": 109.0, + "epoch": 2.9761904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 1.8680611241347555e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1000, + "step_time": 6.500024541281164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 149.125, + "completions/min_length": 119.0, + "epoch": 2.9791666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8579438848377893e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1001, + "step_time": 6.310653881169856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 125.0, + "completions/mean_length": 104.0, + "completions/min_length": 87.0, + "epoch": 2.982142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8478478614103682e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1002, + "step_time": 5.820445710327476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 126.875, + "completions/min_length": 89.0, + "epoch": 2.9851190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.837773122023114e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1003, + "step_time": 5.973406627774239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 242.0, + "completions/mean_length": 145.0, + "completions/min_length": 106.0, + "epoch": 2.988095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.90625, + "learning_rate": 1.827719734702932e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1004, + "step_time": 6.745954179205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 301.625, + "completions/min_length": 159.0, + "epoch": 2.991071428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.90625, + "learning_rate": 1.8176877673325554e-07, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1005, + "step_time": 8.669526489917189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 125.625, + "completions/min_length": 103.0, + "epoch": 2.994047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8076772876500828e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1006, + "step_time": 6.556436249054968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 190.75, + "completions/min_length": 101.0, + "epoch": 2.9970238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7976883632485258e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1007, + "step_time": 8.787401237059385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 299.0, + "completions/mean_length": 168.75, + "completions/min_length": 104.0, + "epoch": 3.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7877210615753473e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1008, + "step_time": 6.806178995873779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 192.125, + "completions/min_length": 111.0, + "epoch": 3.0029761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7777754499320103e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1009, + "step_time": 9.037047743331641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 254.0, + "completions/mean_length": 191.75, + "completions/min_length": 146.0, + "epoch": 3.005952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7678515954735219e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1010, + "step_time": 6.558428992051631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 307.0, + "completions/mean_length": 191.5, + "completions/min_length": 125.0, + "epoch": 3.0089285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 1.7579495652079786e-07, + "loss": -4.470348358154297e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1011, + "step_time": 6.8775933012366295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 178.5, + "completions/min_length": 136.0, + "epoch": 3.011904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.748069425996116e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1012, + "step_time": 6.847863717935979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 108.25, + "completions/min_length": 94.0, + "epoch": 3.0148809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.738211244550856e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1013, + "step_time": 5.564119768794626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 207.5, + "completions/min_length": 131.0, + "epoch": 3.017857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7283750874368573e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1014, + "step_time": 8.61478887591511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 154.0, + "completions/min_length": 110.0, + "epoch": 3.0208333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7185610210700652e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1015, + "step_time": 6.727978754788637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 176.0, + "completions/mean_length": 144.375, + "completions/min_length": 98.0, + "epoch": 3.0238095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7087691117172613e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1016, + "step_time": 6.085083536803722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 321.0, + "completions/mean_length": 201.375, + "completions/min_length": 117.0, + "epoch": 3.0267857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.25, + "learning_rate": 1.6989994254956218e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1017, + "step_time": 7.019639581907541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 261.0, + "completions/mean_length": 179.25, + "completions/min_length": 115.0, + "epoch": 3.0297619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.4375, + "learning_rate": 1.689252028372264e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1018, + "step_time": 6.618381773121655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 388.0, + "completions/mean_length": 198.25, + "completions/min_length": 116.0, + "epoch": 3.0327380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5625, + "learning_rate": 1.679526986163804e-07, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1019, + "step_time": 7.501423327252269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 320.0, + "completions/mean_length": 213.625, + "completions/min_length": 71.0, + "epoch": 3.0357142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6698243645359178e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1020, + "step_time": 6.939109621103853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 151.375, + "completions/min_length": 111.0, + "epoch": 3.0386904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.5, + "learning_rate": 1.6601442290028868e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1021, + "step_time": 6.327049874700606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 126.5, + "completions/min_length": 80.0, + "epoch": 3.0416666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6504866449271632e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1022, + "step_time": 6.47642540698871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 268.0, + "completions/mean_length": 164.375, + "completions/min_length": 110.0, + "epoch": 3.044642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.21875, + "learning_rate": 1.6408516775189267e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1023, + "step_time": 6.550088417250663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 152.75, + "completions/min_length": 101.0, + "epoch": 3.0476190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.631239391835646e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1024, + "step_time": 6.78944347333163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 171.25, + "completions/min_length": 135.0, + "epoch": 3.050595238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.53125, + "learning_rate": 1.6216498527816326e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1025, + "step_time": 6.309940095059574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 131.0, + "completions/mean_length": 97.875, + "completions/min_length": 71.0, + "epoch": 3.0535714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6120831251076118e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1026, + "step_time": 5.844083239790052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 152.25, + "completions/min_length": 107.0, + "epoch": 3.056547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6025392734102817e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1027, + "step_time": 6.438396316021681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 141.375, + "completions/min_length": 113.0, + "epoch": 3.0595238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 1.5930183621318739e-07, + "loss": 5.960464477539063e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1028, + "step_time": 6.281002670060843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 453.0, + "completions/mean_length": 234.25, + "completions/min_length": 88.0, + "epoch": 3.0625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5835204555597225e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1029, + "step_time": 7.994571283925325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 164.875, + "completions/min_length": 115.0, + "epoch": 3.0654761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.574045617825831e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1030, + "step_time": 6.815342294983566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 232.625, + "completions/min_length": 144.0, + "epoch": 3.068452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5645939129064334e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1031, + "step_time": 8.82362227095291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 127.0, + "completions/min_length": 94.0, + "epoch": 3.0714285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.555165404621567e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1032, + "step_time": 6.178638786077499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 194.0, + "completions/mean_length": 137.375, + "completions/min_length": 97.0, + "epoch": 3.074404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5457601566346402e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1033, + "step_time": 6.054104306269437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 281.375, + "completions/min_length": 144.0, + "epoch": 3.0773809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.625, + "learning_rate": 1.536378232452003e-07, + "loss": -4.470348358154297e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1034, + "step_time": 8.87504196818918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.0, + "completions/mean_length": 183.75, + "completions/min_length": 135.0, + "epoch": 3.080357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5270196954225173e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1035, + "step_time": 6.7609713627025485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 175.5, + "completions/min_length": 70.0, + "epoch": 3.0833333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.517684608737129e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1036, + "step_time": 6.781629477161914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 122.25, + "completions/min_length": 85.0, + "epoch": 3.0863095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5083730354284447e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1037, + "step_time": 6.209830439649522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 112.875, + "completions/min_length": 88.0, + "epoch": 3.0892857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4990850383703e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1038, + "step_time": 6.4156482219696045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 131.0, + "completions/mean_length": 110.625, + "completions/min_length": 88.0, + "epoch": 3.0922619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4898206802773404e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1039, + "step_time": 5.751496635843068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 138.0, + "completions/min_length": 118.0, + "epoch": 3.0952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4805800237045957e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1040, + "step_time": 6.070421673357487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 159.0, + "completions/mean_length": 116.125, + "completions/min_length": 88.0, + "epoch": 3.0982142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.471363131047057e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1041, + "step_time": 6.12163055408746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.0, + "completions/mean_length": 155.625, + "completions/min_length": 100.0, + "epoch": 3.1011904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4621700645392566e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1042, + "step_time": 6.88521254202351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 175.0, + "completions/mean_length": 128.25, + "completions/min_length": 95.0, + "epoch": 3.1041666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.1875, + "learning_rate": 1.4530008862548472e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1043, + "step_time": 5.792818726040423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 168.75, + "completions/min_length": 136.0, + "epoch": 3.107142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4438556581061817e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1044, + "step_time": 6.298948595765978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/mean_length": 118.5, + "completions/min_length": 77.0, + "epoch": 3.1101190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.434734441843899e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1045, + "step_time": 5.67336820717901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 182.125, + "completions/min_length": 139.0, + "epoch": 3.113095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.15625, + "learning_rate": 1.4256372990565014e-07, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1046, + "step_time": 6.85081537021324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 165.5, + "completions/min_length": 124.0, + "epoch": 3.1160714285714284, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.5, + "learning_rate": 1.4165642911699434e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1047, + "step_time": 6.869097623042762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 228.625, + "completions/min_length": 161.0, + "epoch": 3.119047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.5625, + "learning_rate": 1.4075154794472148e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1048, + "step_time": 8.444829663727432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 251.625, + "completions/min_length": 134.0, + "epoch": 3.1220238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.9375, + "learning_rate": 1.3984909249879273e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1049, + "step_time": 6.933821368031204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 157.0, + "completions/mean_length": 112.75, + "completions/min_length": 83.0, + "epoch": 3.125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3894906887279028e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1050, + "step_time": 6.004031118005514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 165.125, + "completions/min_length": 108.0, + "epoch": 3.1279761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 1.3805148314387588e-07, + "loss": 5.960464477539063e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1051, + "step_time": 6.24251586291939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 363.0, + "completions/mean_length": 260.75, + "completions/min_length": 117.0, + "epoch": 3.130952380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.25, + "learning_rate": 1.3715634137275051e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1052, + "step_time": 7.707896555308253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 156.875, + "completions/min_length": 111.0, + "epoch": 3.1339285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.362636496036128e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1053, + "step_time": 6.717554149217904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 140.625, + "completions/min_length": 117.0, + "epoch": 3.136904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.1875, + "learning_rate": 1.35373413864118e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1054, + "step_time": 6.361561857629567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/mean_length": 210.625, + "completions/min_length": 141.0, + "epoch": 3.1398809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.8125, + "learning_rate": 1.344856401653382e-07, + "loss": -7.450580596923828e-09, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1055, + "step_time": 8.236020309850574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 113.75, + "completions/min_length": 81.0, + "epoch": 3.142857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3360033450172104e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1056, + "step_time": 5.997306917794049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 205.625, + "completions/min_length": 120.0, + "epoch": 3.1458333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.327175028510495e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1057, + "step_time": 8.345689526759088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 461.0, + "completions/mean_length": 204.75, + "completions/min_length": 132.0, + "epoch": 3.1488095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3183715117440142e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1058, + "step_time": 8.115433414001018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 117.625, + "completions/min_length": 87.0, + "epoch": 3.1517857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3095928541610933e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1059, + "step_time": 5.719858943950385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 195.0, + "completions/min_length": 141.0, + "epoch": 3.1547619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3008391150372015e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1060, + "step_time": 6.8530996148474514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/mean_length": 206.625, + "completions/min_length": 116.0, + "epoch": 3.1577380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 1.2921103534795568e-07, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1061, + "step_time": 6.779381617903709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 287.5, + "completions/min_length": 190.0, + "epoch": 3.1607142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.375, + "learning_rate": 1.2834066284267185e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1062, + "step_time": 8.92471473896876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/mean_length": 210.625, + "completions/min_length": 140.0, + "epoch": 3.1636904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.03125, + "learning_rate": 1.274727998648196e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1063, + "step_time": 7.811417555902153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 234.0, + "completions/mean_length": 153.75, + "completions/min_length": 109.0, + "epoch": 3.1666666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2660745227440494e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1064, + "step_time": 6.33535169204697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 196.625, + "completions/min_length": 114.0, + "epoch": 3.169642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.257446259144494e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1065, + "step_time": 6.785946337040514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 193.0, + "completions/min_length": 135.0, + "epoch": 3.1726190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 1.2488432661095066e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1066, + "step_time": 7.072503077797592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 166.125, + "completions/min_length": 136.0, + "epoch": 3.175595238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 1.2402656017284318e-07, + "loss": -5.960464477539063e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1067, + "step_time": 6.811713930685073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 218.375, + "completions/min_length": 106.0, + "epoch": 3.1785714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2317133239195864e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1068, + "step_time": 8.7295275577344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 121.75, + "completions/min_length": 83.0, + "epoch": 3.181547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2231864904298745e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1069, + "step_time": 5.769393462687731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 130.25, + "completions/min_length": 83.0, + "epoch": 3.1845238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.214685158834392e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1070, + "step_time": 5.791110063903034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 177.0, + "completions/mean_length": 126.75, + "completions/min_length": 93.0, + "epoch": 3.1875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2062093865360457e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1071, + "step_time": 5.8168012336827815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 142.875, + "completions/min_length": 119.0, + "epoch": 3.1904761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1977592307651536e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1072, + "step_time": 6.062745335046202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/mean_length": 228.0, + "completions/min_length": 128.0, + "epoch": 3.193452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.71875, + "learning_rate": 1.1893347485790689e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1073, + "step_time": 7.341813320759684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 160.25, + "completions/min_length": 103.0, + "epoch": 3.1964285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1809359968617893e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1074, + "step_time": 6.408021342009306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 313.0, + "completions/mean_length": 158.0, + "completions/min_length": 87.0, + "epoch": 3.199404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1725630323235758e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1075, + "step_time": 6.8137418031692505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.0, + "completions/mean_length": 175.625, + "completions/min_length": 129.0, + "epoch": 3.2023809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1642159115005679e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1076, + "step_time": 7.2981703309342265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 121.875, + "completions/min_length": 94.0, + "epoch": 3.205357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1558946907544031e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1077, + "step_time": 5.919026772957295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 115.25, + "completions/min_length": 83.0, + "epoch": 3.2083333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1475994262718347e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1078, + "step_time": 5.739186911843717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 188.375, + "completions/min_length": 114.0, + "epoch": 3.2113095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.139330174064354e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1079, + "step_time": 6.74772163014859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 109.125, + "completions/min_length": 73.0, + "epoch": 3.2142857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1310869899678121e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1080, + "step_time": 6.214003401808441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 160.25, + "completions/min_length": 118.0, + "epoch": 3.2172619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 13.5625, + "learning_rate": 1.1228699296420423e-07, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1081, + "step_time": 6.476400917861611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/mean_length": 211.625, + "completions/min_length": 128.0, + "epoch": 3.2202380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1146790485704832e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1082, + "step_time": 6.859066037926823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 145.375, + "completions/min_length": 120.0, + "epoch": 3.2232142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1065144020598066e-07, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1083, + "step_time": 6.22510750638321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 229.625, + "completions/min_length": 136.0, + "epoch": 3.2261904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0983760452395413e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1084, + "step_time": 8.510248958598822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 216.0, + "completions/min_length": 112.0, + "epoch": 3.2291666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.34375, + "learning_rate": 1.0902640330617036e-07, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1085, + "step_time": 8.733330812305212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 281.0, + "completions/mean_length": 186.625, + "completions/min_length": 154.0, + "epoch": 3.232142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0821784203004236e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1086, + "step_time": 6.79125311691314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 302.0, + "completions/mean_length": 215.5, + "completions/min_length": 95.0, + "epoch": 3.2351190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.074119261551577e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1087, + "step_time": 6.8559641959145665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 173.5, + "completions/min_length": 110.0, + "epoch": 3.238095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0660866112324168e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1088, + "step_time": 6.865543650928885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 129.75, + "completions/min_length": 108.0, + "epoch": 3.2410714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.058080523581204e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1089, + "step_time": 5.807693558279425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 238.0, + "completions/mean_length": 162.875, + "completions/min_length": 129.0, + "epoch": 3.244047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0501010526568437e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1090, + "step_time": 6.479287730995566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 135.375, + "completions/min_length": 94.0, + "epoch": 3.2470238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5625, + "learning_rate": 1.0421482523385173e-07, + "loss": -2.2351741790771484e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1091, + "step_time": 6.6212848029099405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 223.0, + "completions/mean_length": 145.125, + "completions/min_length": 110.0, + "epoch": 3.25, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0342221763253206e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1092, + "step_time": 6.17328952299431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 272.0, + "completions/mean_length": 211.5, + "completions/min_length": 171.0, + "epoch": 3.2529761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.71875, + "learning_rate": 1.0263228781359035e-07, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1093, + "step_time": 6.811699502170086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 207.75, + "completions/min_length": 142.0, + "epoch": 3.255952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0184504111081027e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1094, + "step_time": 6.9164935243315995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 161.75, + "completions/min_length": 88.0, + "epoch": 3.2589285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.3125, + "learning_rate": 1.0106048283985868e-07, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1095, + "step_time": 6.420325467828661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 135.875, + "completions/min_length": 98.0, + "epoch": 3.261904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.375, + "learning_rate": 1.0027861829824952e-07, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1096, + "step_time": 6.450817598029971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 163.125, + "completions/min_length": 101.0, + "epoch": 3.2648809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 9.94994527653078e-08, + "loss": -5.960464477539063e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1097, + "step_time": 6.533733366057277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 313.625, + "completions/min_length": 160.0, + "epoch": 3.267857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.872299150213454e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1098, + "step_time": 8.604700156021863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 274.0, + "completions/mean_length": 176.5, + "completions/min_length": 107.0, + "epoch": 3.2708333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 9.794923975157082e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1099, + "step_time": 6.893732239957899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 139.75, + "completions/min_length": 99.0, + "epoch": 3.2738095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.9375, + "learning_rate": 9.717820273816247e-08, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1100, + "step_time": 6.389982988126576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/mean_length": 201.25, + "completions/min_length": 129.0, + "epoch": 3.2767857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.5, + "learning_rate": 9.640988566812475e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1101, + "step_time": 7.009368951898068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 327.0, + "completions/mean_length": 187.75, + "completions/min_length": 112.0, + "epoch": 3.2797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.564429372930749e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1102, + "step_time": 7.439621958881617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 105.75, + "completions/min_length": 89.0, + "epoch": 3.2827380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.488143209115956e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1103, + "step_time": 5.426293505821377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 250.0, + "completions/mean_length": 148.625, + "completions/min_length": 105.0, + "epoch": 3.2857142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.412130590469437e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1104, + "step_time": 6.450218760874122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 318.0, + "completions/mean_length": 194.0, + "completions/min_length": 112.0, + "epoch": 3.2886904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.5625, + "learning_rate": 9.336392030245472e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1105, + "step_time": 7.213898214045912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 195.875, + "completions/min_length": 130.0, + "epoch": 3.2916666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.03125, + "learning_rate": 9.260928039847865e-08, + "loss": 7.450580596923828e-09, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1106, + "step_time": 6.720996948890388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 166.625, + "completions/min_length": 113.0, + "epoch": 3.294642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.185739128826453e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1107, + "step_time": 6.840813474729657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 372.0, + "completions/mean_length": 210.75, + "completions/min_length": 148.0, + "epoch": 3.2976190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.125, + "learning_rate": 9.110825804873667e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1108, + "step_time": 7.566392516251653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/mean_length": 146.0, + "completions/min_length": 105.0, + "epoch": 3.300595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.036188573821119e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1109, + "step_time": 6.436754907947034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 130.0, + "completions/mean_length": 112.625, + "completions/min_length": 94.0, + "epoch": 3.3035714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.961827939636196e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1110, + "step_time": 5.294576084241271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 198.0, + "completions/min_length": 138.0, + "epoch": 3.306547619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.125, + "learning_rate": 8.887744404418585e-08, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1111, + "step_time": 8.530205436050892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 174.0, + "completions/mean_length": 134.25, + "completions/min_length": 104.0, + "epoch": 3.3095238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.813938468397014e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1112, + "step_time": 5.628140595275909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 188.0, + "completions/mean_length": 160.5, + "completions/min_length": 132.0, + "epoch": 3.3125, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.6875, + "learning_rate": 8.740410629925744e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1113, + "step_time": 5.865061074029654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 416.0, + "completions/mean_length": 267.875, + "completions/min_length": 155.0, + "epoch": 3.3154761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.4375, + "learning_rate": 8.667161385481286e-08, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1114, + "step_time": 7.756610126234591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 212.0, + "completions/mean_length": 129.625, + "completions/min_length": 99.0, + "epoch": 3.318452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.594191229659015e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1115, + "step_time": 5.98375166580081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 348.0, + "completions/mean_length": 191.5, + "completions/min_length": 102.0, + "epoch": 3.3214285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.4375, + "learning_rate": 8.521500655169822e-08, + "loss": 5.21540641784668e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1116, + "step_time": 6.975738062057644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 146.25, + "completions/min_length": 101.0, + "epoch": 3.324404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.449090152836819e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1117, + "step_time": 6.41777729196474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 154.0, + "completions/mean_length": 125.75, + "completions/min_length": 109.0, + "epoch": 3.3273809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.37696021159201e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1118, + "step_time": 5.887773503083736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 138.5, + "completions/min_length": 123.0, + "epoch": 3.330357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.305111318472969e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1119, + "step_time": 5.801386933308095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 271.0, + "completions/mean_length": 223.5, + "completions/min_length": 164.0, + "epoch": 3.3333333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.233543958619593e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1120, + "step_time": 6.8976191859692335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 140.25, + "completions/min_length": 124.0, + "epoch": 3.3363095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.162258615270778e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1121, + "step_time": 5.911258845590055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 155.5, + "completions/min_length": 135.0, + "epoch": 3.3392857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.091255769761213e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1122, + "step_time": 6.196356901433319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/mean_length": 104.25, + "completions/min_length": 81.0, + "epoch": 3.3422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.02053590151805e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1123, + "step_time": 5.529158415738493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 209.0, + "completions/mean_length": 129.0, + "completions/min_length": 75.0, + "epoch": 3.3452380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.950099488057787e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1124, + "step_time": 6.437818514183164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 493.0, + "completions/mean_length": 270.25, + "completions/min_length": 149.0, + "epoch": 3.3482142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.879947004982895e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1125, + "step_time": 8.32171667413786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 113.875, + "completions/min_length": 99.0, + "epoch": 3.3511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.81007892597873e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1126, + "step_time": 5.488297466188669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 191.25, + "completions/min_length": 139.0, + "epoch": 3.3541666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.53125, + "learning_rate": 7.740495722810269e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1127, + "step_time": 6.6796671212650836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 274.0, + "completions/mean_length": 158.25, + "completions/min_length": 105.0, + "epoch": 3.357142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.671197865318952e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1128, + "step_time": 6.89415625995025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/mean_length": 127.25, + "completions/min_length": 110.0, + "epoch": 3.3601190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.602185821419498e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1129, + "step_time": 5.955117683857679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/mean_length": 153.125, + "completions/min_length": 97.0, + "epoch": 3.363095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.533460057096752e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1130, + "step_time": 7.032895177602768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 277.75, + "completions/min_length": 153.0, + "epoch": 3.3660714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.46502103640253e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1131, + "step_time": 8.825410533230752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 441.0, + "completions/mean_length": 216.5, + "completions/min_length": 149.0, + "epoch": 3.369047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.39686922145249e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1132, + "step_time": 7.895940988790244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 259.0, + "completions/mean_length": 175.375, + "completions/min_length": 102.0, + "epoch": 3.3720238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.25, + "learning_rate": 7.329005072423e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1133, + "step_time": 6.929906391073018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 188.5, + "completions/min_length": 109.0, + "epoch": 3.375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.261429047548084e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1134, + "step_time": 7.033536670263857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 394.0, + "completions/mean_length": 263.25, + "completions/min_length": 166.0, + "epoch": 3.3779761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.194141603116244e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1135, + "step_time": 7.676567113958299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 158.0, + "completions/mean_length": 117.75, + "completions/min_length": 84.0, + "epoch": 3.380952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.127143193467445e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1136, + "step_time": 5.7708909381181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 200.25, + "completions/min_length": 97.0, + "epoch": 3.3839285714285716, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.5, + "learning_rate": 7.060434270990012e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1137, + "step_time": 8.822875458747149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 146.0, + "completions/mean_length": 98.875, + "completions/min_length": 76.0, + "epoch": 3.386904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.994015286117605e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1138, + "step_time": 5.874928953126073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 338.0, + "completions/mean_length": 213.25, + "completions/min_length": 161.0, + "epoch": 3.3898809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.92788668732613e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1139, + "step_time": 7.248969794251025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 183.0, + "completions/mean_length": 127.25, + "completions/min_length": 84.0, + "epoch": 3.392857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.862048921130759e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1140, + "step_time": 6.35197365609929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 353.0, + "completions/mean_length": 238.875, + "completions/min_length": 136.0, + "epoch": 3.3958333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.40625, + "learning_rate": 6.796502432082901e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1141, + "step_time": 7.591695649083704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 214.0, + "completions/mean_length": 145.125, + "completions/min_length": 86.0, + "epoch": 3.3988095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.731247662767197e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1142, + "step_time": 6.432910277042538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 124.0, + "completions/min_length": 63.0, + "epoch": 3.4017857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.666285053798509e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1143, + "step_time": 6.352426301222295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 198.0, + "completions/min_length": 130.0, + "epoch": 3.4047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.601615043819009e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1144, + "step_time": 7.129181210882962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 134.625, + "completions/min_length": 90.0, + "epoch": 3.4077380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.537238069495132e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1145, + "step_time": 6.4296496021561325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 136.125, + "completions/min_length": 94.0, + "epoch": 3.4107142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.473154565514694e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1146, + "step_time": 6.286195104941726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 242.375, + "completions/min_length": 130.0, + "epoch": 3.4136904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.409364964583919e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1147, + "step_time": 8.33502396941185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 378.0, + "completions/mean_length": 220.25, + "completions/min_length": 72.0, + "epoch": 3.4166666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.345869697424545e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1148, + "step_time": 7.369664690922946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 236.0, + "completions/mean_length": 179.625, + "completions/min_length": 143.0, + "epoch": 3.419642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.282669192770895e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1149, + "step_time": 6.3554031969979405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 147.125, + "completions/min_length": 114.0, + "epoch": 3.4226190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.219763877366984e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1150, + "step_time": 6.416541514918208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 120.875, + "completions/min_length": 94.0, + "epoch": 3.425595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.157154175963664e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1151, + "step_time": 6.230465441942215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.0, + "completions/mean_length": 141.75, + "completions/min_length": 98.0, + "epoch": 3.4285714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.094840511315702e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1152, + "step_time": 6.829170815646648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 284.0, + "completions/mean_length": 193.25, + "completions/min_length": 124.0, + "epoch": 3.431547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.032823304178986e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1153, + "step_time": 6.730505251791328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 216.75, + "completions/min_length": 131.0, + "epoch": 3.4345238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.971102973307645e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1154, + "step_time": 8.749557591043413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 182.75, + "completions/min_length": 143.0, + "epoch": 3.4375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.909679935451234e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1155, + "step_time": 6.82625500112772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 199.0, + "completions/mean_length": 147.5, + "completions/min_length": 121.0, + "epoch": 3.4404761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.848554605351924e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1156, + "step_time": 6.390488464850932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 151.625, + "completions/min_length": 132.0, + "epoch": 3.443452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.787727395741682e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1157, + "step_time": 6.138099364936352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 286.0, + "completions/mean_length": 175.5, + "completions/min_length": 107.0, + "epoch": 3.4464285714285716, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.72719871733951e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1158, + "step_time": 6.802533519919962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 294.0, + "completions/mean_length": 171.125, + "completions/min_length": 127.0, + "epoch": 3.449404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.666968978848657e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1159, + "step_time": 6.89210975728929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 279.0, + "completions/mean_length": 144.0, + "completions/min_length": 96.0, + "epoch": 3.4523809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.607038586953872e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1160, + "step_time": 6.540512334089726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 197.0, + "completions/mean_length": 152.0, + "completions/min_length": 106.0, + "epoch": 3.455357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.5474079463186275e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1161, + "step_time": 5.929131073877215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 167.5, + "completions/min_length": 101.0, + "epoch": 3.4583333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.4880774595824245e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1162, + "step_time": 6.340612177737057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 240.0, + "completions/mean_length": 163.875, + "completions/min_length": 101.0, + "epoch": 3.4613095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.71875, + "learning_rate": 5.429047527358055e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1163, + "step_time": 6.562864427920431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 193.25, + "completions/min_length": 135.0, + "epoch": 3.4642857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.370318548228886e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1164, + "step_time": 6.5176958800293505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/mean_length": 200.75, + "completions/min_length": 132.0, + "epoch": 3.4672619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.96875, + "learning_rate": 5.3118909187462144e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1165, + "step_time": 7.257124089170247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 114.0, + "completions/mean_length": 93.75, + "completions/min_length": 64.0, + "epoch": 3.4702380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.253765033426527e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1166, + "step_time": 5.501200238242745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 295.0, + "completions/mean_length": 199.75, + "completions/min_length": 125.0, + "epoch": 3.4732142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 5.1959412847488785e-08, + "loss": -5.960464477539063e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1167, + "step_time": 7.032076864968985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 174.625, + "completions/min_length": 117.0, + "epoch": 3.4761904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.1384200631522046e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1168, + "step_time": 7.145562831778079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 169.0, + "completions/mean_length": 138.0, + "completions/min_length": 111.0, + "epoch": 3.4791666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.28125, + "learning_rate": 5.0812017570327434e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1169, + "step_time": 6.054321615956724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 131.625, + "completions/min_length": 87.0, + "epoch": 3.482142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.024286752741364e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1170, + "step_time": 5.915059657767415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 358.0, + "completions/mean_length": 210.875, + "completions/min_length": 138.0, + "epoch": 3.4851190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 4.967675434580981e-08, + "loss": -4.470348358154297e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1171, + "step_time": 7.231341772247106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 121.0, + "completions/mean_length": 100.625, + "completions/min_length": 90.0, + "epoch": 3.488095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.911368184803938e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1172, + "step_time": 5.670526131987572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 138.0, + "completions/mean_length": 103.0, + "completions/min_length": 73.0, + "epoch": 3.4910714285714284, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.8553653836094553e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1173, + "step_time": 5.81979101523757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 137.25, + "completions/min_length": 106.0, + "epoch": 3.494047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.799667409141034e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1174, + "step_time": 6.261006162967533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 135.0, + "completions/mean_length": 115.625, + "completions/min_length": 97.0, + "epoch": 3.4970238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.744274637483936e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1175, + "step_time": 5.836515145376325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 142.0, + "completions/mean_length": 125.25, + "completions/min_length": 100.0, + "epoch": 3.5, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0, + "learning_rate": 4.6891874426626124e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1176, + "step_time": 5.602883082348853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 309.0, + "completions/mean_length": 200.25, + "completions/min_length": 122.0, + "epoch": 3.5029761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.0, + "learning_rate": 4.634406196638185e-08, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1177, + "step_time": 7.361192472279072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 113.0, + "completions/min_length": 93.0, + "epoch": 3.505952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5799312693059535e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1178, + "step_time": 5.582809786312282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/mean_length": 124.125, + "completions/min_length": 113.0, + "epoch": 3.508928571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.5257630284928683e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1179, + "step_time": 5.990939236711711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 271.0, + "completions/mean_length": 185.375, + "completions/min_length": 75.0, + "epoch": 3.511904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.471901839955089e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1180, + "step_time": 6.564844700973481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 138.375, + "completions/min_length": 97.0, + "epoch": 3.5148809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.375, + "learning_rate": 4.4183480673754716e-08, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1181, + "step_time": 6.189931057859212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 119.625, + "completions/min_length": 84.0, + "epoch": 3.517857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.365102072361115e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1182, + "step_time": 6.357894309796393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/mean_length": 150.75, + "completions/min_length": 102.0, + "epoch": 3.5208333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.312164214440972e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1183, + "step_time": 6.83518090005964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 162.625, + "completions/min_length": 120.0, + "epoch": 3.5238095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.259534851063346e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1184, + "step_time": 6.5114108719863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 125.75, + "completions/min_length": 103.0, + "epoch": 3.5267857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.2072143375935566e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1185, + "step_time": 6.163492869120091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 158.0, + "completions/min_length": 130.0, + "epoch": 3.5297619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.4375, + "learning_rate": 4.155203027311466e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1186, + "step_time": 6.17434523999691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 235.0, + "completions/mean_length": 188.25, + "completions/min_length": 160.0, + "epoch": 3.5327380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.375, + "learning_rate": 4.103501271409143e-08, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1187, + "step_time": 6.800340131856501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 239.0, + "completions/mean_length": 145.125, + "completions/min_length": 99.0, + "epoch": 3.5357142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.0521094189884696e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1188, + "step_time": 6.325194135308266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 182.0, + "completions/mean_length": 149.375, + "completions/min_length": 103.0, + "epoch": 3.5386904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.6875, + "learning_rate": 4.0010278170587884e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1189, + "step_time": 5.757078991737217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 266.0, + "completions/mean_length": 174.25, + "completions/min_length": 86.0, + "epoch": 3.5416666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 3.950256810534575e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1190, + "step_time": 6.601916356012225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 161.75, + "completions/min_length": 118.0, + "epoch": 3.544642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.899796742233069e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1191, + "step_time": 7.294104955159128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 187.25, + "completions/min_length": 128.0, + "epoch": 3.5476190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.8496479528720096e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1192, + "step_time": 6.589683644939214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 255.0, + "completions/mean_length": 158.125, + "completions/min_length": 85.0, + "epoch": 3.550595238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 3.799810781067298e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1193, + "step_time": 6.4125700537115335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 187.0, + "completions/mean_length": 152.375, + "completions/min_length": 110.0, + "epoch": 3.553571428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.65625, + "learning_rate": 3.7502855633307237e-08, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1194, + "step_time": 5.890692523214966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 208.0, + "completions/min_length": 102.0, + "epoch": 3.556547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.7010726340677034e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1195, + "step_time": 6.609424198046327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 195.0, + "completions/mean_length": 133.75, + "completions/min_length": 106.0, + "epoch": 3.5595238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.652172325574998e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1196, + "step_time": 6.353283160831779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 218.0, + "completions/mean_length": 170.125, + "completions/min_length": 99.0, + "epoch": 3.5625, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.603584968038487e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1197, + "step_time": 6.382861425168812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 151.0, + "completions/mean_length": 122.875, + "completions/min_length": 103.0, + "epoch": 3.5654761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.5553108895309433e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1198, + "step_time": 6.020615044981241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 206.0, + "completions/mean_length": 154.875, + "completions/min_length": 122.0, + "epoch": 3.568452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 3.507350416009791e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1199, + "step_time": 6.490608276799321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 185.625, + "completions/min_length": 147.0, + "epoch": 3.571428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.459703871314945e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1200, + "step_time": 6.991834416985512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 215.0, + "completions/mean_length": 150.5, + "completions/min_length": 110.0, + "epoch": 3.574404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.412371577166578e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1201, + "step_time": 6.214704664889723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 181.0, + "completions/mean_length": 150.125, + "completions/min_length": 130.0, + "epoch": 3.5773809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.365353853163e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1202, + "step_time": 6.311940109822899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 249.0, + "completions/mean_length": 164.25, + "completions/min_length": 106.0, + "epoch": 3.580357142857143, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 3.3186510167784456e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1203, + "step_time": 6.8866479131393135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 203.0, + "completions/mean_length": 132.5, + "completions/min_length": 95.0, + "epoch": 3.5833333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.272263383360979e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1204, + "step_time": 6.450384235940874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 273.0, + "completions/mean_length": 160.5, + "completions/min_length": 98.0, + "epoch": 3.5863095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.84375, + "learning_rate": 3.226191266130329e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1205, + "step_time": 6.639422245789319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 231.0, + "completions/mean_length": 173.5, + "completions/min_length": 123.0, + "epoch": 3.5892857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.180434976175783e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1206, + "step_time": 6.669356579892337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 176.125, + "completions/min_length": 110.0, + "epoch": 3.5922619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.134994822454118e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1207, + "step_time": 6.600267620291561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 188.375, + "completions/min_length": 136.0, + "epoch": 3.5952380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.0898711117874675e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1208, + "step_time": 6.3692787969484925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 288.0, + "completions/mean_length": 168.0, + "completions/min_length": 109.0, + "epoch": 3.5982142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 3.0450641488612816e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1209, + "step_time": 6.719360398128629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 217.0, + "completions/mean_length": 155.125, + "completions/min_length": 115.0, + "epoch": 3.6011904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.0, + "learning_rate": 3.0005742362222305e-08, + "loss": 7.450580596923828e-09, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1210, + "step_time": 6.530192910227925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 176.75, + "completions/min_length": 122.0, + "epoch": 3.6041666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.956401674276221e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1211, + "step_time": 6.261587913148105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 228.0, + "completions/mean_length": 177.0, + "completions/min_length": 96.0, + "epoch": 3.607142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.912546761286333e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1212, + "step_time": 6.515226217918098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 201.0, + "completions/mean_length": 155.375, + "completions/min_length": 115.0, + "epoch": 3.6101190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.869009793370786e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1213, + "step_time": 6.124011619947851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 201.25, + "completions/min_length": 144.0, + "epoch": 3.613095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.46875, + "learning_rate": 2.825791064500993e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1214, + "step_time": 6.996264931280166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 137.0, + "completions/mean_length": 128.625, + "completions/min_length": 118.0, + "epoch": 3.616071428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.7828908664995208e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1215, + "step_time": 5.8593411929905415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 137.25, + "completions/min_length": 93.0, + "epoch": 3.619047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.7403094890381672e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1216, + "step_time": 6.4399099331349134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.0, + "completions/mean_length": 172.5, + "completions/min_length": 131.0, + "epoch": 3.6220238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.65625, + "learning_rate": 2.6980472196359627e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1217, + "step_time": 6.15361242601648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 208.0, + "completions/mean_length": 177.625, + "completions/min_length": 127.0, + "epoch": 3.625, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.96875, + "learning_rate": 2.6561043436572506e-08, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1218, + "step_time": 6.50329936016351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 253.0, + "completions/mean_length": 159.0, + "completions/min_length": 103.0, + "epoch": 3.6279761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.375, + "learning_rate": 2.6144811443097714e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1219, + "step_time": 6.93765261489898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 155.25, + "completions/min_length": 95.0, + "epoch": 3.630952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5731779026427257e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1220, + "step_time": 6.612741391174495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 222.625, + "completions/min_length": 150.0, + "epoch": 3.633928571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 2.5321948975448813e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1221, + "step_time": 8.621089323889464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 245.0, + "completions/mean_length": 161.375, + "completions/min_length": 100.0, + "epoch": 3.636904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 2.491532405742719e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1222, + "step_time": 6.503277706447989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.0, + "completions/mean_length": 256.625, + "completions/min_length": 189.0, + "epoch": 3.6398809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.4511907017985224e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1223, + "step_time": 7.326947073917836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 211.0, + "completions/mean_length": 136.5, + "completions/min_length": 97.0, + "epoch": 3.642857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.411170058108558e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1224, + "step_time": 5.945022232364863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/mean_length": 135.75, + "completions/min_length": 106.0, + "epoch": 3.6458333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.3125, + "learning_rate": 2.3714707449011884e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1225, + "step_time": 6.23795718094334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 324.375, + "completions/min_length": 109.0, + "epoch": 3.6488095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.25, + "learning_rate": 2.3320930302351327e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1226, + "step_time": 8.568667862098664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 169.25, + "completions/min_length": 135.0, + "epoch": 3.6517857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.293037179997559e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1227, + "step_time": 6.7445675428025424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 246.0, + "completions/mean_length": 142.625, + "completions/min_length": 86.0, + "epoch": 3.6547619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.2543034579023566e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1228, + "step_time": 6.2153830118477345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 148.875, + "completions/min_length": 97.0, + "epoch": 3.6577380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.84375, + "learning_rate": 2.2158921254883333e-08, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1229, + "step_time": 6.08201769599691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 126.75, + "completions/min_length": 108.0, + "epoch": 3.6607142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.177803442117443e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1230, + "step_time": 5.990037824958563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 354.0, + "completions/mean_length": 241.875, + "completions/min_length": 174.0, + "epoch": 3.6636904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 2.1400376649730377e-08, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1231, + "step_time": 7.182850937824696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 175.125, + "completions/min_length": 122.0, + "epoch": 3.6666666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.102595049058137e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1232, + "step_time": 6.320400359109044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 347.0, + "completions/mean_length": 227.0, + "completions/min_length": 125.0, + "epoch": 3.669642857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 2.0654758471937096e-08, + "loss": -3.725290298461914e-08, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1233, + "step_time": 7.267035856842995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 119.25, + "completions/min_length": 96.0, + "epoch": 3.6726190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0286803100169503e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1234, + "step_time": 6.244164763018489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 221.0, + "completions/mean_length": 154.125, + "completions/min_length": 116.0, + "epoch": 3.675595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9922086859796072e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1235, + "step_time": 6.670403733383864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 326.0, + "completions/mean_length": 236.875, + "completions/min_length": 158.0, + "epoch": 3.678571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9560612213462835e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1236, + "step_time": 7.255850442219526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 151.0, + "completions/min_length": 127.0, + "epoch": 3.681547619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.625, + "learning_rate": 1.9202381601927885e-08, + "loss": 0.0, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1237, + "step_time": 6.873965756967664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 308.0, + "completions/mean_length": 175.125, + "completions/min_length": 115.0, + "epoch": 3.6845238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.3125, + "learning_rate": 1.884739744404501e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1238, + "step_time": 7.275797336362302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 127.0, + "completions/mean_length": 106.375, + "completions/min_length": 79.0, + "epoch": 3.6875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8495662136746915e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1239, + "step_time": 5.704438271932304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 115.625, + "completions/min_length": 85.0, + "epoch": 3.6904761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8147178055029577e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1240, + "step_time": 5.795591803733259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 220.0, + "completions/mean_length": 147.875, + "completions/min_length": 90.0, + "epoch": 3.693452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7801947551935925e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1241, + "step_time": 6.545267918147147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 131.0, + "completions/mean_length": 96.125, + "completions/min_length": 74.0, + "epoch": 3.696428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.5, + "learning_rate": 1.745997295853985e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1242, + "step_time": 5.864121242891997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/mean_length": 152.125, + "completions/min_length": 121.0, + "epoch": 3.699404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.712125658393082e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1243, + "step_time": 6.165337051264942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 233.0, + "completions/mean_length": 192.0, + "completions/min_length": 155.0, + "epoch": 3.7023809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.6875, + "learning_rate": 1.6785800715197918e-08, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1244, + "step_time": 6.282397579867393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 207.0, + "completions/mean_length": 144.75, + "completions/min_length": 126.0, + "epoch": 3.705357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.64536076174146e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1245, + "step_time": 6.093568724114448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 165.0, + "completions/mean_length": 129.0, + "completions/min_length": 102.0, + "epoch": 3.7083333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.875, + "learning_rate": 1.6124679533623452e-08, + "loss": 0.0, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1246, + "step_time": 6.1914246790111065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 292.0, + "completions/mean_length": 176.75, + "completions/min_length": 120.0, + "epoch": 3.7113095238095237, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.875, + "learning_rate": 1.5799018684820752e-08, + "loss": -4.470348358154297e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1247, + "step_time": 6.7646560040302575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 216.0, + "completions/mean_length": 157.25, + "completions/min_length": 99.0, + "epoch": 3.7142857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.28125, + "learning_rate": 1.5476627269941923e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1248, + "step_time": 6.265919011551887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 118.0, + "completions/mean_length": 102.25, + "completions/min_length": 83.0, + "epoch": 3.7172619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5157507465846264e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1249, + "step_time": 5.682287706062198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 219.0, + "completions/mean_length": 142.0, + "completions/min_length": 91.0, + "epoch": 3.7202380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4841661427302477e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1250, + "step_time": 6.31353773502633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 179.0, + "completions/mean_length": 128.75, + "completions/min_length": 78.0, + "epoch": 3.7232142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4529091286973993e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1251, + "step_time": 6.211642650421709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 155.0, + "completions/mean_length": 104.625, + "completions/min_length": 75.0, + "epoch": 3.7261904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.4219799155404777e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1252, + "step_time": 6.028333657886833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 208.0, + "completions/mean_length": 127.875, + "completions/min_length": 101.0, + "epoch": 3.7291666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3913787121004716e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1253, + "step_time": 5.898204482160509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 314.0, + "completions/mean_length": 166.625, + "completions/min_length": 121.0, + "epoch": 3.732142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3611057250036028e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1254, + "step_time": 6.909268037416041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 282.0, + "completions/mean_length": 168.0, + "completions/min_length": 136.0, + "epoch": 3.7351190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3311611586598826e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1255, + "step_time": 6.599874787032604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 285.625, + "completions/min_length": 149.0, + "epoch": 3.738095238095238, + "frac_reward_zero_std": 0.0, + "grad_norm": 3.640625, + "learning_rate": 1.3015452152617567e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1256, + "step_time": 8.558803633786738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 150.5, + "completions/min_length": 118.0, + "epoch": 3.741071428571429, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.84375, + "learning_rate": 1.272258094782741e-08, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1257, + "step_time": 6.468092813156545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 448.0, + "completions/mean_length": 262.375, + "completions/min_length": 157.0, + "epoch": 3.744047619047619, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.46875, + "learning_rate": 1.2432999949760548e-08, + "loss": 0.0, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1258, + "step_time": 8.417329990770668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 150.0, + "completions/mean_length": 127.875, + "completions/min_length": 112.0, + "epoch": 3.7470238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2146711113733054e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1259, + "step_time": 5.9246182651259005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/mean_length": 202.5, + "completions/min_length": 123.0, + "epoch": 3.75, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1863716372831511e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1260, + "step_time": 7.343193044420332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 248.0, + "completions/mean_length": 154.5, + "completions/min_length": 119.0, + "epoch": 3.7529761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.6875, + "learning_rate": 1.1584017637900068e-08, + "loss": 2.2351741790771484e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1261, + "step_time": 6.746955644804984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 307.0, + "completions/mean_length": 179.0, + "completions/min_length": 111.0, + "epoch": 3.755952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1307616797527453e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1262, + "step_time": 9.03919361019507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 331.0, + "completions/mean_length": 229.625, + "completions/min_length": 94.0, + "epoch": 3.758928571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1034515718034321e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1263, + "step_time": 7.025586795993149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 297.0, + "completions/mean_length": 216.625, + "completions/min_length": 90.0, + "epoch": 3.761904761904762, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.3125, + "learning_rate": 1.0764716243460592e-08, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1264, + "step_time": 6.827644960023463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/mean_length": 144.0, + "completions/min_length": 85.0, + "epoch": 3.7648809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0498220195552964e-08, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1265, + "step_time": 6.898958129808307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 143.0, + "completions/mean_length": 108.625, + "completions/min_length": 86.0, + "epoch": 3.767857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0235029373752757e-08, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1266, + "step_time": 5.9709862330928445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 205.0, + "completions/mean_length": 139.5, + "completions/min_length": 109.0, + "epoch": 3.7708333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.975145555183528e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1267, + "step_time": 7.3350083930417895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 319.0, + "completions/mean_length": 212.875, + "completions/min_length": 144.0, + "epoch": 3.7738095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.718570494639312e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1268, + "step_time": 7.019373428076506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/mean_length": 214.0, + "completions/min_length": 143.0, + "epoch": 3.7767857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.465305924572564e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1269, + "step_time": 6.7627632999792695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 512.0, + "completions/mean_length": 256.875, + "completions/min_length": 109.0, + "epoch": 3.7797619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.215353555082684e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1270, + "step_time": 10.221094739623368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/mean_length": 128.625, + "completions/min_length": 97.0, + "epoch": 3.7827380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.968715073904231e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1271, + "step_time": 5.804490907117724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 200.0, + "completions/mean_length": 150.5, + "completions/min_length": 121.0, + "epoch": 3.7857142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.71875, + "learning_rate": 8.725392146395726e-09, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1272, + "step_time": 6.368190534878522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 175.875, + "completions/min_length": 122.0, + "epoch": 3.7886904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.65625, + "learning_rate": 8.485386415528318e-09, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1273, + "step_time": 6.453891560435295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 225.0, + "completions/mean_length": 135.375, + "completions/min_length": 97.0, + "epoch": 3.7916666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.8125, + "learning_rate": 8.24869950187468e-09, + "loss": -1.4901161193847656e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1274, + "step_time": 6.675582034047693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 139.0, + "completions/mean_length": 101.875, + "completions/min_length": 81.0, + "epoch": 3.794642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.015333003598023e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1275, + "step_time": 5.8482154961675406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 247.0, + "completions/mean_length": 194.5, + "completions/min_length": 125.0, + "epoch": 3.7976190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.7852884964415e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1276, + "step_time": 6.716839849948883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 145.0, + "completions/mean_length": 121.25, + "completions/min_length": 100.0, + "epoch": 3.800595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.558567533717364e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1277, + "step_time": 5.8814581339247525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 163.0, + "completions/mean_length": 122.25, + "completions/min_length": 101.0, + "epoch": 3.803571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.335171646296434e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1278, + "step_time": 5.607497646007687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 243.0, + "completions/mean_length": 163.75, + "completions/min_length": 113.0, + "epoch": 3.806547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.115102342598101e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1279, + "step_time": 6.596497620921582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 295.75, + "completions/min_length": 161.0, + "epoch": 3.8095238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.898361108579831e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1280, + "step_time": 8.75270724389702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 512.0, + "completions/mean_length": 303.25, + "completions/min_length": 128.0, + "epoch": 3.8125, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.684949407727014e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1281, + "step_time": 8.622623350005597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 252.0, + "completions/mean_length": 196.0, + "completions/min_length": 129.0, + "epoch": 3.8154761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.75, + "learning_rate": 6.474868681043577e-09, + "loss": 3.725290298461914e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1282, + "step_time": 6.859939589165151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.0, + "completions/mean_length": 162.5, + "completions/min_length": 108.0, + "epoch": 3.818452380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.268120347041827e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1283, + "step_time": 7.45711486460641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 430.0, + "completions/mean_length": 214.75, + "completions/min_length": 107.0, + "epoch": 3.821428571428571, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.96875, + "learning_rate": 6.064705801732961e-09, + "loss": 2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1284, + "step_time": 8.88307345006615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 227.0, + "completions/mean_length": 141.75, + "completions/min_length": 106.0, + "epoch": 3.824404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.864626418617791e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1285, + "step_time": 6.467599679715931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 154.375, + "completions/min_length": 137.0, + "epoch": 3.8273809523809526, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.625, + "learning_rate": 5.66788354867731e-09, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1286, + "step_time": 6.0470578242093325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 162.0, + "completions/mean_length": 131.75, + "completions/min_length": 103.0, + "epoch": 3.830357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.474478520363535e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1287, + "step_time": 6.860718905925751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 345.0, + "completions/mean_length": 190.625, + "completions/min_length": 91.0, + "epoch": 3.8333333333333335, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.1875, + "learning_rate": 5.284412639590785e-09, + "loss": 0.0, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1288, + "step_time": 7.629284381866455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 222.0, + "completions/mean_length": 159.5, + "completions/min_length": 99.0, + "epoch": 3.8363095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.097687189726696e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1289, + "step_time": 7.8884455328807235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 168.0, + "completions/mean_length": 120.5, + "completions/min_length": 98.0, + "epoch": 3.8392857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 9.5, + "learning_rate": 4.914303431583389e-09, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1290, + "step_time": 7.937106019817293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 172.0, + "completions/mean_length": 143.875, + "completions/min_length": 116.0, + "epoch": 3.8422619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.7342626034093115e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1291, + "step_time": 6.02862943476066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 161.375, + "completions/min_length": 86.0, + "epoch": 3.8452380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 21.625, + "learning_rate": 4.55756592088058e-09, + "loss": 0.0, + "reward": 0.375, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.375, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1292, + "step_time": 6.199097063858062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 340.0, + "completions/mean_length": 192.125, + "completions/min_length": 143.0, + "epoch": 3.8482142857142856, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.375, + "learning_rate": 4.3842145770929265e-09, + "loss": -2.9802322387695312e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1293, + "step_time": 6.923170407302678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 148.0, + "completions/mean_length": 125.625, + "completions/min_length": 110.0, + "epoch": 3.8511904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.214209742553709e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1294, + "step_time": 5.491569495759904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 312.0, + "completions/mean_length": 177.625, + "completions/min_length": 119.0, + "epoch": 3.8541666666666665, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.71875, + "learning_rate": 4.0475525651737504e-09, + "loss": -2.9802322387695312e-08, + "reward": 0.25, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.25, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1295, + "step_time": 7.817410560790449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 342.0, + "completions/mean_length": 223.0, + "completions/min_length": 139.0, + "epoch": 3.857142857142857, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.75, + "learning_rate": 3.884244170259731e-09, + "loss": 1.4901161193847656e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1296, + "step_time": 7.008345567155629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 115.0, + "completions/mean_length": 94.0, + "completions/min_length": 78.0, + "epoch": 3.8601190476190474, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.8125, + "learning_rate": 3.7242856605066984e-09, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1297, + "step_time": 5.490034257993102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 251.0, + "completions/mean_length": 152.0, + "completions/min_length": 87.0, + "epoch": 3.863095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.5676781159905134e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1298, + "step_time": 6.245591544080526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 270.0, + "completions/mean_length": 201.375, + "completions/min_length": 133.0, + "epoch": 3.866071428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.4144225941605266e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1299, + "step_time": 6.806523882783949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 315.0, + "completions/mean_length": 152.375, + "completions/min_length": 95.0, + "epoch": 3.869047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.264520129832471e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1300, + "step_time": 8.173590070102364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 190.25, + "completions/min_length": 129.0, + "epoch": 3.8720238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.34375, + "learning_rate": 3.1179717351815237e-09, + "loss": -1.4901161193847656e-08, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1301, + "step_time": 7.056623946875334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 322.0, + "completions/mean_length": 210.375, + "completions/min_length": 117.0, + "epoch": 3.875, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.9747783997354224e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1302, + "step_time": 7.205672121141106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 232.0, + "completions/mean_length": 157.625, + "completions/min_length": 112.0, + "epoch": 3.8779761904761907, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.125, + "learning_rate": 2.8349410903677486e-09, + "loss": 2.9802322387695312e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1303, + "step_time": 6.498258818872273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 204.0, + "completions/mean_length": 151.625, + "completions/min_length": 116.0, + "epoch": 3.880952380952381, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.6984607512914316e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1304, + "step_time": 6.283308745827526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 112.0, + "completions/mean_length": 100.75, + "completions/min_length": 92.0, + "epoch": 3.883928571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.5653383040524224e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1305, + "step_time": 5.421855765860528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 311.0, + "completions/mean_length": 162.125, + "completions/min_length": 91.0, + "epoch": 3.886904761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.435574647523475e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1306, + "step_time": 6.729405515827239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 191.75, + "completions/min_length": 130.0, + "epoch": 3.8898809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.3091706578979854e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1307, + "step_time": 7.427042928058654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 229.0, + "completions/mean_length": 143.5, + "completions/min_length": 106.0, + "epoch": 3.892857142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.1861271886840506e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1308, + "step_time": 6.044458698946983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 153.125, + "completions/min_length": 95.0, + "epoch": 3.8958333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.0664450706988636e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1309, + "step_time": 6.0974430157803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 136.0, + "completions/mean_length": 125.375, + "completions/min_length": 106.0, + "epoch": 3.8988095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.9501251120631056e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1310, + "step_time": 5.677486295811832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 260.0, + "completions/mean_length": 196.625, + "completions/min_length": 134.0, + "epoch": 3.9017857142857144, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.8371680981951188e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1311, + "step_time": 6.593896828126162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 257.0, + "completions/mean_length": 162.625, + "completions/min_length": 109.0, + "epoch": 3.9047619047619047, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.7275747918062412e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1312, + "step_time": 6.737913504242897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 509.0, + "completions/mean_length": 216.75, + "completions/min_length": 129.0, + "epoch": 3.9077380952380953, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.6213459328950352e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1313, + "step_time": 8.99765222799033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 376.0, + "completions/mean_length": 174.0, + "completions/min_length": 114.0, + "epoch": 3.9107142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5184822387426244e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1314, + "step_time": 7.275036124046892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 486.0, + "completions/mean_length": 241.875, + "completions/min_length": 131.0, + "epoch": 3.9136904761904763, + "frac_reward_zero_std": 0.0, + "grad_norm": 4.3125, + "learning_rate": 1.4189844039078636e-09, + "loss": 0.0, + "reward": 0.125, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.125, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1315, + "step_time": 8.224617638159543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 133.0, + "completions/mean_length": 114.0, + "completions/min_length": 93.0, + "epoch": 3.9166666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.322853100222454e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1316, + "step_time": 5.86413128208369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 343.0, + "completions/mean_length": 176.125, + "completions/min_length": 86.0, + "epoch": 3.919642857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.2300889767866695e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1317, + "step_time": 7.404607364907861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 277.0, + "completions/mean_length": 235.875, + "completions/min_length": 180.0, + "epoch": 3.9226190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.1406926599646372e-09, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1318, + "step_time": 8.130943387746811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 178.0, + "completions/mean_length": 123.75, + "completions/min_length": 91.0, + "epoch": 3.925595238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.054664753380452e-09, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1319, + "step_time": 5.8562651528045535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 304.0, + "completions/mean_length": 189.5, + "completions/min_length": 107.0, + "epoch": 3.928571428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 9.720058379138473e-10, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1320, + "step_time": 7.193830838892609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 150.75, + "completions/min_length": 106.0, + "epoch": 3.931547619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.927164716964753e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1321, + "step_time": 6.223311827052385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 171.0, + "completions/mean_length": 144.0, + "completions/min_length": 112.0, + "epoch": 3.9345238095238093, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.167971901079096e-10, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1322, + "step_time": 5.783737162128091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 357.0, + "completions/mean_length": 189.125, + "completions/min_length": 136.0, + "epoch": 3.9375, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 7.442485057722048e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1323, + "step_time": 7.890357214957476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 149.0, + "completions/mean_length": 119.375, + "completions/min_length": 92.0, + "epoch": 3.9404761904761907, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.750709085544537e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1324, + "step_time": 5.96123132808134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 202.0, + "completions/mean_length": 159.875, + "completions/min_length": 126.0, + "epoch": 3.943452380952381, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.5625, + "learning_rate": 6.092648655572907e-10, + "loss": 4.470348358154297e-08, + "reward": 0.625, + "reward_std": 0.5175491571426392, + "rewards/DirectReward/mean": 0.625, + "rewards/DirectReward/std": 0.5175492167472839, + "step": 1325, + "step_time": 6.422630371060222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 129.0, + "completions/mean_length": 104.125, + "completions/min_length": 87.0, + "epoch": 3.946428571428571, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 5.468308211179496e-10, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1326, + "step_time": 5.730039000976831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 230.0, + "completions/mean_length": 184.0, + "completions/min_length": 146.0, + "epoch": 3.949404761904762, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.877691968051545e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1327, + "step_time": 6.1284917700104415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 298.0, + "completions/mean_length": 168.75, + "completions/min_length": 113.0, + "epoch": 3.9523809523809526, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.320803914162341e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1328, + "step_time": 6.688936653081328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 189.0, + "completions/mean_length": 149.5, + "completions/min_length": 111.0, + "epoch": 3.955357142857143, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.797647809745119e-10, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1329, + "step_time": 7.264287807047367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 272.0, + "completions/mean_length": 207.5, + "completions/min_length": 120.0, + "epoch": 3.9583333333333335, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 3.3082271872686416e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1330, + "step_time": 6.796444323845208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 164.0, + "completions/mean_length": 112.625, + "completions/min_length": 90.0, + "epoch": 3.9613095238095237, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.852545351409996e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1331, + "step_time": 5.588733697775751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 385.0, + "completions/mean_length": 217.125, + "completions/min_length": 152.0, + "epoch": 3.9642857142857144, + "frac_reward_zero_std": 0.0, + "grad_norm": 5.46875, + "learning_rate": 2.4306053790357217e-10, + "loss": 2.9802322387695312e-08, + "reward": 0.875, + "reward_std": 0.3535533845424652, + "rewards/DirectReward/mean": 0.875, + "rewards/DirectReward/std": 0.3535533845424652, + "step": 1332, + "step_time": 7.662103648297489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 180.0, + "completions/mean_length": 162.625, + "completions/min_length": 146.0, + "epoch": 3.9672619047619047, + "frac_reward_zero_std": 0.0, + "grad_norm": 7.46875, + "learning_rate": 2.0424101191790499e-10, + "loss": 2.9802322387695312e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1333, + "step_time": 6.132612264249474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 330.0, + "completions/mean_length": 175.375, + "completions/min_length": 101.0, + "epoch": 3.9702380952380953, + "frac_reward_zero_std": 0.0, + "grad_norm": 8.75, + "learning_rate": 1.6879621930226963e-10, + "loss": 4.470348358154297e-08, + "reward": 0.75, + "reward_std": 0.4629100561141968, + "rewards/DirectReward/mean": 0.75, + "rewards/DirectReward/std": 0.4629100561141968, + "step": 1334, + "step_time": 7.420854262076318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 305.0, + "completions/mean_length": 165.5, + "completions/min_length": 90.0, + "epoch": 3.9732142857142856, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.3672639938777653e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1335, + "step_time": 7.488191920798272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 341.0, + "completions/mean_length": 183.625, + "completions/min_length": 101.0, + "epoch": 3.9761904761904763, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.0803176871709841e-10, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1336, + "step_time": 6.925938067026436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 306.0, + "completions/mean_length": 179.0, + "completions/min_length": 107.0, + "epoch": 3.9791666666666665, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 8.27125210428603e-11, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1337, + "step_time": 6.62006901204586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 191.0, + "completions/mean_length": 139.375, + "completions/min_length": 106.0, + "epoch": 3.982142857142857, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.076882732625189e-11, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1338, + "step_time": 6.286676642950624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 185.0, + "completions/mean_length": 128.5, + "completions/min_length": 86.0, + "epoch": 3.9851190476190474, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 4.220083573608368e-11, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1339, + "step_time": 6.156113667879254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 285.0, + "completions/mean_length": 183.5, + "completions/min_length": 113.0, + "epoch": 3.988095238095238, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 2.700867164762144e-11, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1340, + "step_time": 8.354582559783012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 237.0, + "completions/mean_length": 157.25, + "completions/min_length": 96.0, + "epoch": 3.991071428571429, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 1.5192437641753376e-11, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1341, + "step_time": 7.569937977008522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 329.0, + "completions/mean_length": 196.125, + "completions/min_length": 107.0, + "epoch": 3.994047619047619, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 6.752213504268578e-12, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 0.0, + "rewards/DirectReward/std": 0.0, + "step": 1342, + "step_time": 6.869039259850979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 333.0, + "completions/mean_length": 196.75, + "completions/min_length": 119.0, + "epoch": 3.9970238095238093, + "frac_reward_zero_std": 0.0, + "grad_norm": 6.0625, + "learning_rate": 1.6880562255794372e-12, + "loss": 2.9802322387695312e-08, + "reward": 0.5, + "reward_std": 0.5345224738121033, + "rewards/DirectReward/mean": 0.5, + "rewards/DirectReward/std": 0.5345224738121033, + "step": 1343, + "step_time": 7.312326672952622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 258.0, + "completions/mean_length": 134.5, + "completions/min_length": 98.0, + "epoch": 4.0, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.0, + "learning_rate": 0.0, + "loss": 0.0, + "reward": 1.0, + "reward_std": 0.0, + "rewards/DirectReward/mean": 1.0, + "rewards/DirectReward/std": 0.0, + "step": 1344, + "step_time": 6.793088011909276 + } + ], + "logging_steps": 1, + "max_steps": 1344, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 168, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}