| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.00053, |
| "eval_steps": 500, |
| "global_step": 53, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 243.0, |
| "completions/max_terminated_length": 243.0, |
| "completions/mean_length": 10.46875, |
| "completions/mean_terminated_length": 10.46875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4529912807047367, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.004571504425257444, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.0017, |
| "num_tokens": 138384.0, |
| "reward": -5.380378246307373, |
| "reward_std": 12.681097030639648, |
| "rewards/rollout_reward_func/mean": -5.380378246307373, |
| "rewards/rollout_reward_func/std": 12.681096076965332, |
| "sampling/importance_sampling_ratio/max": 0.9966611266136169, |
| "sampling/importance_sampling_ratio/mean": 0.8889783024787903, |
| "sampling/importance_sampling_ratio/min": 0.32991692423820496, |
| "sampling/sampling_logp_difference/max": 0.6742424964904785, |
| "sampling/sampling_logp_difference/mean": 0.024776440113782883, |
| "step": 1, |
| "step_time": 81.77255414799993 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4529912807047367, |
| "epoch": 2e-05, |
| "grad_norm": 0.00460062688216567, |
| "kl": 0.0, |
| "learning_rate": 2.2857142857142855e-07, |
| "loss": 0.0017, |
| "step": 2, |
| "step_time": 15.840274740999803 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4952227398753166, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0019434797577559948, |
| "kl": 0.0006141539687405384, |
| "learning_rate": 4.571428571428571e-07, |
| "loss": 0.0, |
| "num_tokens": 287971.0, |
| "reward": -3.4194936752319336, |
| "reward_std": 9.754758834838867, |
| "rewards/rollout_reward_func/mean": -3.4194936752319336, |
| "rewards/rollout_reward_func/std": 9.754758834838867, |
| "sampling/importance_sampling_ratio/max": 1.0373786687850952, |
| "sampling/importance_sampling_ratio/mean": 0.9113750457763672, |
| "sampling/importance_sampling_ratio/min": 0.833645224571228, |
| "sampling/sampling_logp_difference/max": 0.21647244691848755, |
| "sampling/sampling_logp_difference/mean": 0.03790571913123131, |
| "step": 3, |
| "step_time": 83.32544364500018 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4965079165995121, |
| "epoch": 4e-05, |
| "grad_norm": 0.001877624075859785, |
| "kl": 0.000584022186821187, |
| "learning_rate": 6.857142857142857e-07, |
| "loss": 0.0, |
| "step": 4, |
| "step_time": 16.546222256999727 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5797880366444588, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00421608192846179, |
| "kl": 0.0006787570603137283, |
| "learning_rate": 9.142857142857142e-07, |
| "loss": -0.0, |
| "num_tokens": 438675.0, |
| "reward": -0.2843954265117645, |
| "reward_std": 11.412687301635742, |
| "rewards/rollout_reward_func/mean": -0.2843954265117645, |
| "rewards/rollout_reward_func/std": 11.412688255310059, |
| "sampling/importance_sampling_ratio/max": 1.057585597038269, |
| "sampling/importance_sampling_ratio/mean": 0.8996330499649048, |
| "sampling/importance_sampling_ratio/min": 0.8375532031059265, |
| "sampling/sampling_logp_difference/max": 0.18906748294830322, |
| "sampling/sampling_logp_difference/mean": 0.041610121726989746, |
| "step": 5, |
| "step_time": 82.4154398549997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5783220157027245, |
| "epoch": 6e-05, |
| "grad_norm": 0.003988980781286955, |
| "kl": 0.0004869329411576473, |
| "learning_rate": 1.1428571428571428e-06, |
| "loss": -0.0, |
| "step": 6, |
| "step_time": 16.066873880999992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5536964721977711, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0016283580334857106, |
| "kl": 0.00039194995474467476, |
| "learning_rate": 1.3714285714285715e-06, |
| "loss": 0.0, |
| "num_tokens": 584635.0, |
| "reward": -7.030133247375488, |
| "reward_std": 13.085518836975098, |
| "rewards/rollout_reward_func/mean": -7.030133247375488, |
| "rewards/rollout_reward_func/std": 13.085518836975098, |
| "sampling/importance_sampling_ratio/max": 0.9451111555099487, |
| "sampling/importance_sampling_ratio/mean": 0.8901357650756836, |
| "sampling/importance_sampling_ratio/min": 0.6380032896995544, |
| "sampling/sampling_logp_difference/max": 0.3521350622177124, |
| "sampling/sampling_logp_difference/mean": 0.042222920805215836, |
| "step": 7, |
| "step_time": 80.50391445700006 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5454729907214642, |
| "epoch": 8e-05, |
| "grad_norm": 0.0024409866891801357, |
| "kl": 0.00014830356569106584, |
| "learning_rate": 1.6e-06, |
| "loss": 0.0, |
| "step": 8, |
| "step_time": 16.513566811999908 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.47564223781228065, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00832037627696991, |
| "kl": 0.0007051239214987959, |
| "learning_rate": 1.8285714285714284e-06, |
| "loss": 0.0, |
| "num_tokens": 733233.0, |
| "reward": -8.504392623901367, |
| "reward_std": 10.406264305114746, |
| "rewards/rollout_reward_func/mean": -8.504392623901367, |
| "rewards/rollout_reward_func/std": 10.406264305114746, |
| "sampling/importance_sampling_ratio/max": 1.115485668182373, |
| "sampling/importance_sampling_ratio/mean": 0.9117836952209473, |
| "sampling/importance_sampling_ratio/min": 0.8292141556739807, |
| "sampling/sampling_logp_difference/max": 0.2870664596557617, |
| "sampling/sampling_logp_difference/mean": 0.03817388415336609, |
| "step": 9, |
| "step_time": 81.75653769800033 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.48243317380547523, |
| "epoch": 0.0001, |
| "grad_norm": 0.008465186692774296, |
| "kl": 0.0010313481093007226, |
| "learning_rate": 2.057142857142857e-06, |
| "loss": 0.0, |
| "step": 10, |
| "step_time": 16.598842530999946 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.5141099877655506, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0013674289220944047, |
| "kl": 8.758902610139785e-05, |
| "learning_rate": 2.2857142857142856e-06, |
| "loss": 0.0, |
| "num_tokens": 885401.0, |
| "reward": -0.5494611263275146, |
| "reward_std": 10.671172142028809, |
| "rewards/rollout_reward_func/mean": -0.5494611263275146, |
| "rewards/rollout_reward_func/std": 10.671172142028809, |
| "sampling/importance_sampling_ratio/max": 0.9451060891151428, |
| "sampling/importance_sampling_ratio/mean": 0.8876197338104248, |
| "sampling/importance_sampling_ratio/min": 0.6805641651153564, |
| "sampling/sampling_logp_difference/max": 0.2943337857723236, |
| "sampling/sampling_logp_difference/mean": 0.04072599858045578, |
| "step": 11, |
| "step_time": 80.9672206400005 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5212433487176895, |
| "epoch": 0.00012, |
| "grad_norm": 0.00168048485647887, |
| "kl": 0.00033407722384026783, |
| "learning_rate": 2.5142857142857142e-06, |
| "loss": 0.0, |
| "step": 12, |
| "step_time": 16.082450073999098 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.46110106259584427, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0006600260967388749, |
| "kl": 0.00035950390563144197, |
| "learning_rate": 2.742857142857143e-06, |
| "loss": -0.0, |
| "num_tokens": 1028599.0, |
| "reward": -3.1979265213012695, |
| "reward_std": 14.811416625976562, |
| "rewards/rollout_reward_func/mean": -3.1979265213012695, |
| "rewards/rollout_reward_func/std": 14.811417579650879, |
| "sampling/importance_sampling_ratio/max": 0.9470726251602173, |
| "sampling/importance_sampling_ratio/mean": 0.9061694145202637, |
| "sampling/importance_sampling_ratio/min": 0.8211163282394409, |
| "sampling/sampling_logp_difference/max": 0.14348044991493225, |
| "sampling/sampling_logp_difference/mean": 0.03530801087617874, |
| "step": 13, |
| "step_time": 80.93144869100024 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.4607865922152996, |
| "epoch": 0.00014, |
| "grad_norm": 0.0008775219903327525, |
| "kl": 9.266982237932098e-05, |
| "learning_rate": 2.9714285714285716e-06, |
| "loss": -0.0, |
| "step": 14, |
| "step_time": 16.417307382000217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.015625, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.84375, |
| "completions/mean_terminated_length": 2.84375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.6254608854651451, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0023997484240680933, |
| "kl": 0.002952657545392867, |
| "learning_rate": 3.2e-06, |
| "loss": 0.0, |
| "num_tokens": 1181364.0, |
| "reward": -5.002313137054443, |
| "reward_std": 11.715435981750488, |
| "rewards/rollout_reward_func/mean": -5.002313137054443, |
| "rewards/rollout_reward_func/std": 11.715435981750488, |
| "sampling/importance_sampling_ratio/max": 1.3015769720077515, |
| "sampling/importance_sampling_ratio/mean": 0.9032159447669983, |
| "sampling/importance_sampling_ratio/min": 0.6611238121986389, |
| "sampling/sampling_logp_difference/max": 0.412276029586792, |
| "sampling/sampling_logp_difference/mean": 0.05298498272895813, |
| "step": 15, |
| "step_time": 86.0248277200003 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.6279397383332253, |
| "epoch": 0.00016, |
| "grad_norm": 0.003235182724893093, |
| "kl": 0.00362776829857836, |
| "learning_rate": 3.428571428571428e-06, |
| "loss": 0.0, |
| "step": 16, |
| "step_time": 16.061590983999395 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5620961934328079, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.001932546147145331, |
| "kl": 0.0007415264718311221, |
| "learning_rate": 3.657142857142857e-06, |
| "loss": -0.0, |
| "num_tokens": 1330148.0, |
| "reward": -3.097731113433838, |
| "reward_std": 13.016914367675781, |
| "rewards/rollout_reward_func/mean": -3.097731113433838, |
| "rewards/rollout_reward_func/std": 13.016914367675781, |
| "sampling/importance_sampling_ratio/max": 1.3107270002365112, |
| "sampling/importance_sampling_ratio/mean": 0.8959370255470276, |
| "sampling/importance_sampling_ratio/min": 0.7702041268348694, |
| "sampling/sampling_logp_difference/max": 0.38533973693847656, |
| "sampling/sampling_logp_difference/mean": 0.04856353998184204, |
| "step": 17, |
| "step_time": 84.13615993699977 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02083333395421505, |
| "entropy": 0.5507431365549564, |
| "epoch": 0.00018, |
| "grad_norm": 0.0014166560722514987, |
| "kl": 0.00446120251346116, |
| "learning_rate": 3.885714285714286e-06, |
| "loss": -0.0, |
| "step": 18, |
| "step_time": 16.494719347999762 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5130873061716557, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.002229799050837755, |
| "kl": 0.00022485230880420204, |
| "learning_rate": 4.114285714285714e-06, |
| "loss": 0.0, |
| "num_tokens": 1479520.0, |
| "reward": -3.4886345863342285, |
| "reward_std": 15.442611694335938, |
| "rewards/rollout_reward_func/mean": -3.4886345863342285, |
| "rewards/rollout_reward_func/std": 15.442611694335938, |
| "sampling/importance_sampling_ratio/max": 0.9724913835525513, |
| "sampling/importance_sampling_ratio/mean": 0.897862434387207, |
| "sampling/importance_sampling_ratio/min": 0.7611527442932129, |
| "sampling/sampling_logp_difference/max": 0.1756596565246582, |
| "sampling/sampling_logp_difference/mean": 0.03822677582502365, |
| "step": 19, |
| "step_time": 83.84734981699967 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5135630927979946, |
| "epoch": 0.0002, |
| "grad_norm": 0.0020390080753713846, |
| "kl": 0.00010164715294536109, |
| "learning_rate": 4.342857142857142e-06, |
| "loss": 0.0, |
| "step": 20, |
| "step_time": 16.066128179999396 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.5136846676468849, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0008278603199869394, |
| "kl": 0.0005749665239136448, |
| "learning_rate": 4.571428571428571e-06, |
| "loss": -0.0, |
| "num_tokens": 1625572.0, |
| "reward": 0.6307157278060913, |
| "reward_std": 12.084759712219238, |
| "rewards/rollout_reward_func/mean": 0.6307157278060913, |
| "rewards/rollout_reward_func/std": 12.084757804870605, |
| "sampling/importance_sampling_ratio/max": 0.9621063470840454, |
| "sampling/importance_sampling_ratio/mean": 0.9004920721054077, |
| "sampling/importance_sampling_ratio/min": 0.8443179726600647, |
| "sampling/sampling_logp_difference/max": 0.15061461925506592, |
| "sampling/sampling_logp_difference/mean": 0.03823810815811157, |
| "step": 21, |
| "step_time": 83.75205460899997 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.5070930048823357, |
| "epoch": 0.00022, |
| "grad_norm": 0.0008373188320547342, |
| "kl": 0.00025310701903435984, |
| "learning_rate": 4.8e-06, |
| "loss": -0.0, |
| "step": 22, |
| "step_time": 15.982322978999036 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.473282802850008, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0020822216756641865, |
| "kl": 0.00038502290840369824, |
| "learning_rate": 5.0285714285714285e-06, |
| "loss": -0.0, |
| "num_tokens": 1780247.0, |
| "reward": -1.9041584730148315, |
| "reward_std": 13.668906211853027, |
| "rewards/rollout_reward_func/mean": -1.9041584730148315, |
| "rewards/rollout_reward_func/std": 13.668906211853027, |
| "sampling/importance_sampling_ratio/max": 0.9427170753479004, |
| "sampling/importance_sampling_ratio/mean": 0.9054326415061951, |
| "sampling/importance_sampling_ratio/min": 0.7578762769699097, |
| "sampling/sampling_logp_difference/max": 0.18389169871807098, |
| "sampling/sampling_logp_difference/mean": 0.03585001453757286, |
| "step": 23, |
| "step_time": 90.08848715500017 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4743204452097416, |
| "epoch": 0.00024, |
| "grad_norm": 0.0022170657757669687, |
| "kl": 7.387449461759843e-05, |
| "learning_rate": 5.257142857142857e-06, |
| "loss": 0.0, |
| "step": 24, |
| "step_time": 15.96505278000086 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4709370397031307, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0005548119661398232, |
| "kl": 0.0005451691670259606, |
| "learning_rate": 5.485714285714286e-06, |
| "loss": -0.0, |
| "num_tokens": 1935185.0, |
| "reward": -3.296872854232788, |
| "reward_std": 10.132695198059082, |
| "rewards/rollout_reward_func/mean": -3.296872854232788, |
| "rewards/rollout_reward_func/std": 10.132695198059082, |
| "sampling/importance_sampling_ratio/max": 1.0497726202011108, |
| "sampling/importance_sampling_ratio/mean": 0.9049290418624878, |
| "sampling/importance_sampling_ratio/min": 0.48642516136169434, |
| "sampling/sampling_logp_difference/max": 0.6218838691711426, |
| "sampling/sampling_logp_difference/mean": 0.041815631091594696, |
| "step": 25, |
| "step_time": 90.80210218299999 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.46433135867118835, |
| "epoch": 0.00026, |
| "grad_norm": 0.001882780808955431, |
| "kl": 0.01631506085379897, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0, |
| "step": 26, |
| "step_time": 16.107132911000917 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4543347842991352, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0012202052166685462, |
| "kl": 0.002381903061632329, |
| "learning_rate": 5.942857142857143e-06, |
| "loss": 0.0, |
| "num_tokens": 2086059.0, |
| "reward": -4.066007614135742, |
| "reward_std": 9.549392700195312, |
| "rewards/rollout_reward_func/mean": -4.066007614135742, |
| "rewards/rollout_reward_func/std": 9.549392700195312, |
| "sampling/importance_sampling_ratio/max": 0.9443690180778503, |
| "sampling/importance_sampling_ratio/mean": 0.9025586843490601, |
| "sampling/importance_sampling_ratio/min": 0.6863695979118347, |
| "sampling/sampling_logp_difference/max": 0.2400798797607422, |
| "sampling/sampling_logp_difference/mean": 0.03597673028707504, |
| "step": 27, |
| "step_time": 92.19677682200108 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.4513773135840893, |
| "epoch": 0.00028, |
| "grad_norm": 0.002780377632007003, |
| "kl": 0.0004442356521394686, |
| "learning_rate": 6.171428571428571e-06, |
| "loss": 0.0, |
| "step": 28, |
| "step_time": 16.044156099000247 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.84375, |
| "completions/mean_terminated_length": 2.84375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4494730308651924, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0006526823271997273, |
| "kl": 0.00010942593371510156, |
| "learning_rate": 6.4e-06, |
| "loss": 0.0, |
| "num_tokens": 2241739.0, |
| "reward": -8.519765853881836, |
| "reward_std": 11.446259498596191, |
| "rewards/rollout_reward_func/mean": -8.519765853881836, |
| "rewards/rollout_reward_func/std": 11.446259498596191, |
| "sampling/importance_sampling_ratio/max": 0.9423424005508423, |
| "sampling/importance_sampling_ratio/mean": 0.9169149994850159, |
| "sampling/importance_sampling_ratio/min": 0.8564605116844177, |
| "sampling/sampling_logp_difference/max": 0.13965635001659393, |
| "sampling/sampling_logp_difference/mean": 0.031654730439186096, |
| "step": 29, |
| "step_time": 92.4542784769992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.44314464554190636, |
| "epoch": 0.0003, |
| "grad_norm": 0.0007136130589060485, |
| "kl": 0.00014493030903395265, |
| "learning_rate": 6.628571428571428e-06, |
| "loss": 0.0, |
| "step": 30, |
| "step_time": 16.182491193999795 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.875, |
| "completions/mean_terminated_length": 2.875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4988393858075142, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.007201060187071562, |
| "kl": 0.0007445381734214607, |
| "learning_rate": 6.857142857142856e-06, |
| "loss": 0.0001, |
| "num_tokens": 2384946.0, |
| "reward": -4.137698173522949, |
| "reward_std": 11.706729888916016, |
| "rewards/rollout_reward_func/mean": -4.137698173522949, |
| "rewards/rollout_reward_func/std": 11.706729888916016, |
| "sampling/importance_sampling_ratio/max": 1.0587443113327026, |
| "sampling/importance_sampling_ratio/mean": 0.9121928811073303, |
| "sampling/importance_sampling_ratio/min": 0.7692723870277405, |
| "sampling/sampling_logp_difference/max": 0.1996639221906662, |
| "sampling/sampling_logp_difference/mean": 0.03824968636035919, |
| "step": 31, |
| "step_time": 87.52751164399933 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.012500000186264515, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02291666716337204, |
| "entropy": 0.4917365238070488, |
| "epoch": 0.00032, |
| "grad_norm": 0.002689385786652565, |
| "kl": 0.0006912436326729221, |
| "learning_rate": 7.085714285714285e-06, |
| "loss": 0.0, |
| "step": 32, |
| "step_time": 15.88975830299978 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.42307131364941597, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0014049713499844074, |
| "kl": 0.00045210494954517344, |
| "learning_rate": 7.314285714285714e-06, |
| "loss": 0.0, |
| "num_tokens": 2534295.0, |
| "reward": -1.3761987686157227, |
| "reward_std": 12.530879974365234, |
| "rewards/rollout_reward_func/mean": -1.3761987686157227, |
| "rewards/rollout_reward_func/std": 12.530879020690918, |
| "sampling/importance_sampling_ratio/max": 1.0053565502166748, |
| "sampling/importance_sampling_ratio/mean": 0.9225521683692932, |
| "sampling/importance_sampling_ratio/min": 0.8554979562759399, |
| "sampling/sampling_logp_difference/max": 0.15607085824012756, |
| "sampling/sampling_logp_difference/mean": 0.03046201914548874, |
| "step": 33, |
| "step_time": 86.13867495900058 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.4185350500047207, |
| "epoch": 0.00034, |
| "grad_norm": 0.001140501000918448, |
| "kl": 0.00017268433839490172, |
| "learning_rate": 7.542857142857142e-06, |
| "loss": 0.0, |
| "step": 34, |
| "step_time": 15.995409347000532 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.39532871544361115, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.009359397925436497, |
| "kl": 0.0016290315243168152, |
| "learning_rate": 7.771428571428572e-06, |
| "loss": -0.0, |
| "num_tokens": 2688362.0, |
| "reward": -4.614232540130615, |
| "reward_std": 10.228343963623047, |
| "rewards/rollout_reward_func/mean": -4.614232540130615, |
| "rewards/rollout_reward_func/std": 10.228343963623047, |
| "sampling/importance_sampling_ratio/max": 1.5226150751113892, |
| "sampling/importance_sampling_ratio/mean": 0.9340947866439819, |
| "sampling/importance_sampling_ratio/min": 0.7190265655517578, |
| "sampling/sampling_logp_difference/max": 0.4914824962615967, |
| "sampling/sampling_logp_difference/mean": 0.03537113964557648, |
| "step": 35, |
| "step_time": 93.26275390599994 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.3946211449801922, |
| "epoch": 0.00036, |
| "grad_norm": 0.008305351249873638, |
| "kl": 0.00206446714582853, |
| "learning_rate": 8e-06, |
| "loss": -0.0, |
| "step": 36, |
| "step_time": 16.673777301999507 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4220520183444023, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.002266058698296547, |
| "kl": 0.0012106694839530974, |
| "learning_rate": 7.999999999958871e-06, |
| "loss": -0.0, |
| "num_tokens": 2838851.0, |
| "reward": -5.611568450927734, |
| "reward_std": 12.478642463684082, |
| "rewards/rollout_reward_func/mean": -5.611568450927734, |
| "rewards/rollout_reward_func/std": 12.478641510009766, |
| "sampling/importance_sampling_ratio/max": 1.103140950202942, |
| "sampling/importance_sampling_ratio/mean": 0.929234504699707, |
| "sampling/importance_sampling_ratio/min": 0.7560043334960938, |
| "sampling/sampling_logp_difference/max": 0.3681096136569977, |
| "sampling/sampling_logp_difference/mean": 0.03542977571487427, |
| "step": 37, |
| "step_time": 94.46365324899944 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.4241916239261627, |
| "epoch": 0.00038, |
| "grad_norm": 0.001786528853699565, |
| "kl": 0.0011816255682788324, |
| "learning_rate": 7.999999999835487e-06, |
| "loss": -0.0, |
| "step": 38, |
| "step_time": 16.13328046199922 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4868612252175808, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.004135461524128914, |
| "kl": 0.0050761070551743614, |
| "learning_rate": 7.999999999629846e-06, |
| "loss": -0.0, |
| "num_tokens": 2990804.0, |
| "reward": -6.989686965942383, |
| "reward_std": 12.253325462341309, |
| "rewards/rollout_reward_func/mean": -6.989686965942383, |
| "rewards/rollout_reward_func/std": 12.253325462341309, |
| "sampling/importance_sampling_ratio/max": 1.0075410604476929, |
| "sampling/importance_sampling_ratio/mean": 0.9061470031738281, |
| "sampling/importance_sampling_ratio/min": 0.2844204604625702, |
| "sampling/sampling_logp_difference/max": 1.051173210144043, |
| "sampling/sampling_logp_difference/mean": 0.04147646576166153, |
| "step": 39, |
| "step_time": 96.24385869400021 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.010416666977107525, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02083333395421505, |
| "entropy": 0.4855256490409374, |
| "epoch": 0.0004, |
| "grad_norm": 0.0031671926844865084, |
| "kl": 0.004727449944766704, |
| "learning_rate": 7.99999999934195e-06, |
| "loss": -0.0, |
| "step": 40, |
| "step_time": 16.856322315999932 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.3505343496799469, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.004083838313817978, |
| "kl": 0.0013105008611091762, |
| "learning_rate": 7.999999998971795e-06, |
| "loss": 0.0, |
| "num_tokens": 3145365.0, |
| "reward": -3.9754014015197754, |
| "reward_std": 10.032164573669434, |
| "rewards/rollout_reward_func/mean": -3.9754014015197754, |
| "rewards/rollout_reward_func/std": 10.032164573669434, |
| "sampling/importance_sampling_ratio/max": 0.9533951282501221, |
| "sampling/importance_sampling_ratio/mean": 0.9258453249931335, |
| "sampling/importance_sampling_ratio/min": 0.7874615788459778, |
| "sampling/sampling_logp_difference/max": 0.17639052867889404, |
| "sampling/sampling_logp_difference/mean": 0.026364009827375412, |
| "step": 41, |
| "step_time": 96.03926057699982 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.34893185272812843, |
| "epoch": 0.00042, |
| "grad_norm": 0.0008124930318444967, |
| "kl": 0.000726719064914505, |
| "learning_rate": 7.999999998519386e-06, |
| "loss": -0.0, |
| "step": 42, |
| "step_time": 15.88754204000088 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 5.0, |
| "completions/max_terminated_length": 5.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.41375133767724037, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0036737218033522367, |
| "kl": 0.004247826599566906, |
| "learning_rate": 7.99999999798472e-06, |
| "loss": 0.0, |
| "num_tokens": 3301615.0, |
| "reward": -3.794240951538086, |
| "reward_std": 12.325447082519531, |
| "rewards/rollout_reward_func/mean": -3.794240951538086, |
| "rewards/rollout_reward_func/std": 12.325447082519531, |
| "sampling/importance_sampling_ratio/max": 0.9757733345031738, |
| "sampling/importance_sampling_ratio/mean": 0.9054876565933228, |
| "sampling/importance_sampling_ratio/min": 0.39631277322769165, |
| "sampling/sampling_logp_difference/max": 0.7294319272041321, |
| "sampling/sampling_logp_difference/mean": 0.037186965346336365, |
| "step": 43, |
| "step_time": 101.71214155000143 |
| }, |
| { |
| "clip_ratio/high_max": 0.02500000037252903, |
| "clip_ratio/high_mean": 0.012500000186264515, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.012500000186264515, |
| "entropy": 0.4078464061021805, |
| "epoch": 0.00044, |
| "grad_norm": 0.0016561471857130527, |
| "kl": 0.005151898421900114, |
| "learning_rate": 7.999999997367799e-06, |
| "loss": 0.0, |
| "step": 44, |
| "step_time": 16.91709048499979 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 3.0, |
| "completions/mean_terminated_length": 3.0, |
| "completions/min_length": 3.0, |
| "completions/min_terminated_length": 3.0, |
| "entropy": 0.35123512521386147, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.002406093757599592, |
| "kl": 0.0009607337087800261, |
| "learning_rate": 7.999999996668619e-06, |
| "loss": 0.0, |
| "num_tokens": 3452891.0, |
| "reward": -2.987896203994751, |
| "reward_std": 12.457817077636719, |
| "rewards/rollout_reward_func/mean": -2.987896203994751, |
| "rewards/rollout_reward_func/std": 12.457817077636719, |
| "sampling/importance_sampling_ratio/max": 0.9862170219421387, |
| "sampling/importance_sampling_ratio/mean": 0.9278237819671631, |
| "sampling/importance_sampling_ratio/min": 0.7392672300338745, |
| "sampling/sampling_logp_difference/max": 0.24572673439979553, |
| "sampling/sampling_logp_difference/mean": 0.0269648227840662, |
| "step": 45, |
| "step_time": 92.76488730100027 |
| }, |
| { |
| "clip_ratio/high_max": 0.02083333395421505, |
| "clip_ratio/high_mean": 0.010416666977107525, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.010416666977107525, |
| "entropy": 0.3466106317937374, |
| "epoch": 0.00046, |
| "grad_norm": 0.0016426184447482228, |
| "kl": 0.0028608875945792533, |
| "learning_rate": 7.999999995887185e-06, |
| "loss": 0.0, |
| "step": 46, |
| "step_time": 15.851301255999715 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.96875, |
| "completions/mean_terminated_length": 2.96875, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.35224514454603195, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00269598257727921, |
| "kl": 0.0013323712864803383, |
| "learning_rate": 7.999999995023493e-06, |
| "loss": -0.0, |
| "num_tokens": 3604685.0, |
| "reward": -0.2975272238254547, |
| "reward_std": 14.587063789367676, |
| "rewards/rollout_reward_func/mean": -0.2975272238254547, |
| "rewards/rollout_reward_func/std": 14.587064743041992, |
| "sampling/importance_sampling_ratio/max": 1.1633646488189697, |
| "sampling/importance_sampling_ratio/mean": 0.9449476599693298, |
| "sampling/importance_sampling_ratio/min": 0.8831871747970581, |
| "sampling/sampling_logp_difference/max": 0.24419668316841125, |
| "sampling/sampling_logp_difference/mean": 0.02657265216112137, |
| "step": 47, |
| "step_time": 90.8274718849998 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.35505227744579315, |
| "epoch": 0.00048, |
| "grad_norm": 0.0026499112136662006, |
| "kl": 0.0012647637449845206, |
| "learning_rate": 7.999999994077545e-06, |
| "loss": -0.0, |
| "step": 48, |
| "step_time": 16.71245220000037 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.35509930923581123, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0029402568470686674, |
| "kl": 0.003937347410101211, |
| "learning_rate": 7.999999993049342e-06, |
| "loss": 0.0, |
| "num_tokens": 3748779.0, |
| "reward": -4.499238967895508, |
| "reward_std": 12.109779357910156, |
| "rewards/rollout_reward_func/mean": -4.499238967895508, |
| "rewards/rollout_reward_func/std": 12.10977840423584, |
| "sampling/importance_sampling_ratio/max": 1.2103843688964844, |
| "sampling/importance_sampling_ratio/mean": 0.9383784532546997, |
| "sampling/importance_sampling_ratio/min": 0.6798091530799866, |
| "sampling/sampling_logp_difference/max": 0.3193366229534149, |
| "sampling/sampling_logp_difference/mean": 0.031250130385160446, |
| "step": 49, |
| "step_time": 88.47513171899982 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.3556080162525177, |
| "epoch": 0.0005, |
| "grad_norm": 0.003199104219675064, |
| "kl": 0.0025984440217143856, |
| "learning_rate": 7.999999991938882e-06, |
| "loss": 0.0, |
| "step": 50, |
| "step_time": 16.247764421000284 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.90625, |
| "completions/mean_terminated_length": 2.90625, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.4164017327129841, |
| "epoch": 0.00051, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0021409487817436457, |
| "kl": 0.006027625652677671, |
| "learning_rate": 7.999999990746166e-06, |
| "loss": 0.0, |
| "num_tokens": 3894529.0, |
| "reward": -1.7900453805923462, |
| "reward_std": 14.370148658752441, |
| "rewards/rollout_reward_func/mean": -1.7900453805923462, |
| "rewards/rollout_reward_func/std": 14.370148658752441, |
| "sampling/importance_sampling_ratio/max": 1.1930700540542603, |
| "sampling/importance_sampling_ratio/mean": 0.9282512664794922, |
| "sampling/importance_sampling_ratio/min": 0.5870267152786255, |
| "sampling/sampling_logp_difference/max": 0.5323870182037354, |
| "sampling/sampling_logp_difference/mean": 0.031058136373758316, |
| "step": 51, |
| "step_time": 86.91906849599945 |
| }, |
| { |
| "clip_ratio/high_max": 0.03125, |
| "clip_ratio/high_mean": 0.015625, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.015625, |
| "entropy": 0.4149097427725792, |
| "epoch": 0.00052, |
| "grad_norm": 0.0021878276020288467, |
| "kl": 0.006098713455685356, |
| "learning_rate": 7.999999989471194e-06, |
| "loss": 0.0, |
| "step": 52, |
| "step_time": 16.74750956799926 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 3.0, |
| "completions/max_terminated_length": 3.0, |
| "completions/mean_length": 2.9375, |
| "completions/mean_terminated_length": 2.9375, |
| "completions/min_length": 2.0, |
| "completions/min_terminated_length": 2.0, |
| "entropy": 0.353309016674757, |
| "epoch": 0.00053, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.002742928918451071, |
| "kl": 0.0019847122130158823, |
| "learning_rate": 7.999999988113964e-06, |
| "loss": -0.0, |
| "num_tokens": 4050163.0, |
| "reward": -5.597317218780518, |
| "reward_std": 9.173243522644043, |
| "rewards/rollout_reward_func/mean": -5.597317218780518, |
| "rewards/rollout_reward_func/std": 9.173242568969727, |
| "sampling/importance_sampling_ratio/max": 1.0058647394180298, |
| "sampling/importance_sampling_ratio/mean": 0.9295854568481445, |
| "sampling/importance_sampling_ratio/min": 0.7418098449707031, |
| "sampling/sampling_logp_difference/max": 0.21907085180282593, |
| "sampling/sampling_logp_difference/mean": 0.02789393812417984, |
| "step": 53, |
| "step_time": 95.40283962799822 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 600000, |
| "num_input_tokens_seen": 4050163, |
| "num_train_epochs": 6, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
