diff --git "a/trainer_state.json" "b/trainer_state.json" --- "a/trainer_state.json" +++ "b/trainer_state.json" @@ -16,31 +16,31 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1178.0, - "completions/max_terminated_length": 1178.0, - "completions/mean_length": 280.09375, - "completions/mean_terminated_length": 280.09375, + "completions/max_length": 813.0, + "completions/max_terminated_length": 813.0, + "completions/mean_length": 309.25, + "completions/mean_terminated_length": 309.25, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.3659224039874971, + "entropy": 0.3283487097360194, "epoch": 1e-05, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8075896501541138, + "grad_norm": 1.5004478693008423, "kl": 0.0, "learning_rate": 0.0, - "loss": -0.0378, - "num_tokens": 37691.0, - "reward": -1.0071403980255127, - "reward_std": 0.036310724914073944, - "rewards/rollout_reward_func/mean": -1.0071403980255127, - "rewards/rollout_reward_func/std": 0.0389554463326931, - "sampling/importance_sampling_ratio/max": 1.6058107614517212, - "sampling/importance_sampling_ratio/mean": 1.0605039596557617, - "sampling/importance_sampling_ratio/min": 0.6887973546981812, - "sampling/sampling_logp_difference/max": 0.4979521632194519, - "sampling/sampling_logp_difference/mean": 0.04375916346907616, + "loss": -0.0177, + "num_tokens": 38624.0, + "reward": -0.17298494279384613, + "reward_std": 1.0476001501083374, + "rewards/rollout_reward_func/mean": -0.17298494279384613, + "rewards/rollout_reward_func/std": 1.0311574935913086, + "sampling/importance_sampling_ratio/max": 2.268721342086792, + "sampling/importance_sampling_ratio/mean": 0.9687087535858154, + "sampling/importance_sampling_ratio/min": 0.44848719239234924, + "sampling/sampling_logp_difference/max": 0.8390021324157715, + "sampling/sampling_logp_difference/mean": 0.0607302263379097, "step": 1, - "step_time": 11.142912297999601 + "step_time": 9.972317869000562 }, { "clip_ratio/high_max": 0.0, @@ -48,1022 +48,1022 @@ "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, - "entropy": 0.3659224039874971, + "entropy": 0.3283487097360194, "epoch": 2e-05, - "grad_norm": 0.8133119344711304, + "grad_norm": 1.51764714717865, "kl": 0.0, "learning_rate": 2.8571428571428573e-06, - "loss": -0.0378, + "loss": -0.0177, "step": 2, - "step_time": 6.929950346999931 + "step_time": 6.341064500000357 }, { - "clip_ratio/high_max": 0.05902777845039964, - "clip_ratio/high_mean": 0.02951388922519982, - "clip_ratio/low_mean": 0.031250000931322575, - "clip_ratio/low_min": 0.02083333395421505, - "clip_ratio/region_mean": 0.06076389015652239, + "clip_ratio/high_max": 0.010416666977107525, + "clip_ratio/high_mean": 0.0052083334885537624, + "clip_ratio/low_mean": 0.02682291716337204, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0320312506519258, "completions/clipped_ratio": 0.0, - "completions/max_length": 1232.0, - "completions/max_terminated_length": 1232.0, - "completions/mean_length": 310.90625, - "completions/mean_terminated_length": 310.90625, + "completions/max_length": 800.0, + "completions/max_terminated_length": 800.0, + "completions/mean_length": 344.1875, + "completions/mean_terminated_length": 344.1875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.4082131152972579, + "entropy": 0.41025719512254, "epoch": 3e-05, "frac_reward_zero_std": 0.0, - "grad_norm": 0.716623067855835, - "kl": 0.0072153940433281605, + "grad_norm": 0.8690568804740906, + "kl": 0.011901523646884016, "learning_rate": 5.7142857142857145e-06, - "loss": -0.0034, - "num_tokens": 79271.0, - "reward": -0.9292834997177124, - "reward_std": 0.20812131464481354, - "rewards/rollout_reward_func/mean": -0.9292834997177124, - "rewards/rollout_reward_func/std": 0.3781917691230774, - "sampling/importance_sampling_ratio/max": 1.392927646636963, - "sampling/importance_sampling_ratio/mean": 1.0064890384674072, - "sampling/importance_sampling_ratio/min": 0.6378100514411926, - "sampling/sampling_logp_difference/max": 0.53961181640625, - "sampling/sampling_logp_difference/mean": 0.039233192801475525, + "loss": -0.0118, + "num_tokens": 81269.0, + "reward": -0.5246932506561279, + "reward_std": 0.7359878420829773, + "rewards/rollout_reward_func/mean": -0.5246932506561279, + "rewards/rollout_reward_func/std": 0.8779017925262451, + "sampling/importance_sampling_ratio/max": 1.7051993608474731, + "sampling/importance_sampling_ratio/mean": 0.9748166799545288, + "sampling/importance_sampling_ratio/min": 0.3005872368812561, + "sampling/sampling_logp_difference/max": 1.0160791873931885, + "sampling/sampling_logp_difference/mean": 0.060047175735235214, "step": 3, - "step_time": 10.315485310000213 + "step_time": 8.97435381700052 }, { - "clip_ratio/high_max": 0.0069444444961845875, - "clip_ratio/high_mean": 0.0034722222480922937, - "clip_ratio/low_mean": 0.027083334047347307, + "clip_ratio/high_max": 0.02291666716337204, + "clip_ratio/high_mean": 0.01145833358168602, + "clip_ratio/low_mean": 0.03958333400078118, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0305555562954396, - "entropy": 0.40274658892303705, + "clip_ratio/region_mean": 0.0510416675824672, + "entropy": 0.4073071926832199, "epoch": 4e-05, - "grad_norm": 0.5002186298370361, - "kl": 0.008539682462242126, + "grad_norm": 0.6595498323440552, + "kl": 0.008635511804641283, "learning_rate": 8.571428571428573e-06, - "loss": -0.0029, + "loss": -0.014, "step": 4, - "step_time": 6.540185272000144 + "step_time": 6.212162247000833 }, { - "clip_ratio/high_max": 0.012500000186264515, - "clip_ratio/high_mean": 0.0062500000931322575, - "clip_ratio/low_mean": 0.015625000465661287, + "clip_ratio/high_max": 0.04583333432674408, + "clip_ratio/high_mean": 0.02291666716337204, + "clip_ratio/low_mean": 0.033333334140479565, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.021875000558793545, + "clip_ratio/region_mean": 0.056250001303851604, "completions/clipped_ratio": 0.0, - "completions/max_length": 1022.0, - "completions/max_terminated_length": 1022.0, - "completions/mean_length": 225.15625, - "completions/mean_terminated_length": 225.15625, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 230.53125, + "completions/mean_terminated_length": 230.53125, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.2807091409340501, + "entropy": 0.343780972994864, "epoch": 5e-05, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7028033137321472, - "kl": 0.047651219653289445, + "grad_norm": 0.8115894198417664, + "kl": 0.0070509411821149115, "learning_rate": 1.1428571428571429e-05, - "loss": -0.017, - "num_tokens": 115228.0, - "reward": -0.8011959791183472, - "reward_std": 0.4463964104652405, - "rewards/rollout_reward_func/mean": -0.8011959791183472, - "rewards/rollout_reward_func/std": 0.6310392618179321, - "sampling/importance_sampling_ratio/max": 1.9444010257720947, - "sampling/importance_sampling_ratio/mean": 0.9612840414047241, - "sampling/importance_sampling_ratio/min": 0.2667683959007263, - "sampling/sampling_logp_difference/max": 0.8543260097503662, - "sampling/sampling_logp_difference/mean": 0.042739734053611755, + "loss": -0.0135, + "num_tokens": 117398.0, + "reward": -0.2601696848869324, + "reward_std": 1.092232346534729, + "rewards/rollout_reward_func/mean": -0.2601696848869324, + "rewards/rollout_reward_func/std": 1.0665321350097656, + "sampling/importance_sampling_ratio/max": 1.834230661392212, + "sampling/importance_sampling_ratio/mean": 0.9836117029190063, + "sampling/importance_sampling_ratio/min": 0.43075546622276306, + "sampling/sampling_logp_difference/max": 0.8512740135192871, + "sampling/sampling_logp_difference/mean": 0.06270835548639297, "step": 5, - "step_time": 9.484377300000233 + "step_time": 8.83143492499903 }, { - "clip_ratio/high_max": 0.018750000279396772, - "clip_ratio/high_mean": 0.009375000139698386, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.035416667349636555, + "clip_ratio/high_mean": 0.017708333674818277, + "clip_ratio/low_mean": 0.042708334513008595, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.009375000139698386, - "entropy": 0.28512229211628437, + "clip_ratio/region_mean": 0.06041666818782687, + "entropy": 0.3260791110806167, "epoch": 6e-05, - "grad_norm": 0.8833194971084595, - "kl": 0.011992669685696455, + "grad_norm": 0.5751013159751892, + "kl": 0.020635396516809124, "learning_rate": 1.4285714285714285e-05, - "loss": -0.0202, + "loss": -0.0158, "step": 6, - "step_time": 6.462289121999902 + "step_time": 6.322354749000624 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.031250000931322575, + "clip_ratio/high_max": 0.037500000558793545, + "clip_ratio/high_mean": 0.018750000279396772, + "clip_ratio/low_mean": 0.0377976200543344, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.031250000931322575, + "clip_ratio/region_mean": 0.056547620333731174, "completions/clipped_ratio": 0.0, - "completions/max_length": 1142.0, - "completions/max_terminated_length": 1142.0, - "completions/mean_length": 308.25, - "completions/mean_terminated_length": 308.25, + "completions/max_length": 800.0, + "completions/max_terminated_length": 800.0, + "completions/mean_length": 297.28125, + "completions/mean_terminated_length": 297.28125, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.3624303378164768, + "entropy": 0.3736900817602873, "epoch": 7e-05, "frac_reward_zero_std": 0.0, - "grad_norm": 0.885384738445282, - "kl": 0.005326485250407131, + "grad_norm": 1.933777928352356, + "kl": 0.025599397898986354, "learning_rate": 1.7142857142857145e-05, - "loss": -0.008, - "num_tokens": 155454.0, - "reward": -0.8657017350196838, - "reward_std": 0.3935210704803467, - "rewards/rollout_reward_func/mean": -0.8657017350196838, - "rewards/rollout_reward_func/std": 0.5249258875846863, - "sampling/importance_sampling_ratio/max": 1.14151930809021, - "sampling/importance_sampling_ratio/mean": 0.9441938400268555, - "sampling/importance_sampling_ratio/min": 0.6393771767616272, - "sampling/sampling_logp_difference/max": 0.4876275062561035, - "sampling/sampling_logp_difference/mean": 0.037204425781965256, + "loss": 0.0014, + "num_tokens": 157273.0, + "reward": -0.49023765325546265, + "reward_std": 0.844656229019165, + "rewards/rollout_reward_func/mean": -0.49023765325546265, + "rewards/rollout_reward_func/std": 0.873151957988739, + "sampling/importance_sampling_ratio/max": 1.580020546913147, + "sampling/importance_sampling_ratio/mean": 0.9382479786872864, + "sampling/importance_sampling_ratio/min": 0.3616046607494354, + "sampling/sampling_logp_difference/max": 0.9256019592285156, + "sampling/sampling_logp_difference/mean": 0.055381715297698975, "step": 7, - "step_time": 10.03836135199981 + "step_time": 8.842048482999417 }, { - "clip_ratio/high_max": 0.013194444589316845, - "clip_ratio/high_mean": 0.0065972222946584225, - "clip_ratio/low_mean": 0.036805556854233146, - "clip_ratio/low_min": 0.012500000186264515, - "clip_ratio/region_mean": 0.04340277914889157, - "entropy": 0.3480250327847898, + "clip_ratio/high_max": 0.05000000074505806, + "clip_ratio/high_mean": 0.02500000037252903, + "clip_ratio/low_mean": 0.06770833535119891, + "clip_ratio/low_min": 0.0062500000931322575, + "clip_ratio/region_mean": 0.09270833572372794, + "entropy": 0.33360453927889466, "epoch": 8e-05, - "grad_norm": 0.6112654805183411, - "kl": 0.007768443218083121, + "grad_norm": 0.44949227571487427, + "kl": 0.07357259346099454, "learning_rate": 2e-05, - "loss": -0.0087, + "loss": 0.0006, "step": 8, - "step_time": 6.732798993000642 + "step_time": 6.202019984999879 }, { - "clip_ratio/high_max": 0.005434782709926367, - "clip_ratio/high_mean": 0.0027173913549631834, - "clip_ratio/low_mean": 0.0062500000931322575, + "clip_ratio/high_max": 0.031845238991081715, + "clip_ratio/high_mean": 0.015922619495540857, + "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.00896739144809544, + "clip_ratio/region_mean": 0.03258928656578064, "completions/clipped_ratio": 0.0, - "completions/max_length": 1163.0, - "completions/max_terminated_length": 1163.0, - "completions/mean_length": 258.75, - "completions/mean_terminated_length": 258.75, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.27044630981981754, + "completions/max_length": 790.0, + "completions/max_terminated_length": 790.0, + "completions/mean_length": 331.4375, + "completions/mean_terminated_length": 331.4375, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.2882281211204827, "epoch": 9e-05, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7710390090942383, - "kl": 0.02225630160091896, + "grad_norm": 0.8685204982757568, + "kl": 0.11152700683487637, "learning_rate": 2.2857142857142858e-05, - "loss": -0.01, - "num_tokens": 193934.0, - "reward": -0.9323001503944397, - "reward_std": 0.20817707479000092, - "rewards/rollout_reward_func/mean": -0.9323001503944397, - "rewards/rollout_reward_func/std": 0.37744954228401184, - "sampling/importance_sampling_ratio/max": 1.3011233806610107, - "sampling/importance_sampling_ratio/mean": 0.9557175636291504, - "sampling/importance_sampling_ratio/min": 0.5675007700920105, - "sampling/sampling_logp_difference/max": 0.6186580657958984, - "sampling/sampling_logp_difference/mean": 0.04273971542716026, + "loss": -0.0045, + "num_tokens": 198079.0, + "reward": -0.17929740250110626, + "reward_std": 1.0320875644683838, + "rewards/rollout_reward_func/mean": -0.17929740250110626, + "rewards/rollout_reward_func/std": 0.9943534731864929, + "sampling/importance_sampling_ratio/max": 1.4182045459747314, + "sampling/importance_sampling_ratio/mean": 0.9032353162765503, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1037428379058838, + "sampling/sampling_logp_difference/mean": 0.07490409910678864, "step": 9, - "step_time": 10.003510364000476 + "step_time": 8.829083078998792 }, { - "clip_ratio/high_max": 0.014363354537636042, - "clip_ratio/high_mean": 0.007181677268818021, - "clip_ratio/low_mean": 0.06041666818782687, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.06759834592230618, - "entropy": 0.24492651503533125, + "clip_ratio/high_max": 0.08452381193637848, + "clip_ratio/high_mean": 0.04226190596818924, + "clip_ratio/low_mean": 0.08697916846722364, + "clip_ratio/low_min": 0.02500000037252903, + "clip_ratio/region_mean": 0.12924107443541288, + "entropy": 0.2659805165603757, "epoch": 0.0001, - "grad_norm": 0.3038092851638794, - "kl": 0.061313093479839154, + "grad_norm": 0.9906594157218933, + "kl": 0.9256867625904306, "learning_rate": 2.5714285714285714e-05, - "loss": -0.0121, + "loss": -0.0059, "step": 10, - "step_time": 6.334576341999991 + "step_time": 5.6760745649989985 }, { - "clip_ratio/high_max": 0.010416666977107525, - "clip_ratio/high_mean": 0.0052083334885537624, - "clip_ratio/low_mean": 0.01666666753590107, - "clip_ratio/low_min": 0.012500000186264515, - "clip_ratio/region_mean": 0.021875001024454832, + "clip_ratio/high_max": 0.012500000186264515, + "clip_ratio/high_mean": 0.0062500000931322575, + "clip_ratio/low_mean": 0.031250000931322575, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.03750000102445483, "completions/clipped_ratio": 0.0, - "completions/max_length": 1808.0, - "completions/max_terminated_length": 1808.0, - "completions/mean_length": 366.3125, - "completions/mean_terminated_length": 366.3125, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 331.59375, + "completions/mean_terminated_length": 331.59375, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.32991205714643, + "entropy": 0.28942853957414627, "epoch": 0.00011, "frac_reward_zero_std": 0.0, - "grad_norm": 0.708510160446167, - "kl": 0.1540648011941812, + "grad_norm": 1.6359856128692627, + "kl": 0.6161578541941708, "learning_rate": 2.857142857142857e-05, - "loss": -0.0489, - "num_tokens": 236260.0, - "reward": -0.9286847114562988, - "reward_std": 0.211622416973114, - "rewards/rollout_reward_func/mean": -0.9286847114562988, - "rewards/rollout_reward_func/std": 0.3866735100746155, - "sampling/importance_sampling_ratio/max": 1.9185539484024048, - "sampling/importance_sampling_ratio/mean": 0.9692364931106567, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 1.5063650608062744, - "sampling/sampling_logp_difference/mean": 0.07806695252656937, + "loss": -0.0416, + "num_tokens": 239294.0, + "reward": -0.11925028264522552, + "reward_std": 1.0386744737625122, + "rewards/rollout_reward_func/mean": -0.11925028264522552, + "rewards/rollout_reward_func/std": 1.0166317224502563, + "sampling/importance_sampling_ratio/max": 2.841062307357788, + "sampling/importance_sampling_ratio/mean": 0.9466050267219543, + "sampling/importance_sampling_ratio/min": 0.10215280205011368, + "sampling/sampling_logp_difference/max": 2.3139443397521973, + "sampling/sampling_logp_difference/mean": 0.10802102088928223, "step": 11, - "step_time": 12.247949407000306 + "step_time": 9.289636140998937 }, { - "clip_ratio/high_max": 0.05850694561377168, - "clip_ratio/high_mean": 0.032725695287808776, - "clip_ratio/low_mean": 0.06458333553746343, - "clip_ratio/low_min": 0.033333334140479565, - "clip_ratio/region_mean": 0.0973090308252722, - "entropy": 0.3409234285354614, + "clip_ratio/high_max": 0.07916666846722364, + "clip_ratio/high_mean": 0.03958333423361182, + "clip_ratio/low_mean": 0.12559524131938815, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.16517857601866126, + "entropy": 0.2683787359856069, "epoch": 0.00012, - "grad_norm": 0.527890682220459, - "kl": 0.5000243990944, + "grad_norm": 26.53432846069336, + "kl": 6.239889086456969, "learning_rate": 3.142857142857143e-05, - "loss": -0.0513, + "loss": -0.0345, "step": 12, - "step_time": 7.004255187000126 + "step_time": 5.24321525400137 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.018978387233801186, + "clip_ratio/high_max": 0.015625, + "clip_ratio/high_mean": 0.0078125, + "clip_ratio/low_mean": 0.052083334885537624, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.018978387233801186, + "clip_ratio/region_mean": 0.059895834885537624, "completions/clipped_ratio": 0.0, - "completions/max_length": 1584.0, - "completions/max_terminated_length": 1584.0, - "completions/mean_length": 398.375, - "completions/mean_terminated_length": 398.375, + "completions/max_length": 790.0, + "completions/max_terminated_length": 790.0, + "completions/mean_length": 329.875, + "completions/mean_terminated_length": 329.875, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.28245622059330344, + "entropy": 0.2237180758966133, "epoch": 0.00013, "frac_reward_zero_std": 0.0, - "grad_norm": 1.366127371788025, - "kl": 0.26600568945286795, + "grad_norm": 0.7478111386299133, + "kl": 0.8387041406749631, "learning_rate": 3.428571428571429e-05, - "loss": 0.0152, - "num_tokens": 278805.0, - "reward": -0.6331639289855957, - "reward_std": 0.7617669105529785, - "rewards/rollout_reward_func/mean": -0.6331639289855957, - "rewards/rollout_reward_func/std": 0.7844160199165344, - "sampling/importance_sampling_ratio/max": 1.327277660369873, - "sampling/importance_sampling_ratio/mean": 0.8719787001609802, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 1.2040467262268066, - "sampling/sampling_logp_difference/mean": 0.07095030695199966, + "loss": -0.0369, + "num_tokens": 279647.0, + "reward": -0.056309767067432404, + "reward_std": 0.940862774848938, + "rewards/rollout_reward_func/mean": -0.056309767067432404, + "rewards/rollout_reward_func/std": 0.9333992004394531, + "sampling/importance_sampling_ratio/max": 2.758878231048584, + "sampling/importance_sampling_ratio/mean": 1.106706142425537, + "sampling/importance_sampling_ratio/min": 0.19236239790916443, + "sampling/sampling_logp_difference/max": 1.6484246253967285, + "sampling/sampling_logp_difference/mean": 0.08536527305841446, "step": 13, - "step_time": 12.302737500999683 + "step_time": 9.787576804999844 }, { - "clip_ratio/high_max": 0.0416666679084301, - "clip_ratio/high_mean": 0.02083333395421505, - "clip_ratio/low_mean": 0.13363321172073483, - "clip_ratio/low_min": 0.04375000111758709, - "clip_ratio/region_mean": 0.15446654567494988, - "entropy": 0.2645200351253152, + "clip_ratio/high_max": 0.10312500223517418, + "clip_ratio/high_mean": 0.05156250111758709, + "clip_ratio/low_mean": 0.07291666883975267, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.12447916995733976, + "entropy": 0.20552951516583562, "epoch": 0.00014, - "grad_norm": 0.6336849927902222, - "kl": 1.7774635917339765, + "grad_norm": 0.38508477807044983, + "kl": 0.8960464763877098, "learning_rate": 3.7142857142857143e-05, - "loss": 0.0159, + "loss": -0.0386, "step": 14, - "step_time": 6.674852292999958 + "step_time": 5.245387445001143 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.05131448491010815, - "clip_ratio/low_min": 0.010416666977107525, - "clip_ratio/region_mean": 0.05131448491010815, + "clip_ratio/high_max": 0.1145833358168602, + "clip_ratio/high_mean": 0.0572916679084301, + "clip_ratio/low_mean": 0.039062500931322575, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.09635416837409139, "completions/clipped_ratio": 0.0, - "completions/max_length": 2135.0, - "completions/max_terminated_length": 2135.0, - "completions/mean_length": 362.1875, - "completions/mean_terminated_length": 362.1875, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 359.5, + "completions/mean_terminated_length": 359.5, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.33170084515586495, + "entropy": 0.2046172993723303, "epoch": 0.00015, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5385106801986694, - "kl": 0.5612458096584305, + "grad_norm": 0.8827261328697205, + "kl": 1.1722645154222846, "learning_rate": 4e-05, - "loss": 0.0529, - "num_tokens": 321363.0, - "reward": -0.22063349187374115, - "reward_std": 1.032537579536438, - "rewards/rollout_reward_func/mean": -0.22063349187374115, - "rewards/rollout_reward_func/std": 1.0019752979278564, - "sampling/importance_sampling_ratio/max": 2.0951614379882812, - "sampling/importance_sampling_ratio/mean": 1.0781480073928833, - "sampling/importance_sampling_ratio/min": 0.40334585309028625, - "sampling/sampling_logp_difference/max": 0.7475135326385498, - "sampling/sampling_logp_difference/mean": 0.05335211753845215, + "loss": -0.0224, + "num_tokens": 322119.0, + "reward": 0.6023371815681458, + "reward_std": 0.8078142404556274, + "rewards/rollout_reward_func/mean": 0.6023371815681458, + "rewards/rollout_reward_func/std": 0.8529617190361023, + "sampling/importance_sampling_ratio/max": 1.5152462720870972, + "sampling/importance_sampling_ratio/mean": 0.7405616044998169, + "sampling/importance_sampling_ratio/min": 0.03419314697384834, + "sampling/sampling_logp_difference/max": 2.0772204399108887, + "sampling/sampling_logp_difference/mean": 0.19836057722568512, "step": 15, - "step_time": 13.901658290000341 + "step_time": 9.927378570999736 }, { - "clip_ratio/high_max": 0.04583333432674408, - "clip_ratio/high_mean": 0.02291666716337204, - "clip_ratio/low_mean": 0.07991071639116853, + "clip_ratio/high_max": 0.033333334140479565, + "clip_ratio/high_mean": 0.016666667070239782, + "clip_ratio/low_mean": 0.12708333600312471, "clip_ratio/low_min": 0.02083333395421505, - "clip_ratio/region_mean": 0.10282738448586315, - "entropy": 0.33263772330246866, + "clip_ratio/region_mean": 0.1437500030733645, + "entropy": 0.12798613973427564, "epoch": 0.00016, - "grad_norm": 0.3616006076335907, - "kl": 0.5915351018775254, + "grad_norm": 25.750520706176758, + "kl": 33.92647529486567, "learning_rate": 4.2857142857142856e-05, - "loss": 0.0492, + "loss": 0.0152, "step": 16, - "step_time": 7.827889540000115 + "step_time": 5.281957132000571 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.0416666679084301, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.052083334885537624, "completions/clipped_ratio": 0.0, - "completions/max_length": 1066.0, - "completions/max_terminated_length": 1066.0, - "completions/mean_length": 146.75, - "completions/mean_terminated_length": 146.75, + "completions/max_length": 726.0, + "completions/max_terminated_length": 726.0, + "completions/mean_length": 162.9375, + "completions/mean_terminated_length": 162.9375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.21940526392427273, + "entropy": 0.18993338634027168, "epoch": 0.00017, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7156424522399902, - "kl": 0.9828777723014355, + "grad_norm": 0.7148876190185547, + "kl": 2.529468312859535, "learning_rate": 4.5714285714285716e-05, - "loss": 0.0274, - "num_tokens": 354064.0, - "reward": 0.18792903423309326, - "reward_std": 1.0650506019592285, - "rewards/rollout_reward_func/mean": 0.18792903423309326, - "rewards/rollout_reward_func/std": 1.025183081626892, - "sampling/importance_sampling_ratio/max": 1.465952754020691, - "sampling/importance_sampling_ratio/mean": 0.9368042945861816, - "sampling/importance_sampling_ratio/min": 0.2505682706832886, - "sampling/sampling_logp_difference/max": 1.1606903076171875, - "sampling/sampling_logp_difference/mean": 0.06353487074375153, + "loss": -0.0248, + "num_tokens": 355338.0, + "reward": 0.1518549919128418, + "reward_std": 0.9506906867027283, + "rewards/rollout_reward_func/mean": 0.1518549919128418, + "rewards/rollout_reward_func/std": 1.0281562805175781, + "sampling/importance_sampling_ratio/max": 1.88484525680542, + "sampling/importance_sampling_ratio/mean": 0.8383225202560425, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.9152507781982422, + "sampling/sampling_logp_difference/mean": 0.1459488719701767, "step": 17, - "step_time": 10.211389272000588 + "step_time": 9.555490649999683 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.11508049350231886, - "clip_ratio/low_min": 0.033333334140479565, - "clip_ratio/region_mean": 0.11508049350231886, - "entropy": 0.19974320210167207, + "clip_ratio/high_max": 0.02291666716337204, + "clip_ratio/high_mean": 0.01145833358168602, + "clip_ratio/low_mean": 0.07812500093132257, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0895833345130086, + "entropy": 0.18508516627480276, "epoch": 0.00018, - "grad_norm": 2.1809680461883545, - "kl": 2.973201504908502, + "grad_norm": 3.493649482727051, + "kl": 18.445159916533157, "learning_rate": 4.8571428571428576e-05, - "loss": 0.0255, + "loss": -0.0124, "step": 18, - "step_time": 5.537876568999991 + "step_time": 5.199743792999925 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.02291666716337204, + "clip_ratio/low_mean": 0.016666667070239782, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.02291666716337204, + "clip_ratio/region_mean": 0.016666667070239782, "completions/clipped_ratio": 0.0, - "completions/max_length": 1190.0, - "completions/max_terminated_length": 1190.0, - "completions/mean_length": 183.71875, - "completions/mean_terminated_length": 183.71875, + "completions/max_length": 755.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 289.65625, + "completions/mean_terminated_length": 289.65625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.2391725802444853, + "entropy": 0.08967042146832682, "epoch": 0.00019, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5152996182441711, - "kl": 0.628770228009671, + "grad_norm": 0.8586666584014893, + "kl": 0.6446084327762946, "learning_rate": 5.142857142857143e-05, - "loss": 0.0056, - "num_tokens": 389262.0, - "reward": 0.09779167175292969, - "reward_std": 0.9968031048774719, - "rewards/rollout_reward_func/mean": 0.09779167175292969, - "rewards/rollout_reward_func/std": 1.0246973037719727, - "sampling/importance_sampling_ratio/max": 1.6519438028335571, - "sampling/importance_sampling_ratio/mean": 0.9000259637832642, - "sampling/importance_sampling_ratio/min": 0.026913145557045937, - "sampling/sampling_logp_difference/max": 3.1608994007110596, - "sampling/sampling_logp_difference/mean": 0.10347369313240051, + "loss": -0.0188, + "num_tokens": 393926.0, + "reward": 0.7382206916809082, + "reward_std": 0.785092830657959, + "rewards/rollout_reward_func/mean": 0.7382206916809082, + "rewards/rollout_reward_func/std": 0.8255225419998169, + "sampling/importance_sampling_ratio/max": 1.0873334407806396, + "sampling/importance_sampling_ratio/mean": 0.9170267581939697, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1249277591705322, + "sampling/sampling_logp_difference/mean": 0.034125279635190964, "step": 19, - "step_time": 11.03900628299948 + "step_time": 9.765665665999677 }, { - "clip_ratio/high_max": 0.012500000186264515, - "clip_ratio/high_mean": 0.0062500000931322575, - "clip_ratio/low_mean": 0.14107143180444837, - "clip_ratio/low_min": 0.051190477795898914, - "clip_ratio/region_mean": 0.14732143189758062, - "entropy": 0.25456089232466184, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.05833333497866988, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.05833333497866988, + "entropy": 0.1150879084598273, "epoch": 0.0002, - "grad_norm": 0.3765561580657959, - "kl": 1.1224204124882817, + "grad_norm": 0.8072354793548584, + "kl": 1.808403274975717, "learning_rate": 5.428571428571428e-05, - "loss": 0.0047, + "loss": -0.0181, "step": 20, - "step_time": 6.163490915999773 + "step_time": 5.240651811000134 }, { - "clip_ratio/high_max": 0.010416666977107525, - "clip_ratio/high_mean": 0.0052083334885537624, - "clip_ratio/low_mean": 0.007812500232830644, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.004464285913854837, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.013020833721384406, + "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, - "completions/max_length": 2228.0, - "completions/max_terminated_length": 2228.0, - "completions/mean_length": 374.4375, - "completions/mean_terminated_length": 374.4375, + "completions/max_length": 790.0, + "completions/max_terminated_length": 790.0, + "completions/mean_length": 364.0625, + "completions/mean_terminated_length": 364.0625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.20231306218192913, + "entropy": 0.11025293295097072, "epoch": 0.00021, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5450725555419922, - "kl": 1.0656099561601877, + "grad_norm": 0.4521680772304535, + "kl": 0.5892425078200176, "learning_rate": 5.714285714285714e-05, - "loss": -0.0062, - "num_tokens": 432157.0, - "reward": 0.3385208547115326, - "reward_std": 0.9284825921058655, - "rewards/rollout_reward_func/mean": 0.3385208547115326, - "rewards/rollout_reward_func/std": 0.9307616949081421, - "sampling/importance_sampling_ratio/max": 1.735904335975647, - "sampling/importance_sampling_ratio/mean": 0.8049801588058472, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 2.504056453704834, - "sampling/sampling_logp_difference/mean": 0.09936025738716125, + "loss": -0.0034, + "num_tokens": 436489.0, + "reward": 0.650588870048523, + "reward_std": 0.8124028444290161, + "rewards/rollout_reward_func/mean": 0.650588870048523, + "rewards/rollout_reward_func/std": 0.8418978452682495, + "sampling/importance_sampling_ratio/max": 1.2940930128097534, + "sampling/importance_sampling_ratio/mean": 0.8965916037559509, + "sampling/importance_sampling_ratio/min": 0.24205046892166138, + "sampling/sampling_logp_difference/max": 1.3280644416809082, + "sampling/sampling_logp_difference/mean": 0.05257989466190338, "step": 21, - "step_time": 14.117857149999963 + "step_time": 9.344274248000602 }, { - "clip_ratio/high_max": 0.017361111473292112, - "clip_ratio/high_mean": 0.011805556016042829, - "clip_ratio/low_mean": 0.07820799155160785, - "clip_ratio/low_min": 0.027472528629004955, - "clip_ratio/region_mean": 0.09001354733482003, - "entropy": 0.20029474921466317, + "clip_ratio/high_max": 0.07916666846722364, + "clip_ratio/high_mean": 0.03958333423361182, + "clip_ratio/low_mean": 0.02113095298409462, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.06071428768336773, + "entropy": 0.1187713702529436, "epoch": 0.00022, - "grad_norm": 0.38578858971595764, - "kl": 1.4086124375462532, + "grad_norm": 0.12441845238208771, + "kl": 0.6386879712226801, "learning_rate": 6e-05, - "loss": -0.008, + "loss": -0.0045, "step": 22, - "step_time": 7.929054158000099 + "step_time": 5.255650161999256 }, { - "clip_ratio/high_max": 0.02500000037252903, - "clip_ratio/high_mean": 0.012500000186264515, - "clip_ratio/low_mean": 0.010156250093132257, + "clip_ratio/high_max": 0.012500000186264515, + "clip_ratio/high_mean": 0.0062500000931322575, + "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.022656250279396772, + "clip_ratio/region_mean": 0.016666667070239782, "completions/clipped_ratio": 0.0, - "completions/max_length": 896.0, - "completions/max_terminated_length": 896.0, - "completions/mean_length": 165.5, - "completions/mean_terminated_length": 165.5, - "completions/min_length": 3.0, - "completions/min_terminated_length": 3.0, - "entropy": 0.12490838440135121, + "completions/max_length": 792.0, + "completions/max_terminated_length": 792.0, + "completions/mean_length": 237.0, + "completions/mean_terminated_length": 237.0, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 0.11261143760930281, "epoch": 0.00023, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5326600074768066, - "kl": 2.6836317628622055, + "grad_norm": 0.41230642795562744, + "kl": 1.1587365122977644, "learning_rate": 6.285714285714286e-05, - "loss": -0.0076, - "num_tokens": 466199.0, - "reward": 0.6161667108535767, - "reward_std": 0.6345345377922058, - "rewards/rollout_reward_func/mean": 0.6161667108535767, - "rewards/rollout_reward_func/std": 0.7305893301963806, - "sampling/importance_sampling_ratio/max": 1.6318753957748413, - "sampling/importance_sampling_ratio/mean": 0.8681706190109253, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 1.4805762767791748, - "sampling/sampling_logp_difference/mean": 0.07480701804161072, + "loss": -0.0302, + "num_tokens": 472819.0, + "reward": 0.6253360509872437, + "reward_std": 0.8840253353118896, + "rewards/rollout_reward_func/mean": 0.6253360509872437, + "rewards/rollout_reward_func/std": 0.8717224597930908, + "sampling/importance_sampling_ratio/max": 2.5167317390441895, + "sampling/importance_sampling_ratio/mean": 0.9689525365829468, + "sampling/importance_sampling_ratio/min": 0.20134451985359192, + "sampling/sampling_logp_difference/max": 1.4174327850341797, + "sampling/sampling_logp_difference/mean": 0.05677894502878189, "step": 23, - "step_time": 9.858622221000587 + "step_time": 9.840772545998789 }, { - "clip_ratio/high_max": 0.026785715483129025, - "clip_ratio/high_mean": 0.013392857741564512, - "clip_ratio/low_mean": 0.01927083358168602, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.03266369132325053, - "entropy": 0.10265930875902995, + "clip_ratio/high_max": 0.033333334140479565, + "clip_ratio/high_mean": 0.016666667070239782, + "clip_ratio/low_mean": 0.0739583345130086, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.09062500158324838, + "entropy": 0.11950548080494627, "epoch": 0.00024, - "grad_norm": 2.3974478244781494, - "kl": 3.5795856304466724, + "grad_norm": 0.7795855402946472, + "kl": 3.3370795460650697, "learning_rate": 6.571428571428571e-05, - "loss": -0.0019, + "loss": -0.0304, "step": 24, - "step_time": 5.317549161999295 + "step_time": 5.254853659001128 }, { - "clip_ratio/high_max": 0.02500000037252903, - "clip_ratio/high_mean": 0.012500000186264515, - "clip_ratio/low_mean": 0.016964286100119352, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.03750000102445483, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.029464286286383867, + "clip_ratio/region_mean": 0.03750000102445483, "completions/clipped_ratio": 0.0, - "completions/max_length": 1176.0, - "completions/max_terminated_length": 1176.0, - "completions/mean_length": 240.1875, - "completions/mean_terminated_length": 240.1875, - "completions/min_length": 3.0, - "completions/min_terminated_length": 3.0, - "entropy": 0.137992134608794, + "completions/max_length": 763.0, + "completions/max_terminated_length": 763.0, + "completions/mean_length": 287.03125, + "completions/mean_terminated_length": 287.03125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 0.17286038988095243, "epoch": 0.00025, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6068210601806641, - "kl": 1.5051679126918316, + "grad_norm": 0.6135607957839966, + "kl": 2.3912164487410337, "learning_rate": 6.857142857142858e-05, - "loss": -0.002, - "num_tokens": 503259.0, - "reward": 0.6975833177566528, - "reward_std": 0.7012296319007874, - "rewards/rollout_reward_func/mean": 0.6975833177566528, - "rewards/rollout_reward_func/std": 0.7207034826278687, - "sampling/importance_sampling_ratio/max": 1.254854679107666, - "sampling/importance_sampling_ratio/mean": 0.838233470916748, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 2.63065767288208, - "sampling/sampling_logp_difference/mean": 0.094658762216568, + "loss": -0.0093, + "num_tokens": 511378.0, + "reward": 0.5773664712905884, + "reward_std": 0.6574704051017761, + "rewards/rollout_reward_func/mean": 0.5773664712905884, + "rewards/rollout_reward_func/std": 0.837111234664917, + "sampling/importance_sampling_ratio/max": 2.1233153343200684, + "sampling/importance_sampling_ratio/mean": 0.9125764966011047, + "sampling/importance_sampling_ratio/min": 0.2271677851676941, + "sampling/sampling_logp_difference/max": 1.4825472831726074, + "sampling/sampling_logp_difference/mean": 0.07279175519943237, "step": 25, - "step_time": 10.299735158999283 + "step_time": 9.7455945050001 }, { - "clip_ratio/high_max": 0.04642857238650322, - "clip_ratio/high_mean": 0.029464286286383867, - "clip_ratio/low_mean": 0.041778274811804295, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.07124256109818816, - "entropy": 0.16137297594104894, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.06562500214204192, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.06562500214204192, + "entropy": 0.20867955926223658, "epoch": 0.00026, - "grad_norm": 5.5549468994140625, - "kl": 11.136886082589626, + "grad_norm": 2.795644998550415, + "kl": 3.72620632359758, "learning_rate": 7.142857142857143e-05, - "loss": 0.0342, + "loss": 0.0015, "step": 26, - "step_time": 5.75506067900028 + "step_time": 5.24237381399962 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, + "clip_ratio/high_max": 0.012500000186264515, + "clip_ratio/high_mean": 0.0062500000931322575, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, - "completions/max_length": 734.0, - "completions/max_terminated_length": 734.0, - "completions/mean_length": 146.28125, - "completions/mean_terminated_length": 146.28125, - "completions/min_length": 3.0, - "completions/min_terminated_length": 3.0, - "entropy": 0.15740215915138833, + "completions/max_length": 800.0, + "completions/max_terminated_length": 800.0, + "completions/mean_length": 250.34375, + "completions/mean_terminated_length": 250.34375, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 0.2467669948528055, "epoch": 0.00027, "frac_reward_zero_std": 0.0, - "grad_norm": 0.7164216041564941, - "kl": 1.4110768623650074, + "grad_norm": 0.33079302310943604, + "kl": 2.3681478649377823, "learning_rate": 7.428571428571429e-05, - "loss": -0.0054, - "num_tokens": 536415.0, - "reward": 0.613898515701294, - "reward_std": 0.7614923715591431, - "rewards/rollout_reward_func/mean": 0.613898515701294, - "rewards/rollout_reward_func/std": 0.7739964723587036, - "sampling/importance_sampling_ratio/max": 1.3575035333633423, - "sampling/importance_sampling_ratio/mean": 0.9464373588562012, - "sampling/importance_sampling_ratio/min": 0.18349826335906982, - "sampling/sampling_logp_difference/max": 1.3662632703781128, - "sampling/sampling_logp_difference/mean": 0.043687544763088226, + "loss": -0.0214, + "num_tokens": 547864.0, + "reward": 0.4095150828361511, + "reward_std": 0.8980504274368286, + "rewards/rollout_reward_func/mean": 0.4095150828361511, + "rewards/rollout_reward_func/std": 0.9758495688438416, + "sampling/importance_sampling_ratio/max": 1.3963953256607056, + "sampling/importance_sampling_ratio/mean": 0.9021573066711426, + "sampling/importance_sampling_ratio/min": 0.0, + "sampling/sampling_logp_difference/max": 1.1349234580993652, + "sampling/sampling_logp_difference/mean": 0.06076967716217041, "step": 27, - "step_time": 9.54420504699965 + "step_time": 9.648788904000867 }, { - "clip_ratio/high_max": 0.0062500000931322575, - "clip_ratio/high_mean": 0.0031250000465661287, - "clip_ratio/low_mean": 0.05654762079939246, + "clip_ratio/high_max": 0.02500000037252903, + "clip_ratio/high_mean": 0.012500000186264515, + "clip_ratio/low_mean": 0.07916666800156236, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.05967262154445052, - "entropy": 0.14962338455370627, + "clip_ratio/region_mean": 0.09166666772216558, + "entropy": 0.25531527734710835, "epoch": 0.00028, - "grad_norm": 0.2844441533088684, - "kl": 1.628791993483901, + "grad_norm": 0.2908107340335846, + "kl": 2.5475200429791585, "learning_rate": 7.714285714285715e-05, - "loss": -0.0084, + "loss": -0.0238, "step": 28, - "step_time": 5.213027034000561 + "step_time": 5.273828296999909 }, { - "clip_ratio/high_max": 0.03281250037252903, - "clip_ratio/high_mean": 0.016406250186264515, - "clip_ratio/low_mean": 0.001953125, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.018359375186264515, + "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, - "completions/max_length": 1858.0, - "completions/max_terminated_length": 1858.0, - "completions/mean_length": 245.40625, - "completions/mean_terminated_length": 245.40625, + "completions/max_length": 782.0, + "completions/max_terminated_length": 782.0, + "completions/mean_length": 269.84375, + "completions/mean_terminated_length": 269.84375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.1823743417917285, + "entropy": 0.19788118088035844, "epoch": 0.00029, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5472207069396973, - "kl": 1.2950569400563836, + "grad_norm": 0.5320978760719299, + "kl": 1.1093568538781255, "learning_rate": 8e-05, - "loss": 0.0192, - "num_tokens": 572908.0, - "reward": 0.7323958873748779, - "reward_std": 0.75600266456604, - "rewards/rollout_reward_func/mean": 0.7323958873748779, - "rewards/rollout_reward_func/std": 0.7951676249504089, - "sampling/importance_sampling_ratio/max": 1.7107934951782227, - "sampling/importance_sampling_ratio/mean": 1.0277063846588135, - "sampling/importance_sampling_ratio/min": 0.6659436225891113, - "sampling/sampling_logp_difference/max": 0.7972254753112793, - "sampling/sampling_logp_difference/mean": 0.0571967251598835, + "loss": -0.0198, + "num_tokens": 585139.0, + "reward": 0.5855833292007446, + "reward_std": 0.8924044966697693, + "rewards/rollout_reward_func/mean": 0.5855833292007446, + "rewards/rollout_reward_func/std": 0.9158775210380554, + "sampling/importance_sampling_ratio/max": 1.1781891584396362, + "sampling/importance_sampling_ratio/mean": 0.9113141894340515, + "sampling/importance_sampling_ratio/min": 0.3537837862968445, + "sampling/sampling_logp_difference/max": 1.0374321937561035, + "sampling/sampling_logp_difference/mean": 0.045079514384269714, "step": 29, - "step_time": 12.326968075999275 + "step_time": 9.653419245999885 }, { - "clip_ratio/high_max": 0.07599431928247213, - "clip_ratio/high_mean": 0.03799715964123607, - "clip_ratio/low_mean": 0.029296875, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0729166679084301, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0672940332442522, - "entropy": 0.18252644933818374, + "clip_ratio/region_mean": 0.0729166679084301, + "entropy": 0.1806143777212128, "epoch": 0.0003, - "grad_norm": 0.42486661672592163, - "kl": 1.2426311783492565, + "grad_norm": 0.2963355481624603, + "kl": 1.555780888069421, "learning_rate": 8.285714285714287e-05, - "loss": 0.0204, + "loss": -0.0207, "step": 30, - "step_time": 7.674688885000478 + "step_time": 5.263566929000717 }, { - "clip_ratio/high_max": 0.02083333395421505, - "clip_ratio/high_mean": 0.010416666977107525, - "clip_ratio/low_mean": 0.010156250093132257, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.018229166977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.020572917070239782, + "clip_ratio/region_mean": 0.018229166977107525, "completions/clipped_ratio": 0.0, - "completions/max_length": 1072.0, - "completions/max_terminated_length": 1072.0, - "completions/mean_length": 203.8125, - "completions/mean_terminated_length": 203.8125, + "completions/max_length": 792.0, + "completions/max_terminated_length": 792.0, + "completions/mean_length": 253.3125, + "completions/mean_terminated_length": 253.3125, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.18441346354666166, + "entropy": 0.22652981715509668, "epoch": 0.00031, "frac_reward_zero_std": 0.0, - "grad_norm": 0.8369215130805969, - "kl": 2.210440620779991, + "grad_norm": 0.2698533833026886, + "kl": 2.315908302552998, "learning_rate": 8.571428571428571e-05, - "loss": 0.0219, - "num_tokens": 609478.0, - "reward": 0.6868749856948853, - "reward_std": 0.723414421081543, - "rewards/rollout_reward_func/mean": 0.6868749856948853, - "rewards/rollout_reward_func/std": 0.7201352715492249, - "sampling/importance_sampling_ratio/max": 2.42289662361145, - "sampling/importance_sampling_ratio/mean": 1.0437120199203491, - "sampling/importance_sampling_ratio/min": 0.0, - "sampling/sampling_logp_difference/max": 1.2931618690490723, - "sampling/sampling_logp_difference/mean": 0.06052036210894585, + "loss": -0.0034, + "num_tokens": 623293.0, + "reward": 0.7680624723434448, + "reward_std": 0.6883844137191772, + "rewards/rollout_reward_func/mean": 0.7680624723434448, + "rewards/rollout_reward_func/std": 0.6767417192459106, + "sampling/importance_sampling_ratio/max": 1.7574447393417358, + "sampling/importance_sampling_ratio/mean": 0.9467287063598633, + "sampling/importance_sampling_ratio/min": 0.36096832156181335, + "sampling/sampling_logp_difference/max": 1.1958250999450684, + "sampling/sampling_logp_difference/mean": 0.05750691145658493, "step": 31, - "step_time": 9.324151609999944 + "step_time": 9.186187056999188 }, { - "clip_ratio/high_max": 0.0476190485060215, - "clip_ratio/high_mean": 0.026650432962924242, - "clip_ratio/low_mean": 0.03255208348855376, + "clip_ratio/high_max": 0.06250000186264515, + "clip_ratio/high_mean": 0.0416666679084301, + "clip_ratio/low_mean": 0.04531250102445483, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.059202516451478004, - "entropy": 0.18722298718057573, + "clip_ratio/region_mean": 0.08697916893288493, + "entropy": 0.20133131777402014, "epoch": 0.00032, - "grad_norm": 0.1640804558992386, - "kl": 2.2689803540706635, + "grad_norm": 0.23155829310417175, + "kl": 3.2348447116091847, "learning_rate": 8.857142857142857e-05, - "loss": 0.0193, + "loss": -0.0032, "step": 32, - "step_time": 6.513984409999921 + "step_time": 5.697598966000442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0052083334885537624, + "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0052083334885537624, + "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 941.0, - "completions/max_terminated_length": 941.0, - "completions/mean_length": 260.71875, - "completions/mean_terminated_length": 260.71875, + "completions/max_length": 764.0, + "completions/max_terminated_length": 764.0, + "completions/mean_length": 331.1875, + "completions/mean_terminated_length": 331.1875, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.3189265597611666, + "entropy": 0.09508319746237248, "epoch": 0.00033, "frac_reward_zero_std": 0.0, - "grad_norm": 1.0061767101287842, - "kl": 1.4462116956710815, + "grad_norm": 0.2788764536380768, + "kl": 1.784507680684328, "learning_rate": 9.142857142857143e-05, - "loss": 0.0183, - "num_tokens": 648894.0, - "reward": 1.0101041793823242, - "reward_std": 0.3137112855911255, - "rewards/rollout_reward_func/mean": 1.0101041793823242, - "rewards/rollout_reward_func/std": 0.4550347924232483, - "sampling/importance_sampling_ratio/max": 2.6488871574401855, - "sampling/importance_sampling_ratio/mean": 1.0141404867172241, - "sampling/importance_sampling_ratio/min": 0.8440296649932861, - "sampling/sampling_logp_difference/max": 0.6730160713195801, - "sampling/sampling_logp_difference/mean": 0.037406109273433685, + "loss": 0.0006, + "num_tokens": 664964.0, + "reward": 0.9909583330154419, + "reward_std": 0.43575918674468994, + "rewards/rollout_reward_func/mean": 0.9909583330154419, + "rewards/rollout_reward_func/std": 0.42085883021354675, + "sampling/importance_sampling_ratio/max": 1.4408613443374634, + "sampling/importance_sampling_ratio/mean": 1.006264567375183, + "sampling/importance_sampling_ratio/min": 0.6942558288574219, + "sampling/sampling_logp_difference/max": 0.3652302026748657, + "sampling/sampling_logp_difference/mean": 0.016773704439401627, "step": 33, - "step_time": 9.245252041999493 + "step_time": 9.260905197000739 }, { - "clip_ratio/high_max": 0.03219697065651417, - "clip_ratio/high_mean": 0.016098485328257084, - "clip_ratio/low_mean": 0.03906250186264515, - "clip_ratio/low_min": 0.015625, - "clip_ratio/region_mean": 0.05516098719090223, - "entropy": 0.2540598356135888, + "clip_ratio/high_max": 0.033333334140479565, + "clip_ratio/high_mean": 0.016666667070239782, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.016666667070239782, + "entropy": 0.10259381297510117, "epoch": 0.00034, - "grad_norm": 3.8695592880249023, - "kl": 5.638480260968208, + "grad_norm": 0.266944944858551, + "kl": 1.788492577150464, "learning_rate": 9.428571428571429e-05, - "loss": 0.0256, + "loss": -0.0005, "step": 34, - "step_time": 6.270834967000383 + "step_time": 5.712432919000548 }, { - "clip_ratio/high_max": 0.008928571827709675, - "clip_ratio/high_mean": 0.004464285913854837, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.02500000037252903, + "clip_ratio/high_mean": 0.012500000186264515, + "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.004464285913854837, + "clip_ratio/region_mean": 0.018750000279396772, "completions/clipped_ratio": 0.0, - "completions/max_length": 996.0, - "completions/max_terminated_length": 996.0, - "completions/mean_length": 313.28125, - "completions/mean_terminated_length": 313.28125, + "completions/max_length": 791.0, + "completions/max_terminated_length": 791.0, + "completions/mean_length": 330.0625, + "completions/mean_terminated_length": 330.0625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.1834048866876401, + "entropy": 0.1567590031772852, "epoch": 0.00035, "frac_reward_zero_std": 0.0, - "grad_norm": 0.25010761618614197, - "kl": 2.1367476396262646, + "grad_norm": 0.8924462199211121, + "kl": 1.397438894797233, "learning_rate": 9.714285714285715e-05, "loss": 0.0022, - "num_tokens": 688634.0, - "reward": 0.755062460899353, - "reward_std": 0.5053232312202454, - "rewards/rollout_reward_func/mean": 0.755062460899353, - "rewards/rollout_reward_func/std": 0.5986501574516296, - "sampling/importance_sampling_ratio/max": 1.6370441913604736, - "sampling/importance_sampling_ratio/mean": 0.9969434142112732, - "sampling/importance_sampling_ratio/min": 0.5411856174468994, - "sampling/sampling_logp_difference/max": 0.6061768531799316, - "sampling/sampling_logp_difference/mean": 0.024847794324159622, + "num_tokens": 705241.0, + "reward": 0.8070625066757202, + "reward_std": 0.5308109521865845, + "rewards/rollout_reward_func/mean": 0.8070625066757202, + "rewards/rollout_reward_func/std": 0.5344893336296082, + "sampling/importance_sampling_ratio/max": 1.2556607723236084, + "sampling/importance_sampling_ratio/mean": 0.8737887144088745, + "sampling/importance_sampling_ratio/min": 0.6537890434265137, + "sampling/sampling_logp_difference/max": 0.5137598514556885, + "sampling/sampling_logp_difference/mean": 0.04966939985752106, "step": 35, - "step_time": 9.242927256999792 + "step_time": 8.738019706001523 }, { - "clip_ratio/high_max": 0.034598215483129025, - "clip_ratio/high_mean": 0.017299107741564512, - "clip_ratio/low_mean": 0.00937500037252903, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.026674108114093542, - "entropy": 0.1706730479199905, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.0812500030733645, + "clip_ratio/low_min": 0.037500000558793545, + "clip_ratio/region_mean": 0.09166667005047202, + "entropy": 0.1577406688593328, "epoch": 0.00036, - "grad_norm": 0.17176270484924316, - "kl": 2.141612984240055, + "grad_norm": 0.34831127524375916, + "kl": 1.6028189342468977, "learning_rate": 0.0001, - "loss": 0.0007, + "loss": 0.0041, "step": 36, - "step_time": 6.4294925179997335 + "step_time": 6.175464659000681 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.035416667349636555, + "clip_ratio/high_mean": 0.017708333674818277, + "clip_ratio/low_mean": 0.012500000186264515, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.030208333861082792, "completions/clipped_ratio": 0.0, - "completions/max_length": 1151.0, - "completions/max_terminated_length": 1151.0, - "completions/mean_length": 213.9375, - "completions/mean_terminated_length": 213.9375, + "completions/max_length": 800.0, + "completions/max_terminated_length": 800.0, + "completions/mean_length": 302.90625, + "completions/mean_terminated_length": 302.90625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.17484768042049836, + "entropy": 0.13853387208655477, "epoch": 0.00037, "frac_reward_zero_std": 0.0, - "grad_norm": 0.26422277092933655, - "kl": 1.4826668202877045, + "grad_norm": 0.35072028636932373, + "kl": 2.474678736180067, "learning_rate": 9.999736485702831e-05, - "loss": -0.005, - "num_tokens": 724933.0, - "reward": 0.7631042003631592, - "reward_std": 0.6272491216659546, - "rewards/rollout_reward_func/mean": 0.7631042003631592, - "rewards/rollout_reward_func/std": 0.6655160188674927, - "sampling/importance_sampling_ratio/max": 1.4438666105270386, - "sampling/importance_sampling_ratio/mean": 0.9936277270317078, - "sampling/importance_sampling_ratio/min": 0.41797348856925964, - "sampling/sampling_logp_difference/max": 0.7619318962097168, - "sampling/sampling_logp_difference/mean": 0.03127425163984299, + "loss": 0.0054, + "num_tokens": 744387.0, + "reward": 0.7394791841506958, + "reward_std": 0.5725609064102173, + "rewards/rollout_reward_func/mean": 0.7394791841506958, + "rewards/rollout_reward_func/std": 0.5643346905708313, + "sampling/importance_sampling_ratio/max": 2.0006418228149414, + "sampling/importance_sampling_ratio/mean": 0.9308086633682251, + "sampling/importance_sampling_ratio/min": 0.3774648904800415, + "sampling/sampling_logp_difference/max": 0.973365068435669, + "sampling/sampling_logp_difference/mean": 0.07168807089328766, "step": 37, - "step_time": 9.695621070000243 + "step_time": 8.840384572999938 }, { - "clip_ratio/high_max": 0.033928572200238705, - "clip_ratio/high_mean": 0.016964286100119352, - "clip_ratio/low_mean": 0.02187500079162419, + "clip_ratio/high_max": 0.07083333469927311, + "clip_ratio/high_mean": 0.04062500037252903, + "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.03883928689174354, - "entropy": 0.15785920085181715, + "clip_ratio/region_mean": 0.051041667349636555, + "entropy": 0.14499974623322487, "epoch": 0.00038, - "grad_norm": 0.16681881248950958, - "kl": 1.522850140929222, + "grad_norm": 0.4481445848941803, + "kl": 2.4228835767135024, "learning_rate": 9.998945979845876e-05, - "loss": -0.0062, + "loss": 0.0064, "step": 38, - "step_time": 6.750669727000513 + "step_time": 6.192694740999286 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.016666667070239782, "completions/clipped_ratio": 0.0, - "completions/max_length": 1154.0, - "completions/max_terminated_length": 1154.0, - "completions/mean_length": 231.53125, - "completions/mean_terminated_length": 231.53125, + "completions/max_length": 819.0, + "completions/max_terminated_length": 819.0, + "completions/mean_length": 202.6875, + "completions/mean_terminated_length": 202.6875, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.11877059687685687, + "entropy": 0.11037659808062017, "epoch": 0.00039, "frac_reward_zero_std": 0.0, - "grad_norm": 0.24095091223716736, - "kl": 1.5396657940000296, + "grad_norm": 0.43659037351608276, + "kl": 3.106099121272564, "learning_rate": 9.997628593527586e-05, - "loss": -0.0042, - "num_tokens": 762128.0, - "reward": 0.7163541316986084, - "reward_std": 0.6136023998260498, - "rewards/rollout_reward_func/mean": 0.7163541316986084, - "rewards/rollout_reward_func/std": 0.610213577747345, - "sampling/importance_sampling_ratio/max": 1.3014845848083496, - "sampling/importance_sampling_ratio/mean": 1.0244951248168945, - "sampling/importance_sampling_ratio/min": 0.889356791973114, - "sampling/sampling_logp_difference/max": 0.2697678804397583, - "sampling/sampling_logp_difference/mean": 0.01579492911696434, + "loss": -0.0117, + "num_tokens": 780659.0, + "reward": 0.8005208373069763, + "reward_std": 0.5633570551872253, + "rewards/rollout_reward_func/mean": 0.8005208373069763, + "rewards/rollout_reward_func/std": 0.5414054989814758, + "sampling/importance_sampling_ratio/max": 1.4577741622924805, + "sampling/importance_sampling_ratio/mean": 1.0116441249847412, + "sampling/importance_sampling_ratio/min": 0.5580413341522217, + "sampling/sampling_logp_difference/max": 0.5817064642906189, + "sampling/sampling_logp_difference/mean": 0.02959112450480461, "step": 39, - "step_time": 9.640163768999855 + "step_time": 8.845459910000045 }, { - "clip_ratio/high_max": 0.01785714365541935, - "clip_ratio/high_mean": 0.008928571827709675, - "clip_ratio/low_mean": 0.018750000279396772, + "clip_ratio/high_max": 0.010416666977107525, + "clip_ratio/high_mean": 0.0052083334885537624, + "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.027678572107106447, - "entropy": 0.10194598157977453, + "clip_ratio/region_mean": 0.01145833358168602, + "entropy": 0.09388253488577902, "epoch": 0.0004, - "grad_norm": 0.06580612063407898, - "kl": 1.5702023096382618, + "grad_norm": 0.09148859232664108, + "kl": 3.296179547905922, "learning_rate": 9.995784511894694e-05, - "loss": -0.006, + "loss": -0.0116, "step": 40, - "step_time": 6.767393307999555 + "step_time": 6.22067503000153 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.004464285913854837, + "clip_ratio/high_max": 0.012500000186264515, + "clip_ratio/high_mean": 0.0062500000931322575, + "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.004464285913854837, + "clip_ratio/region_mean": 0.01145833358168602, "completions/clipped_ratio": 0.0, - "completions/max_length": 1234.0, - "completions/max_terminated_length": 1234.0, - "completions/mean_length": 262.125, - "completions/mean_terminated_length": 262.125, + "completions/max_length": 792.0, + "completions/max_terminated_length": 792.0, + "completions/mean_length": 266.15625, + "completions/mean_terminated_length": 266.15625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.11627826229232596, + "entropy": 0.11680553923361003, "epoch": 0.00041, "frac_reward_zero_std": 0.0, - "grad_norm": 0.1414710283279419, - "kl": 1.8666857853531837, + "grad_norm": 0.2674156725406647, + "kl": 4.4445560202002525, "learning_rate": 9.993413994116206e-05, - "loss": -0.0112, - "num_tokens": 800926.0, - "reward": 0.6219791769981384, - "reward_std": 0.6369723677635193, - "rewards/rollout_reward_func/mean": 0.6219791769981384, - "rewards/rollout_reward_func/std": 0.7292099595069885, - "sampling/importance_sampling_ratio/max": 1.110335350036621, - "sampling/importance_sampling_ratio/mean": 0.9453052282333374, - "sampling/importance_sampling_ratio/min": 0.4698891043663025, - "sampling/sampling_logp_difference/max": 0.7473087310791016, - "sampling/sampling_logp_difference/mean": 0.026049025356769562, + "loss": -0.0101, + "num_tokens": 819586.0, + "reward": 0.6338750123977661, + "reward_std": 0.6909469366073608, + "rewards/rollout_reward_func/mean": 0.6338750123977661, + "rewards/rollout_reward_func/std": 0.6954667568206787, + "sampling/importance_sampling_ratio/max": 1.8524731397628784, + "sampling/importance_sampling_ratio/mean": 0.9724992513656616, + "sampling/importance_sampling_ratio/min": 0.5049148201942444, + "sampling/sampling_logp_difference/max": 0.5966453552246094, + "sampling/sampling_logp_difference/mean": 0.029140068218111992, "step": 41, - "step_time": 10.046313624000959 + "step_time": 8.712896530001217 }, { - "clip_ratio/high_max": 0.012500000186264515, - "clip_ratio/high_mean": 0.0062500000931322575, - "clip_ratio/low_mean": 0.0062500000931322575, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.012500000186264515, - "entropy": 0.11148970657086466, + "clip_ratio/high_max": 0.02291666716337204, + "clip_ratio/high_mean": 0.01145833358168602, + "clip_ratio/low_mean": 0.02083333395421505, + "clip_ratio/low_min": 0.010416666977107525, + "clip_ratio/region_mean": 0.03229166753590107, + "entropy": 0.13011209177784622, "epoch": 0.00042, - "grad_norm": 0.18441708385944366, - "kl": 1.8012165799736977, + "grad_norm": 0.2672519385814667, + "kl": 4.70941611379385, "learning_rate": 9.990517373346957e-05, - "loss": -0.0113, + "loss": -0.0098, "step": 42, - "step_time": 6.999212858999272 + "step_time": 6.221397426999829 }, { - "clip_ratio/high_max": 0.008928571827709675, - "clip_ratio/high_mean": 0.004464285913854837, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.004464285913854837, + "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, - "completions/max_length": 1055.0, - "completions/max_terminated_length": 1055.0, - "completions/mean_length": 225.75, - "completions/mean_terminated_length": 225.75, + "completions/max_length": 801.0, + "completions/max_terminated_length": 801.0, + "completions/mean_length": 245.0625, + "completions/mean_terminated_length": 245.0625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.07133681878985954, + "entropy": 0.13342926558107138, "epoch": 0.00043, "frac_reward_zero_std": 0.0, - "grad_norm": 0.13256920874118805, - "kl": 1.7951918244361877, + "grad_norm": 0.16639478504657745, + "kl": 3.947339281439781, "learning_rate": 9.98709505668081e-05, - "loss": -0.0056, - "num_tokens": 838155.0, - "reward": 0.7849375009536743, - "reward_std": 0.504135251045227, - "rewards/rollout_reward_func/mean": 0.7849375009536743, - "rewards/rollout_reward_func/std": 0.5278446674346924, - "sampling/importance_sampling_ratio/max": 1.3283132314682007, - "sampling/importance_sampling_ratio/mean": 1.0156052112579346, - "sampling/importance_sampling_ratio/min": 0.6983067393302917, - "sampling/sampling_logp_difference/max": 0.3479170799255371, - "sampling/sampling_logp_difference/mean": 0.013128189370036125, + "loss": -0.0047, + "num_tokens": 857433.0, + "reward": 0.8313541412353516, + "reward_std": 0.5034892559051514, + "rewards/rollout_reward_func/mean": 0.8313541412353516, + "rewards/rollout_reward_func/std": 0.5211474299430847, + "sampling/importance_sampling_ratio/max": 1.58366858959198, + "sampling/importance_sampling_ratio/mean": 0.9993729591369629, + "sampling/importance_sampling_ratio/min": 0.7741520404815674, + "sampling/sampling_logp_difference/max": 0.457059383392334, + "sampling/sampling_logp_difference/mean": 0.015617373399436474, "step": 43, - "step_time": 9.243569922999086 + "step_time": 8.727938867999455 }, { - "clip_ratio/high_max": 0.037500000558793545, - "clip_ratio/high_mean": 0.018750000279396772, - "clip_ratio/low_mean": 0.018750000279396772, + "clip_ratio/high_max": 0.08541666902601719, + "clip_ratio/high_mean": 0.042708334513008595, + "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.037500000558793545, - "entropy": 0.0734066000659368, + "clip_ratio/region_mean": 0.05312500149011612, + "entropy": 0.18604831025004387, "epoch": 0.00044, - "grad_norm": 0.13717395067214966, - "kl": 1.8053691610693932, + "grad_norm": 0.09616024792194366, + "kl": 3.85291139036417, "learning_rate": 9.983147525093428e-05, - "loss": -0.0058, + "loss": -0.0046, "step": 44, - "step_time": 6.4607698830004665 + "step_time": 6.159271418000571 }, { "clip_ratio/high_max": 0.0, @@ -1072,46 +1072,46 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1415.0, - "completions/max_terminated_length": 1415.0, - "completions/mean_length": 238.96875, - "completions/mean_terminated_length": 238.96875, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.09801992701250128, + "completions/max_length": 801.0, + "completions/max_terminated_length": 801.0, + "completions/mean_length": 267.59375, + "completions/mean_terminated_length": 267.59375, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.20075256517156959, "epoch": 0.00045, "frac_reward_zero_std": 0.0, - "grad_norm": 0.1328868269920349, - "kl": 1.706927303224802, + "grad_norm": 0.15878058969974518, + "kl": 3.450070746243, "learning_rate": 9.978675333374685e-05, - "loss": 0.0016, - "num_tokens": 876303.0, - "reward": 0.7598124742507935, - "reward_std": 0.5679430365562439, - "rewards/rollout_reward_func/mean": 0.7598124742507935, - "rewards/rollout_reward_func/std": 0.5967590808868408, - "sampling/importance_sampling_ratio/max": 1.169681429862976, - "sampling/importance_sampling_ratio/mean": 1.0279597043991089, - "sampling/importance_sampling_ratio/min": 0.8635076880455017, - "sampling/sampling_logp_difference/max": 0.1565859317779541, - "sampling/sampling_logp_difference/mean": 0.012495058588683605, + "loss": 0.0031, + "num_tokens": 896497.0, + "reward": 0.7743124961853027, + "reward_std": 0.5807892084121704, + "rewards/rollout_reward_func/mean": 0.7743124961853027, + "rewards/rollout_reward_func/std": 0.6093071103096008, + "sampling/importance_sampling_ratio/max": 1.3486193418502808, + "sampling/importance_sampling_ratio/mean": 1.006744384765625, + "sampling/importance_sampling_ratio/min": 0.6235369443893433, + "sampling/sampling_logp_difference/max": 0.4660367965698242, + "sampling/sampling_logp_difference/mean": 0.02144547551870346, "step": 45, - "step_time": 10.574091848000535 + "step_time": 8.816122870000072 }, { - "clip_ratio/high_max": 0.05000000074505806, - "clip_ratio/high_mean": 0.02500000037252903, - "clip_ratio/low_mean": 0.004464285913854837, + "clip_ratio/high_max": 0.08125000167638063, + "clip_ratio/high_mean": 0.046875000931322575, + "clip_ratio/low_mean": 0.06250000186264515, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.029464286286383867, - "entropy": 0.10025507608952466, + "clip_ratio/region_mean": 0.1093750037252903, + "entropy": 0.23521287087351084, "epoch": 0.00046, - "grad_norm": 0.06521525233983994, - "kl": 1.6916628777980804, + "grad_norm": 0.20591767132282257, + "kl": 3.419850878417492, "learning_rate": 9.973679110050689e-05, - "loss": 0.001, + "loss": 0.0024, "step": 46, - "step_time": 6.771763879000218 + "step_time": 6.180123464000189 }, { "clip_ratio/high_max": 0.0, @@ -1120,46 +1120,46 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1134.0, - "completions/max_terminated_length": 1134.0, - "completions/mean_length": 202.84375, - "completions/mean_terminated_length": 202.84375, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.0900510510073218, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 215.96875, + "completions/mean_terminated_length": 215.96875, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.2327630678191781, "epoch": 0.00047, "frac_reward_zero_std": 0.0, - "grad_norm": 0.14434900879859924, - "kl": 1.4058816023170948, + "grad_norm": 0.20975789427757263, + "kl": 2.0128674022853374, "learning_rate": 9.968159557295458e-05, - "loss": -0.006, - "num_tokens": 911967.0, - "reward": 0.8627499938011169, - "reward_std": 0.5587868690490723, - "rewards/rollout_reward_func/mean": 0.8627499938011169, - "rewards/rollout_reward_func/std": 0.5566409230232239, - "sampling/importance_sampling_ratio/max": 1.1530331373214722, - "sampling/importance_sampling_ratio/mean": 0.9977372288703918, - "sampling/importance_sampling_ratio/min": 0.5316535234451294, - "sampling/sampling_logp_difference/max": 0.6474550366401672, - "sampling/sampling_logp_difference/mean": 0.016066744923591614, + "loss": 0.008, + "num_tokens": 932581.0, + "reward": 0.9114375114440918, + "reward_std": 0.48371943831443787, + "rewards/rollout_reward_func/mean": 0.9114375114440918, + "rewards/rollout_reward_func/std": 0.48230862617492676, + "sampling/importance_sampling_ratio/max": 1.3701509237289429, + "sampling/importance_sampling_ratio/mean": 1.0438060760498047, + "sampling/importance_sampling_ratio/min": 0.866945207118988, + "sampling/sampling_logp_difference/max": 0.3101191520690918, + "sampling/sampling_logp_difference/mean": 0.02023971453309059, "step": 47, - "step_time": 9.447806593999758 + "step_time": 8.821047410000574 }, { - "clip_ratio/high_max": 0.02142857201397419, - "clip_ratio/high_mean": 0.010714286006987095, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.054166668094694614, + "clip_ratio/high_mean": 0.03229166753590107, + "clip_ratio/low_mean": 0.026041667442768812, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.010714286006987095, - "entropy": 0.09766281770862406, + "clip_ratio/region_mean": 0.05833333497866988, + "entropy": 0.25101289339363575, "epoch": 0.00048, - "grad_norm": 0.06667943298816681, - "kl": 1.405099406838417, + "grad_norm": 0.16837728023529053, + "kl": 2.0083239786326885, "learning_rate": 9.962117450832225e-05, - "loss": -0.0064, + "loss": 0.0069, "step": 48, - "step_time": 6.173912093000354 + "step_time": 6.240356067999073 }, { "clip_ratio/high_max": 0.0, @@ -1168,94 +1168,94 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 658.0, - "completions/max_terminated_length": 658.0, - "completions/mean_length": 215.1875, - "completions/mean_terminated_length": 215.1875, + "completions/max_length": 819.0, + "completions/max_terminated_length": 819.0, + "completions/mean_length": 281.53125, + "completions/mean_terminated_length": 281.53125, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.09605835549155017, + "entropy": 0.28111050836741924, "epoch": 0.00049, "frac_reward_zero_std": 0.0, - "grad_norm": 0.06864096969366074, - "kl": 1.3723388984799385, + "grad_norm": 0.16421398520469666, + "kl": 2.73032259196043, "learning_rate": 9.955553639824423e-05, - "loss": -0.0005, - "num_tokens": 948707.0, - "reward": 0.893958330154419, - "reward_std": 0.47342604398727417, - "rewards/rollout_reward_func/mean": 0.893958330154419, - "rewards/rollout_reward_func/std": 0.47255393862724304, - "sampling/importance_sampling_ratio/max": 1.1744683980941772, - "sampling/importance_sampling_ratio/mean": 0.9887692928314209, - "sampling/importance_sampling_ratio/min": 0.89034104347229, - "sampling/sampling_logp_difference/max": 0.16428709030151367, - "sampling/sampling_logp_difference/mean": 0.010529636405408382, + "loss": 0.0048, + "num_tokens": 971444.0, + "reward": 0.8419374823570251, + "reward_std": 0.5259429216384888, + "rewards/rollout_reward_func/mean": 0.8419374823570251, + "rewards/rollout_reward_func/std": 0.5223779082298279, + "sampling/importance_sampling_ratio/max": 1.170990228652954, + "sampling/importance_sampling_ratio/mean": 0.975490927696228, + "sampling/importance_sampling_ratio/min": 0.775715172290802, + "sampling/sampling_logp_difference/max": 0.2961348295211792, + "sampling/sampling_logp_difference/mean": 0.023050270974636078, "step": 49, - "step_time": 9.043114574000356 + "step_time": 8.83726181899965 }, { - "clip_ratio/high_max": 0.02500000037252903, - "clip_ratio/high_mean": 0.012500000186264515, - "clip_ratio/low_mean": 0.016666667070239782, + "clip_ratio/high_max": 0.058333334513008595, + "clip_ratio/high_mean": 0.029166667256504297, + "clip_ratio/low_mean": 0.03750000102445483, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.029166667256504297, - "entropy": 0.1075448725168826, + "clip_ratio/region_mean": 0.06666666828095913, + "entropy": 0.29891621693968773, "epoch": 0.0005, - "grad_norm": 0.0960911437869072, - "kl": 1.4063957203179598, + "grad_norm": 0.09293846040964127, + "kl": 2.6823730021715164, "learning_rate": 9.948469046756344e-05, - "loss": -0.0009, + "loss": 0.0047, "step": 50, - "step_time": 5.642086785000174 + "step_time": 5.749452210999607 }, { - "clip_ratio/high_max": 0.008928571827709675, - "clip_ratio/high_mean": 0.004464285913854837, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.004464285913854837, + "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, - "completions/max_length": 1071.0, - "completions/max_terminated_length": 1071.0, - "completions/mean_length": 316.09375, - "completions/mean_terminated_length": 316.09375, + "completions/max_length": 755.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 204.34375, + "completions/mean_terminated_length": 204.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.17863493447657675, + "entropy": 0.3413989609107375, "epoch": 0.00051, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6474497318267822, - "kl": 2.192594848573208, + "grad_norm": 0.18468794226646423, + "kl": 1.3214638219214976, "learning_rate": 9.940864667303489e-05, - "loss": 0.0143, - "num_tokens": 989035.0, - "reward": 0.5325833559036255, - "reward_std": 0.8769392967224121, - "rewards/rollout_reward_func/mean": 0.5325833559036255, - "rewards/rollout_reward_func/std": 0.853682279586792, - "sampling/importance_sampling_ratio/max": 1.9499620199203491, - "sampling/importance_sampling_ratio/mean": 0.9530384540557861, - "sampling/importance_sampling_ratio/min": 0.4134676158428192, - "sampling/sampling_logp_difference/max": 0.9089558124542236, - "sampling/sampling_logp_difference/mean": 0.05595976859331131, + "loss": -0.0125, + "num_tokens": 1008196.0, + "reward": 0.807812511920929, + "reward_std": 0.5802521705627441, + "rewards/rollout_reward_func/mean": 0.807812511920929, + "rewards/rollout_reward_func/std": 0.6089832782745361, + "sampling/importance_sampling_ratio/max": 1.5968269109725952, + "sampling/importance_sampling_ratio/mean": 1.017876148223877, + "sampling/importance_sampling_ratio/min": 0.86873459815979, + "sampling/sampling_logp_difference/max": 0.24793577194213867, + "sampling/sampling_logp_difference/mean": 0.02016655169427395, "step": 51, - "step_time": 9.98290348799992 + "step_time": 9.055620564999572 }, { - "clip_ratio/high_max": 0.04642857238650322, - "clip_ratio/high_mean": 0.02321428619325161, - "clip_ratio/low_mean": 0.0703125, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.09352678572759032, - "entropy": 0.14376915276807267, + "clip_ratio/high_max": 0.08333333488553762, + "clip_ratio/high_mean": 0.04791666800156236, + "clip_ratio/low_mean": 0.04687500139698386, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.0947916698642075, + "entropy": 0.3256998546421528, "epoch": 0.00052, - "grad_norm": 10.807016372680664, - "kl": 35.954969733953476, + "grad_norm": 0.16428031027317047, + "kl": 1.3160697827115655, "learning_rate": 9.932741570192633e-05, - "loss": 0.0328, + "loss": -0.0141, "step": 52, - "step_time": 5.674436524999692 + "step_time": 5.678324642999996 }, { "clip_ratio/high_max": 0.0, @@ -1264,94 +1264,94 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1646.0, - "completions/max_terminated_length": 1646.0, - "completions/mean_length": 292.21875, - "completions/mean_terminated_length": 292.21875, - "completions/min_length": 3.0, - "completions/min_terminated_length": 3.0, - "entropy": 0.11845442898629699, + "completions/max_length": 782.0, + "completions/max_terminated_length": 782.0, + "completions/mean_length": 272.8125, + "completions/mean_terminated_length": 272.8125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 0.3858976732008159, "epoch": 0.00053, "frac_reward_zero_std": 0.0, - "grad_norm": 0.2344297617673874, - "kl": 1.753089528530836, + "grad_norm": 0.3984723687171936, + "kl": 1.563775647431612, "learning_rate": 9.924100897051629e-05, - "loss": -0.011, - "num_tokens": 1028265.0, - "reward": 0.9336875081062317, - "reward_std": 0.40108171105384827, - "rewards/rollout_reward_func/mean": 0.9336875081062317, - "rewards/rollout_reward_func/std": 0.44726231694221497, - "sampling/importance_sampling_ratio/max": 1.4263041019439697, - "sampling/importance_sampling_ratio/mean": 1.0030502080917358, - "sampling/importance_sampling_ratio/min": 0.7242881655693054, - "sampling/sampling_logp_difference/max": 0.3650219440460205, - "sampling/sampling_logp_difference/mean": 0.015114575624465942, + "loss": -0.0032, + "num_tokens": 1046805.0, + "reward": 0.609981894493103, + "reward_std": 0.7691247463226318, + "rewards/rollout_reward_func/mean": 0.609981894493103, + "rewards/rollout_reward_func/std": 0.7861785292625427, + "sampling/importance_sampling_ratio/max": 1.9431627988815308, + "sampling/importance_sampling_ratio/mean": 1.0144963264465332, + "sampling/importance_sampling_ratio/min": 0.6214936375617981, + "sampling/sampling_logp_difference/max": 0.620997428894043, + "sampling/sampling_logp_difference/mean": 0.039432793855667114, "step": 53, - "step_time": 11.967371391000597 + "step_time": 9.164593179999429 }, { - "clip_ratio/high_max": 0.028373016510158777, - "clip_ratio/high_mean": 0.014186508255079389, - "clip_ratio/low_mean": 0.05000000121071935, - "clip_ratio/low_min": 0.012500000186264515, - "clip_ratio/region_mean": 0.06418650993146002, - "entropy": 0.18176122102886438, + "clip_ratio/high_max": 0.10833333525806665, + "clip_ratio/high_mean": 0.05416666762903333, + "clip_ratio/low_mean": 0.12916666828095913, + "clip_ratio/low_min": 0.02500000037252903, + "clip_ratio/region_mean": 0.18333333544433117, + "entropy": 0.32949152449145913, "epoch": 0.00054, - "grad_norm": 0.0920347049832344, - "kl": 1.7955459654331207, + "grad_norm": 0.07369755953550339, + "kl": 1.7530988827347755, "learning_rate": 9.914943862248966e-05, - "loss": -0.0119, + "loss": -0.0072, "step": 54, - "step_time": 6.745521994000001 + "step_time": 5.2231750020005165 }, { - "clip_ratio/high_max": 0.029761905781924725, - "clip_ratio/high_mean": 0.014880952890962362, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.014880952890962362, + "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1255.0, - "completions/max_terminated_length": 1255.0, - "completions/mean_length": 414.40625, - "completions/mean_terminated_length": 414.40625, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 293.25, + "completions/mean_terminated_length": 293.25, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, - "entropy": 0.21612564753741026, + "entropy": 0.21693987015169114, "epoch": 0.00055, "frac_reward_zero_std": 0.0, - "grad_norm": 0.6154258847236633, - "kl": 2.4827291294932365, + "grad_norm": 0.17684517800807953, + "kl": 1.3970188610255718, "learning_rate": 9.905271752723088e-05, - "loss": -0.0024, - "num_tokens": 1070717.0, - "reward": 1.012783169746399, - "reward_std": 0.3933926522731781, - "rewards/rollout_reward_func/mean": 1.012783169746399, - "rewards/rollout_reward_func/std": 0.45623427629470825, - "sampling/importance_sampling_ratio/max": 1.7324354648590088, - "sampling/importance_sampling_ratio/mean": 0.9738206267356873, - "sampling/importance_sampling_ratio/min": 0.5882648825645447, - "sampling/sampling_logp_difference/max": 0.5556355714797974, - "sampling/sampling_logp_difference/mean": 0.04333949089050293, + "loss": 0.0015, + "num_tokens": 1085380.0, + "reward": 0.9380208253860474, + "reward_std": 0.4689776301383972, + "rewards/rollout_reward_func/mean": 0.9380208253860474, + "rewards/rollout_reward_func/std": 0.4569273591041565, + "sampling/importance_sampling_ratio/max": 1.0717848539352417, + "sampling/importance_sampling_ratio/mean": 0.9742586612701416, + "sampling/importance_sampling_ratio/min": 0.8290437459945679, + "sampling/sampling_logp_difference/max": 0.22691869735717773, + "sampling/sampling_logp_difference/mean": 0.015425942838191986, "step": 55, - "step_time": 11.013223535000407 + "step_time": 9.6691150559991 }, { - "clip_ratio/high_max": 0.1761904815211892, - "clip_ratio/high_mean": 0.11964286165311933, - "clip_ratio/low_mean": 0.014062500093132257, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.13370536174625158, - "entropy": 0.17086607962846756, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.04583333432674408, + "clip_ratio/low_min": 0.02083333395421505, + "clip_ratio/region_mean": 0.04583333432674408, + "entropy": 0.2014944675611332, "epoch": 0.00056, - "grad_norm": 0.2525999844074249, - "kl": 2.5824186354875565, + "grad_norm": 0.10307826846837997, + "kl": 1.5808675736188889, "learning_rate": 9.895085927801542e-05, - "loss": -0.0014, + "loss": 0.0005, "step": 56, - "step_time": 6.016107872999555 + "step_time": 5.263838866998867 }, { "clip_ratio/high_max": 0.0, @@ -1360,190 +1360,190 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1386.0, - "completions/max_terminated_length": 1386.0, - "completions/mean_length": 284.8125, - "completions/mean_terminated_length": 284.8125, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.18887321837246418, + "completions/max_length": 746.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 342.0, + "completions/mean_terminated_length": 342.0, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.22305068839341402, "epoch": 0.00057, "frac_reward_zero_std": 0.0, - "grad_norm": 0.2054257094860077, - "kl": 1.618610456585884, + "grad_norm": 0.3665313422679901, + "kl": 1.3419982944615185, "learning_rate": 9.884387819009922e-05, - "loss": -0.0089, - "num_tokens": 1110225.0, - "reward": 0.6539791822433472, - "reward_std": 0.5930131673812866, - "rewards/rollout_reward_func/mean": 0.6539791822433472, - "rewards/rollout_reward_func/std": 0.6302005052566528, - "sampling/importance_sampling_ratio/max": 1.8165205717086792, - "sampling/importance_sampling_ratio/mean": 1.0121564865112305, - "sampling/importance_sampling_ratio/min": 0.865464985370636, - "sampling/sampling_logp_difference/max": 0.5969423055648804, - "sampling/sampling_logp_difference/mean": 0.021497800946235657, + "loss": 0.0065, + "num_tokens": 1126718.0, + "reward": 0.7988040447235107, + "reward_std": 0.6602399945259094, + "rewards/rollout_reward_func/mean": 0.7988040447235107, + "rewards/rollout_reward_func/std": 0.6664161682128906, + "sampling/importance_sampling_ratio/max": 1.6880199909210205, + "sampling/importance_sampling_ratio/mean": 1.0258538722991943, + "sampling/importance_sampling_ratio/min": 0.7378169298171997, + "sampling/sampling_logp_difference/max": 0.5609288215637207, + "sampling/sampling_logp_difference/mean": 0.027436792850494385, "step": 57, - "step_time": 11.156232571999908 + "step_time": 9.696612389999245 }, { - "clip_ratio/high_max": 0.058333334513008595, - "clip_ratio/high_mean": 0.029166667256504297, - "clip_ratio/low_mean": 0.021875000093132257, + "clip_ratio/high_max": 0.035416667349636555, + "clip_ratio/high_mean": 0.017708333674818277, + "clip_ratio/low_mean": 0.021875000558793545, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.051041667349636555, - "entropy": 0.18203126941807568, + "clip_ratio/region_mean": 0.03958333423361182, + "entropy": 0.201448664534837, "epoch": 0.00058, - "grad_norm": 0.21828053891658783, - "kl": 1.6002055294811726, + "grad_norm": 0.18699781596660614, + "kl": 1.287380250170827, "learning_rate": 9.873178929870695e-05, - "loss": -0.0086, + "loss": 0.0055, "step": 58, - "step_time": 6.244077477000019 + "step_time": 5.188791715999741 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0078125, + "clip_ratio/high_max": 0.012500000186264515, + "clip_ratio/high_mean": 0.0062500000931322575, + "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0078125, + "clip_ratio/region_mean": 0.0062500000931322575, "completions/clipped_ratio": 0.0, - "completions/max_length": 1694.0, - "completions/max_terminated_length": 1694.0, - "completions/mean_length": 245.75, - "completions/mean_terminated_length": 245.75, + "completions/max_length": 800.0, + "completions/max_terminated_length": 800.0, + "completions/mean_length": 309.75, + "completions/mean_terminated_length": 309.75, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.20505834033247083, + "entropy": 0.29074796379791223, "epoch": 0.00059, "frac_reward_zero_std": 0.0, - "grad_norm": 0.239760160446167, - "kl": 2.8409063071012497, + "grad_norm": 0.3904944062232971, + "kl": 1.6346165742725134, "learning_rate": 9.86146083569188e-05, - "loss": -0.0085, - "num_tokens": 1147717.0, - "reward": 0.6759583353996277, - "reward_std": 0.7167876958847046, - "rewards/rollout_reward_func/mean": 0.6759583353996277, - "rewards/rollout_reward_func/std": 0.8124518990516663, - "sampling/importance_sampling_ratio/max": 2.0188424587249756, - "sampling/importance_sampling_ratio/mean": 1.0246764421463013, - "sampling/importance_sampling_ratio/min": 0.8060900568962097, - "sampling/sampling_logp_difference/max": 0.4889869689941406, - "sampling/sampling_logp_difference/mean": 0.015371443703770638, + "loss": -0.009, + "num_tokens": 1166258.0, + "reward": 0.6937859058380127, + "reward_std": 0.7218152284622192, + "rewards/rollout_reward_func/mean": 0.6937859058380127, + "rewards/rollout_reward_func/std": 0.7825222611427307, + "sampling/importance_sampling_ratio/max": 1.169219970703125, + "sampling/importance_sampling_ratio/mean": 0.976312518119812, + "sampling/importance_sampling_ratio/min": 0.5550835132598877, + "sampling/sampling_logp_difference/max": 0.8605556488037109, + "sampling/sampling_logp_difference/mean": 0.0237528458237648, "step": 59, - "step_time": 12.236902825000243 + "step_time": 9.720285908001188 }, { - "clip_ratio/high_max": 0.015873016323894262, - "clip_ratio/high_mean": 0.007936508161947131, - "clip_ratio/low_mean": 0.024479167070239782, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.032415675232186913, - "entropy": 0.2697412967681885, + "clip_ratio/high_max": 0.08125000074505806, + "clip_ratio/high_mean": 0.05312500009313226, + "clip_ratio/low_mean": 0.08125000074505806, + "clip_ratio/low_min": 0.02500000037252903, + "clip_ratio/region_mean": 0.13437500037252903, + "entropy": 0.2845853242870362, "epoch": 0.0006, - "grad_norm": 1.4479070901870728, - "kl": 1.8084059804677963, + "grad_norm": 0.22719421982765198, + "kl": 1.5133962389081717, "learning_rate": 9.84923518334567e-05, - "loss": 0.0064, + "loss": -0.0116, "step": 60, - "step_time": 6.916969224000468 + "step_time": 5.277798285999324 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, - "completions/max_length": 1141.0, - "completions/max_terminated_length": 1141.0, - "completions/mean_length": 306.1875, - "completions/mean_terminated_length": 306.1875, + "completions/max_length": 746.0, + "completions/max_terminated_length": 746.0, + "completions/mean_length": 270.875, + "completions/mean_terminated_length": 270.875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.2432498256675899, + "entropy": 0.26288399629993364, "epoch": 0.00061, "frac_reward_zero_std": 0.0, - "grad_norm": 0.15688766539096832, - "kl": 1.928163781762123, + "grad_norm": 0.34440797567367554, + "kl": 1.6623874194920063, "learning_rate": 9.83650369103696e-05, - "loss": -0.0154, - "num_tokens": 1187828.0, - "reward": 0.7319375276565552, - "reward_std": 0.5858461856842041, - "rewards/rollout_reward_func/mean": 0.7319375276565552, - "rewards/rollout_reward_func/std": 0.6664270162582397, - "sampling/importance_sampling_ratio/max": 1.3562289476394653, - "sampling/importance_sampling_ratio/mean": 0.9950809478759766, - "sampling/importance_sampling_ratio/min": 0.8087555766105652, - "sampling/sampling_logp_difference/max": 0.3163447380065918, - "sampling/sampling_logp_difference/mean": 0.0226910300552845, + "loss": -0.0132, + "num_tokens": 1205239.0, + "reward": 0.6897708177566528, + "reward_std": 0.7638728618621826, + "rewards/rollout_reward_func/mean": 0.6897708177566528, + "rewards/rollout_reward_func/std": 0.7458044290542603, + "sampling/importance_sampling_ratio/max": 1.2595757246017456, + "sampling/importance_sampling_ratio/mean": 0.981465220451355, + "sampling/importance_sampling_ratio/min": 0.6430744528770447, + "sampling/sampling_logp_difference/max": 0.2841062545776367, + "sampling/sampling_logp_difference/mean": 0.018771031871438026, "step": 61, - "step_time": 10.608102406999706 + "step_time": 9.608220718999746 }, { - "clip_ratio/high_max": 0.030357143841683865, - "clip_ratio/high_mean": 0.015178571920841932, - "clip_ratio/low_mean": 0.010416666977107525, + "clip_ratio/high_max": 0.037500000558793545, + "clip_ratio/high_mean": 0.018750000279396772, + "clip_ratio/low_mean": 0.07395833404734731, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.025595238897949457, - "entropy": 0.22057522786781192, + "clip_ratio/region_mean": 0.09270833432674408, + "entropy": 0.21695934707895503, "epoch": 0.00062, - "grad_norm": 0.11316227912902832, - "kl": 1.9541875012218952, + "grad_norm": 0.2984854280948639, + "kl": 1.5429048892110586, "learning_rate": 9.823268148061883e-05, - "loss": -0.0145, + "loss": -0.0125, "step": 62, - "step_time": 5.81027040199956 + "step_time": 5.2136180739994415 }, { - "clip_ratio/high_max": 0.008928571827709675, - "clip_ratio/high_mean": 0.004464285913854837, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.0052083334885537624, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.004464285913854837, + "clip_ratio/region_mean": 0.015625000465661287, "completions/clipped_ratio": 0.0, - "completions/max_length": 850.0, - "completions/max_terminated_length": 850.0, - "completions/mean_length": 260.40625, - "completions/mean_terminated_length": 260.40625, - "completions/min_length": 3.0, - "completions/min_terminated_length": 3.0, - "entropy": 0.175975039601326, + "completions/max_length": 755.0, + "completions/max_terminated_length": 755.0, + "completions/mean_length": 296.25, + "completions/mean_terminated_length": 296.25, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "entropy": 0.1781742997773108, "epoch": 0.00063, "frac_reward_zero_std": 0.0, - "grad_norm": 0.23777881264686584, - "kl": 1.8976656533777714, + "grad_norm": 0.2683662474155426, + "kl": 1.438841413706541, "learning_rate": 9.809530414556335e-05, - "loss": -0.0073, - "num_tokens": 1226242.0, - "reward": 0.9314790964126587, - "reward_std": 0.4598495364189148, - "rewards/rollout_reward_func/mean": 0.9314790964126587, - "rewards/rollout_reward_func/std": 0.4464755654335022, - "sampling/importance_sampling_ratio/max": 1.409760594367981, - "sampling/importance_sampling_ratio/mean": 1.0067412853240967, - "sampling/importance_sampling_ratio/min": 0.8432673811912537, - "sampling/sampling_logp_difference/max": 0.34805774688720703, - "sampling/sampling_logp_difference/mean": 0.015504386276006699, + "loss": 0.0022, + "num_tokens": 1244800.0, + "reward": 0.8090833425521851, + "reward_std": 0.6406168937683105, + "rewards/rollout_reward_func/mean": 0.8090833425521851, + "rewards/rollout_reward_func/std": 0.6507377028465271, + "sampling/importance_sampling_ratio/max": 1.294932246208191, + "sampling/importance_sampling_ratio/mean": 0.9803900718688965, + "sampling/importance_sampling_ratio/min": 0.5894244313240051, + "sampling/sampling_logp_difference/max": 0.5294467210769653, + "sampling/sampling_logp_difference/mean": 0.021016977727413177, "step": 63, - "step_time": 9.866925459000186 + "step_time": 9.60672285999999 }, { - "clip_ratio/high_max": 0.058928572572767735, - "clip_ratio/high_mean": 0.029464286286383867, - "clip_ratio/low_mean": 0.031250000931322575, + "clip_ratio/high_max": 0.010416666977107525, + "clip_ratio/high_mean": 0.0052083334885537624, + "clip_ratio/low_mean": 0.02083333395421505, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.06071428768336773, - "entropy": 0.21677518263459206, + "clip_ratio/region_mean": 0.026041667442768812, + "entropy": 0.1766265498008579, "epoch": 0.00064, - "grad_norm": 0.10355119407176971, - "kl": 1.9398510716855526, + "grad_norm": 0.11552369594573975, + "kl": 1.4128315187990665, "learning_rate": 9.79529242123455e-05, - "loss": -0.0093, + "loss": 0.0011, "step": 64, - "step_time": 5.271362073999853 + "step_time": 5.218333942000754 }, { "clip_ratio/high_max": 0.0, @@ -1552,276 +1552,276 @@ "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, - "completions/max_length": 1041.0, - "completions/max_terminated_length": 1041.0, - "completions/mean_length": 201.59375, - "completions/mean_terminated_length": 201.59375, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.21963271405547857, + "completions/max_length": 736.0, + "completions/max_terminated_length": 736.0, + "completions/mean_length": 225.34375, + "completions/mean_terminated_length": 225.34375, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.11140737855384941, "epoch": 0.00065, "frac_reward_zero_std": 0.0, - "grad_norm": 0.5803110599517822, - "kl": 3.1230736933648586, + "grad_norm": 0.24003885686397552, + "kl": 1.1965708532370627, "learning_rate": 9.780556169117757e-05, - "loss": -0.0078, - "num_tokens": 1260829.0, - "reward": 0.6616458296775818, - "reward_std": 0.7855631113052368, - "rewards/rollout_reward_func/mean": 0.6616458296775818, - "rewards/rollout_reward_func/std": 0.8183318972587585, - "sampling/importance_sampling_ratio/max": 1.4000798463821411, - "sampling/importance_sampling_ratio/mean": 1.0056772232055664, - "sampling/importance_sampling_ratio/min": 0.71177077293396, - "sampling/sampling_logp_difference/max": 0.34459519386291504, - "sampling/sampling_logp_difference/mean": 0.03253237158060074, + "loss": 0.0126, + "num_tokens": 1280147.0, + "reward": 0.9366666078567505, + "reward_std": 0.43621310591697693, + "rewards/rollout_reward_func/mean": 0.9366666078567505, + "rewards/rollout_reward_func/std": 0.44008782505989075, + "sampling/importance_sampling_ratio/max": 1.020021677017212, + "sampling/importance_sampling_ratio/mean": 0.9703360199928284, + "sampling/importance_sampling_ratio/min": 0.8434741497039795, + "sampling/sampling_logp_difference/max": 0.15884017944335938, + "sampling/sampling_logp_difference/mean": 0.009324941784143448, "step": 65, - "step_time": 10.17715724000027 + "step_time": 9.508668929000578 }, { - "clip_ratio/high_max": 0.11666666995733976, - "clip_ratio/high_mean": 0.062797621358186, - "clip_ratio/low_mean": 0.078125, + "clip_ratio/high_max": 0.008928571827709675, + "clip_ratio/high_mean": 0.004464285913854837, + "clip_ratio/low_mean": 0.016964286100119352, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.14092262228950858, - "entropy": 0.1336969246622175, + "clip_ratio/region_mean": 0.02142857201397419, + "entropy": 0.10581687675585272, "epoch": 0.00066, - "grad_norm": 0.13620424270629883, - "kl": 2.933967538177967, + "grad_norm": 0.08845307677984238, + "kl": 1.2180560289416462, "learning_rate": 9.765323729252955e-05, - "loss": -0.0085, + "loss": 0.0114, "step": 66, - "step_time": 5.5793444789999285 + "step_time": 5.201323110999965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/low_mean": 0.010416666977107525, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.010416666977107525, "completions/clipped_ratio": 0.0, - "completions/max_length": 668.0, - "completions/max_terminated_length": 668.0, - "completions/mean_length": 276.25, - "completions/mean_terminated_length": 276.25, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.16010587196797132, + "completions/max_length": 773.0, + "completions/max_terminated_length": 773.0, + "completions/mean_length": 351.53125, + "completions/mean_terminated_length": 351.53125, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.23107421830354724, "epoch": 0.00067, "frac_reward_zero_std": 0.0, - "grad_norm": 0.12310951948165894, - "kl": 2.098513074219227, + "grad_norm": 0.46222177147865295, + "kl": 1.481215626001358, "learning_rate": 9.749597242421838e-05, - "loss": -0.0122, - "num_tokens": 1300882.0, - "reward": 0.8329999446868896, - "reward_std": 0.5346106290817261, - "rewards/rollout_reward_func/mean": 0.8329999446868896, - "rewards/rollout_reward_func/std": 0.6208124756813049, - "sampling/importance_sampling_ratio/max": 1.1149344444274902, - "sampling/importance_sampling_ratio/mean": 0.9865162372589111, - "sampling/importance_sampling_ratio/min": 0.8121993541717529, - "sampling/sampling_logp_difference/max": 0.21269893646240234, - "sampling/sampling_logp_difference/mean": 0.015977365896105766, + "loss": -0.0006, + "num_tokens": 1322609.0, + "reward": 0.8559374809265137, + "reward_std": 0.6327424049377441, + "rewards/rollout_reward_func/mean": 0.8559374809265137, + "rewards/rollout_reward_func/std": 0.6359191536903381, + "sampling/importance_sampling_ratio/max": 1.5288397073745728, + "sampling/importance_sampling_ratio/mean": 0.9566798210144043, + "sampling/importance_sampling_ratio/min": 0.3740919530391693, + "sampling/sampling_logp_difference/max": 0.5567009449005127, + "sampling/sampling_logp_difference/mean": 0.036363814026117325, "step": 67, - "step_time": 9.141569508000657 + "step_time": 9.655177893000655 }, { - "clip_ratio/high_max": 0.012500000186264515, - "clip_ratio/high_mean": 0.0062500000931322575, - "clip_ratio/low_mean": 0.012500000186264515, + "clip_ratio/high_max": 0.02083333395421505, + "clip_ratio/high_mean": 0.010416666977107525, + "clip_ratio/low_mean": 0.05781250027939677, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.018750000279396772, - "entropy": 0.1498410563217476, + "clip_ratio/region_mean": 0.0682291672565043, + "entropy": 0.24178626375214662, "epoch": 0.00068, - "grad_norm": 0.1016835868358612, - "kl": 2.0821792110800743, + "grad_norm": 0.5767381191253662, + "kl": 3.13630897551775, "learning_rate": 9.733378918839942e-05, - "loss": -0.0124, + "loss": 0.0022, "step": 68, - "step_time": 5.649315163999745 + "step_time": 5.224315012000716 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.010416666977107525, + "clip_ratio/high_mean": 0.0052083334885537624, + "clip_ratio/low_mean": 0.015625, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.020833333488553762, "completions/clipped_ratio": 0.0, - "completions/max_length": 1055.0, - "completions/max_terminated_length": 1055.0, - "completions/mean_length": 288.59375, - "completions/mean_terminated_length": 288.59375, + "completions/max_length": 773.0, + "completions/max_terminated_length": 773.0, + "completions/mean_length": 330.3125, + "completions/mean_terminated_length": 330.3125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.14323098736349493, + "entropy": 0.21850440152047668, "epoch": 0.00069, "frac_reward_zero_std": 0.0, - "grad_norm": 0.11429130285978317, - "kl": 2.039843175560236, + "grad_norm": 0.5954594612121582, + "kl": 1.2401557313278317, "learning_rate": 9.716671037846007e-05, - "loss": -0.0046, - "num_tokens": 1340730.0, - "reward": 0.7588750123977661, - "reward_std": 0.48252782225608826, - "rewards/rollout_reward_func/mean": 0.7588750123977661, - "rewards/rollout_reward_func/std": 0.5914071798324585, - "sampling/importance_sampling_ratio/max": 1.140119194984436, - "sampling/importance_sampling_ratio/mean": 0.9898408651351929, - "sampling/importance_sampling_ratio/min": 0.8956636190414429, - "sampling/sampling_logp_difference/max": 0.13106560707092285, - "sampling/sampling_logp_difference/mean": 0.01080230250954628, + "loss": 0.0312, + "num_tokens": 1363792.0, + "reward": 0.7480416297912598, + "reward_std": 0.5194458365440369, + "rewards/rollout_reward_func/mean": 0.7480416297912598, + "rewards/rollout_reward_func/std": 0.7188701629638672, + "sampling/importance_sampling_ratio/max": 2.412290573120117, + "sampling/importance_sampling_ratio/mean": 1.0539028644561768, + "sampling/importance_sampling_ratio/min": 0.7076537609100342, + "sampling/sampling_logp_difference/max": 0.4878692626953125, + "sampling/sampling_logp_difference/mean": 0.02761038765311241, "step": 69, - "step_time": 9.683018002999916 + "step_time": 9.267615106999983 }, { - "clip_ratio/high_max": 0.07500000111758709, - "clip_ratio/high_mean": 0.04821428656578064, - "clip_ratio/low_mean": 0.021875000093132257, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0700892866589129, - "entropy": 0.1461901287548244, + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.07336309598758817, + "clip_ratio/low_min": 0.01875000074505806, + "clip_ratio/region_mean": 0.07336309598758817, + "entropy": 0.16505320186843164, "epoch": 0.0007, - "grad_norm": 0.1435869336128235, - "kl": 1.9891464561223984, + "grad_norm": 0.4004910886287689, + "kl": 2.1750029074028134, "learning_rate": 9.699475947581644e-05, - "loss": -0.0043, + "loss": 0.0282, "step": 70, - "step_time": 6.030160345999548 + "step_time": 5.664059530000031 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, + "clip_ratio/high_max": 0.0416666679084301, + "clip_ratio/high_mean": 0.02083333395421505, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.02083333395421505, "completions/clipped_ratio": 0.0, - "completions/max_length": 1151.0, - "completions/max_terminated_length": 1151.0, - "completions/mean_length": 270.71875, - "completions/mean_terminated_length": 270.71875, - "completions/min_length": 2.0, - "completions/min_terminated_length": 2.0, - "entropy": 0.15293755863967817, + "completions/max_length": 809.0, + "completions/max_terminated_length": 809.0, + "completions/mean_length": 310.9375, + "completions/mean_terminated_length": 310.9375, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "entropy": 0.16944719175808132, "epoch": 0.00071, "frac_reward_zero_std": 0.0, - "grad_norm": 0.3050353527069092, - "kl": 1.9519358221441507, + "grad_norm": 0.4207824170589447, + "kl": 1.7246448742225766, "learning_rate": 9.681796064661319e-05, - "loss": -0.0008, - "num_tokens": 1379557.0, - "reward": 0.61760413646698, - "reward_std": 0.6345539093017578, - "rewards/rollout_reward_func/mean": 0.61760413646698, - "rewards/rollout_reward_func/std": 0.6240441203117371, - "sampling/importance_sampling_ratio/max": 1.1850391626358032, - "sampling/importance_sampling_ratio/mean": 1.007583737373352, - "sampling/importance_sampling_ratio/min": 0.869886577129364, - "sampling/sampling_logp_difference/max": 0.16973721981048584, - "sampling/sampling_logp_difference/mean": 0.008598901331424713, + "loss": -0.0407, + "num_tokens": 1403906.0, + "reward": 0.27929550409317017, + "reward_std": 0.8404286503791809, + "rewards/rollout_reward_func/mean": 0.27929550409317017, + "rewards/rollout_reward_func/std": 0.8851102590560913, + "sampling/importance_sampling_ratio/max": 1.6808106899261475, + "sampling/importance_sampling_ratio/mean": 0.9820831418037415, + "sampling/importance_sampling_ratio/min": 0.21298153698444366, + "sampling/sampling_logp_difference/max": 1.5466530323028564, + "sampling/sampling_logp_difference/mean": 0.03731834143400192, "step": 71, - "step_time": 10.008160938000401 + "step_time": 8.840721922999819 }, { - "clip_ratio/high_max": 0.029761905781924725, - "clip_ratio/high_mean": 0.014880952890962362, - "clip_ratio/low_mean": 0.021875000093132257, - "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.036755953915417194, - "entropy": 0.13349127455148846, + "clip_ratio/high_max": 0.07648809649981558, + "clip_ratio/high_mean": 0.03824404824990779, + "clip_ratio/low_mean": 0.088020836468786, + "clip_ratio/low_min": 0.02500000037252903, + "clip_ratio/region_mean": 0.12626488565001637, + "entropy": 0.15499942232418107, "epoch": 0.00072, - "grad_norm": 0.11685528606176376, - "kl": 1.9348566476255655, + "grad_norm": 0.1864785999059677, + "kl": 1.339592988602817, "learning_rate": 9.663633873832725e-05, - "loss": -0.0011, + "loss": -0.0408, "step": 72, - "step_time": 6.208596755999906 + "step_time": 6.153265552999983 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, + "clip_ratio/high_max": 0.008928571827709675, + "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, - "completions/max_length": 723.0, - "completions/max_terminated_length": 723.0, - "completions/mean_length": 207.84375, - "completions/mean_terminated_length": 207.84375, + "completions/max_length": 791.0, + "completions/max_terminated_length": 791.0, + "completions/mean_length": 284.34375, + "completions/mean_terminated_length": 284.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.14575498150952626, + "entropy": 0.11808319464034867, "epoch": 0.00073, "frac_reward_zero_std": 0.0, - "grad_norm": 0.1255907416343689, - "kl": 1.1826815884560347, + "grad_norm": 0.272882878780365, + "kl": 0.7641631324077025, "learning_rate": 9.644991927627566e-05, - "loss": -0.006, - "num_tokens": 1416475.0, - "reward": 0.7622500061988831, - "reward_std": 0.5831079483032227, - "rewards/rollout_reward_func/mean": 0.7622500061988831, - "rewards/rollout_reward_func/std": 0.6012318730354309, - "sampling/importance_sampling_ratio/max": 1.1591823101043701, - "sampling/importance_sampling_ratio/mean": 0.98829185962677, - "sampling/importance_sampling_ratio/min": 0.9088780879974365, - "sampling/sampling_logp_difference/max": 0.14767801761627197, - "sampling/sampling_logp_difference/mean": 0.007826489396393299, + "loss": -0.0236, + "num_tokens": 1443272.0, + "reward": 0.8665000200271606, + "reward_std": 0.5289559364318848, + "rewards/rollout_reward_func/mean": 0.8665000200271606, + "rewards/rollout_reward_func/std": 0.6465467810630798, + "sampling/importance_sampling_ratio/max": 1.6293666362762451, + "sampling/importance_sampling_ratio/mean": 1.039879560470581, + "sampling/importance_sampling_ratio/min": 0.9476312398910522, + "sampling/sampling_logp_difference/max": 0.35202598571777344, + "sampling/sampling_logp_difference/mean": 0.015152488835155964, "step": 73, - "step_time": 9.11005223400025 + "step_time": 8.778943822000656 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, - "clip_ratio/low_mean": 0.0, + "clip_ratio/high_max": 0.0591517873108387, + "clip_ratio/high_mean": 0.02957589365541935, + "clip_ratio/low_mean": 0.0062500000931322575, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, - "entropy": 0.1513779065135168, + "clip_ratio/region_mean": 0.03582589374855161, + "entropy": 0.16239579749526456, "epoch": 0.00074, - "grad_norm": 0.12437106668949127, - "kl": 1.1780924815684557, + "grad_norm": 0.12349029630422592, + "kl": 0.8285429883981124, "learning_rate": 9.625872846002834e-05, - "loss": -0.0066, + "loss": -0.0246, "step": 74, - "step_time": 5.63687633499967 + "step_time": 6.164997446000598 }, { - "clip_ratio/high_max": 0.0, - "clip_ratio/high_mean": 0.0, + "clip_ratio/high_max": 0.008928571827709675, + "clip_ratio/high_mean": 0.004464285913854837, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, - "clip_ratio/region_mean": 0.0, + "clip_ratio/region_mean": 0.004464285913854837, "completions/clipped_ratio": 0.0, - "completions/max_length": 705.0, - "completions/max_terminated_length": 705.0, - "completions/mean_length": 281.375, - "completions/mean_terminated_length": 281.375, + "completions/max_length": 793.0, + "completions/max_terminated_length": 793.0, + "completions/mean_length": 340.34375, + "completions/mean_terminated_length": 340.34375, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, - "entropy": 0.18086719792336226, + "entropy": 0.2151845816988498, "epoch": 0.00075, "frac_reward_zero_std": 0.0, - "grad_norm": 0.15126265585422516, - "kl": 1.6214838400483131, + "grad_norm": 0.3326566219329834, + "kl": 1.1586173835676163, "learning_rate": 9.606279315972582e-05, - "loss": 0.002, - "num_tokens": 1455741.0, - "reward": 0.6960416436195374, - "reward_std": 0.6535277366638184, - "rewards/rollout_reward_func/mean": 0.6960416436195374, - "rewards/rollout_reward_func/std": 0.6603111624717712, - "sampling/importance_sampling_ratio/max": 1.1913020610809326, - "sampling/importance_sampling_ratio/mean": 0.9901882410049438, - "sampling/importance_sampling_ratio/min": 0.8861557245254517, - "sampling/sampling_logp_difference/max": 0.17504119873046875, - "sampling/sampling_logp_difference/mean": 0.008757364936172962, + "loss": 0.0356, + "num_tokens": 1484425.0, + "reward": 0.6145625114440918, + "reward_std": 0.8387740850448608, + "rewards/rollout_reward_func/mean": 0.6145625114440918, + "rewards/rollout_reward_func/std": 0.824505627155304, + "sampling/importance_sampling_ratio/max": 1.5896179676055908, + "sampling/importance_sampling_ratio/mean": 0.9876473546028137, + "sampling/importance_sampling_ratio/min": 0.5594227910041809, + "sampling/sampling_logp_difference/max": 0.6865577697753906, + "sampling/sampling_logp_difference/mean": 0.032433029264211655, "step": 75, - "step_time": 8.987358285000255 + "step_time": 8.926528610999867 } ], "logging_steps": 1.0, "max_steps": 300, - "num_input_tokens_seen": 1455741, + "num_input_tokens_seen": 1484425, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": {