| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.8, | |
| "eval_steps": 500, | |
| "global_step": 400, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6791666865348815, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.0, | |
| "completions/mean_length": 236.5416717529297, | |
| "completions/mean_terminated_length": 200.61068420410157, | |
| "completions/min_length": 127.2, | |
| "completions/min_terminated_length": 127.2, | |
| "entropy": 1.070425021648407, | |
| "epoch": 0.01, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.5985119342803955, | |
| "kl": 0.0035211448557674885, | |
| "learning_rate": 4.980000000000001e-06, | |
| "loss": 0.0011, | |
| "num_tokens": 73826.0, | |
| "reward": 0.2776068687438965, | |
| "reward_std": 0.5232907950878143, | |
| "rewards/politeness_reward_func/mean": 0.27760685607790947, | |
| "rewards/politeness_reward_func/std": 0.5940477013587951, | |
| "step": 5, | |
| "step_time": 11.145096338540316 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6208333492279052, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.8, | |
| "completions/mean_length": 232.80417175292968, | |
| "completions/mean_terminated_length": 204.57913513183593, | |
| "completions/min_length": 148.6, | |
| "completions/min_terminated_length": 148.6, | |
| "entropy": 1.070945155620575, | |
| "epoch": 0.02, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3468329906463623, | |
| "kl": 0.02206921111792326, | |
| "learning_rate": 4.955e-06, | |
| "loss": 0.0224, | |
| "num_tokens": 147283.0, | |
| "reward": 0.6233123302459717, | |
| "reward_std": 0.4648557245731354, | |
| "rewards/politeness_reward_func/mean": 0.6233122944831848, | |
| "rewards/politeness_reward_func/std": 0.553675526380539, | |
| "step": 10, | |
| "step_time": 10.835208699852227 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5000000119209289, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.6, | |
| "completions/mean_length": 221.8000030517578, | |
| "completions/mean_terminated_length": 188.9316619873047, | |
| "completions/min_length": 110.8, | |
| "completions/min_terminated_length": 110.8, | |
| "entropy": 1.1163462162017823, | |
| "epoch": 0.03, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3379204273223877, | |
| "kl": 0.04084419272840023, | |
| "learning_rate": 4.93e-06, | |
| "loss": -0.0052, | |
| "num_tokens": 218131.0, | |
| "reward": 0.8122925400733948, | |
| "reward_std": 0.41203114986419676, | |
| "rewards/politeness_reward_func/mean": 0.8122925400733948, | |
| "rewards/politeness_reward_func/std": 0.4989388585090637, | |
| "step": 15, | |
| "step_time": 10.806195054203272 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6791666865348815, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 251.6, | |
| "completions/mean_length": 238.19584045410156, | |
| "completions/mean_terminated_length": 201.80428771972657, | |
| "completions/min_length": 130.2, | |
| "completions/min_terminated_length": 130.2, | |
| "entropy": 1.1251786351203918, | |
| "epoch": 0.04, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3837928771972656, | |
| "kl": 0.0385904410853982, | |
| "learning_rate": 4.9050000000000005e-06, | |
| "loss": -0.0031, | |
| "num_tokens": 292306.0, | |
| "reward": 0.6734941720962524, | |
| "reward_std": 0.448628181219101, | |
| "rewards/politeness_reward_func/mean": 0.6734941720962524, | |
| "rewards/politeness_reward_func/std": 0.509108817577362, | |
| "step": 20, | |
| "step_time": 10.808321141451597 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7333333551883697, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.6, | |
| "completions/mean_length": 239.7041748046875, | |
| "completions/mean_terminated_length": 201.94395141601564, | |
| "completions/min_length": 144.0, | |
| "completions/min_terminated_length": 144.0, | |
| "entropy": 1.0987902224063872, | |
| "epoch": 0.05, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3908112049102783, | |
| "kl": 0.04714170750230551, | |
| "learning_rate": 4.880000000000001e-06, | |
| "loss": 0.013, | |
| "num_tokens": 366731.0, | |
| "reward": 0.7238378047943115, | |
| "reward_std": 0.5012153267860413, | |
| "rewards/politeness_reward_func/mean": 0.7238377809524537, | |
| "rewards/politeness_reward_func/std": 0.5635832965373992, | |
| "step": 25, | |
| "step_time": 10.778460966795683 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7041666984558106, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 251.4, | |
| "completions/mean_length": 238.33334045410157, | |
| "completions/mean_terminated_length": 209.13802490234374, | |
| "completions/min_length": 135.6, | |
| "completions/min_terminated_length": 135.6, | |
| "entropy": 1.0942281484603882, | |
| "epoch": 0.06, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.272942304611206, | |
| "kl": 0.04730561040341854, | |
| "learning_rate": 4.855e-06, | |
| "loss": -0.0003, | |
| "num_tokens": 441211.0, | |
| "reward": 0.7387123763561249, | |
| "reward_std": 0.3938853979110718, | |
| "rewards/politeness_reward_func/mean": 0.7387123882770539, | |
| "rewards/politeness_reward_func/std": 0.5152743279933929, | |
| "step": 30, | |
| "step_time": 10.791689745336772 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6000000178813935, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.8, | |
| "completions/mean_length": 232.95417175292968, | |
| "completions/mean_terminated_length": 200.11821594238282, | |
| "completions/min_length": 126.0, | |
| "completions/min_terminated_length": 126.0, | |
| "entropy": 1.1196449041366576, | |
| "epoch": 0.07, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.515143394470215, | |
| "kl": 0.055315449833869934, | |
| "learning_rate": 4.83e-06, | |
| "loss": 0.0018, | |
| "num_tokens": 514144.0, | |
| "reward": 0.9808868050575257, | |
| "reward_std": 0.3419617712497711, | |
| "rewards/politeness_reward_func/mean": 0.9808867692947387, | |
| "rewards/politeness_reward_func/std": 0.4772939085960388, | |
| "step": 35, | |
| "step_time": 10.767037071287632 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.4333333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 253.2, | |
| "completions/mean_length": 216.89583740234374, | |
| "completions/mean_terminated_length": 186.88653869628905, | |
| "completions/min_length": 90.4, | |
| "completions/min_terminated_length": 90.4, | |
| "entropy": 1.062067198753357, | |
| "epoch": 0.08, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.2335519790649414, | |
| "kl": 0.06222313120961189, | |
| "learning_rate": 4.805000000000001e-06, | |
| "loss": 0.0425, | |
| "num_tokens": 583095.0, | |
| "reward": 0.873325777053833, | |
| "reward_std": 0.4448536515235901, | |
| "rewards/politeness_reward_func/mean": 0.873325777053833, | |
| "rewards/politeness_reward_func/std": 0.49484293460845946, | |
| "step": 40, | |
| "step_time": 10.696332201361656 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5666666746139526, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.6, | |
| "completions/mean_length": 220.82500610351562, | |
| "completions/mean_terminated_length": 175.39512634277344, | |
| "completions/min_length": 75.6, | |
| "completions/min_terminated_length": 75.6, | |
| "entropy": 1.0480997025966645, | |
| "epoch": 0.09, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.452233076095581, | |
| "kl": 0.07400040999054909, | |
| "learning_rate": 4.78e-06, | |
| "loss": 0.0107, | |
| "num_tokens": 653373.0, | |
| "reward": 0.9472709178924561, | |
| "reward_std": 0.40665341913700104, | |
| "rewards/politeness_reward_func/mean": 0.9472708940505982, | |
| "rewards/politeness_reward_func/std": 0.4510787308216095, | |
| "step": 45, | |
| "step_time": 10.80831560343504 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5416666865348816, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.8, | |
| "completions/mean_length": 208.1916717529297, | |
| "completions/mean_terminated_length": 153.92529907226563, | |
| "completions/min_length": 61.8, | |
| "completions/min_terminated_length": 61.8, | |
| "entropy": 1.1236772537231445, | |
| "epoch": 0.1, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.160369634628296, | |
| "kl": 0.08055943846702576, | |
| "learning_rate": 4.755e-06, | |
| "loss": 0.0611, | |
| "num_tokens": 720619.0, | |
| "reward": 0.8770254850387573, | |
| "reward_std": 0.5482801616191864, | |
| "rewards/politeness_reward_func/mean": 0.8770254492759705, | |
| "rewards/politeness_reward_func/std": 0.5923300921916962, | |
| "step": 50, | |
| "step_time": 10.79710350483656 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.4083333373069763, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 245.4, | |
| "completions/mean_length": 180.80833740234374, | |
| "completions/mean_terminated_length": 134.77063598632813, | |
| "completions/min_length": 46.0, | |
| "completions/min_terminated_length": 46.0, | |
| "entropy": 1.2555986642837524, | |
| "epoch": 0.11, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.3433427810668945, | |
| "kl": 0.09562707468867301, | |
| "learning_rate": 4.7300000000000005e-06, | |
| "loss": 0.0773, | |
| "num_tokens": 781261.0, | |
| "reward": 1.092347764968872, | |
| "reward_std": 0.4649295687675476, | |
| "rewards/politeness_reward_func/mean": 1.0923476934432983, | |
| "rewards/politeness_reward_func/std": 0.5502241551876068, | |
| "step": 55, | |
| "step_time": 10.756779878586531 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2875000063329935, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 238.8, | |
| "completions/mean_length": 154.1916702270508, | |
| "completions/mean_terminated_length": 119.38919677734376, | |
| "completions/min_length": 42.8, | |
| "completions/min_terminated_length": 42.8, | |
| "entropy": 1.3319262266159058, | |
| "epoch": 0.12, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3366644382476807, | |
| "kl": 0.14686549603939056, | |
| "learning_rate": 4.705e-06, | |
| "loss": 0.0456, | |
| "num_tokens": 835835.0, | |
| "reward": 1.4423248767852783, | |
| "reward_std": 0.36013287007808686, | |
| "rewards/politeness_reward_func/mean": 1.4423248767852783, | |
| "rewards/politeness_reward_func/std": 0.40978127121925356, | |
| "step": 60, | |
| "step_time": 10.825969719141721 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.41250001192092894, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.4, | |
| "completions/mean_length": 183.07083740234376, | |
| "completions/mean_terminated_length": 133.48724670410155, | |
| "completions/min_length": 46.8, | |
| "completions/min_terminated_length": 46.8, | |
| "entropy": 1.3182121157646178, | |
| "epoch": 0.13, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 25.555892944335938, | |
| "kl": 0.17301606237888337, | |
| "learning_rate": 4.680000000000001e-06, | |
| "loss": 0.0985, | |
| "num_tokens": 896732.0, | |
| "reward": 1.3051372289657592, | |
| "reward_std": 0.43976340293884275, | |
| "rewards/politeness_reward_func/mean": 1.3051372170448303, | |
| "rewards/politeness_reward_func/std": 0.5006832242012024, | |
| "step": 65, | |
| "step_time": 10.645895641297102 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2708333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.4, | |
| "completions/mean_length": 157.8791717529297, | |
| "completions/mean_terminated_length": 123.51410827636718, | |
| "completions/min_length": 41.4, | |
| "completions/min_terminated_length": 41.4, | |
| "entropy": 1.2897038578987121, | |
| "epoch": 0.14, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.8760225772857666, | |
| "kl": 0.19847530573606492, | |
| "learning_rate": 4.655e-06, | |
| "loss": 0.1906, | |
| "num_tokens": 951999.0, | |
| "reward": 1.420637583732605, | |
| "reward_std": 0.430054372549057, | |
| "rewards/politeness_reward_func/mean": 1.420637583732605, | |
| "rewards/politeness_reward_func/std": 0.48181390166282656, | |
| "step": 70, | |
| "step_time": 10.596996573358775 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2083333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 240.4, | |
| "completions/mean_length": 146.60000610351562, | |
| "completions/mean_terminated_length": 118.93135223388671, | |
| "completions/min_length": 43.2, | |
| "completions/min_terminated_length": 43.2, | |
| "entropy": 1.4299420952796935, | |
| "epoch": 0.15, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.174029350280762, | |
| "kl": 0.20222627967596055, | |
| "learning_rate": 4.6300000000000006e-06, | |
| "loss": 0.1187, | |
| "num_tokens": 1004287.0, | |
| "reward": 1.5081899881362915, | |
| "reward_std": 0.33264915347099305, | |
| "rewards/politeness_reward_func/mean": 1.508189868927002, | |
| "rewards/politeness_reward_func/std": 0.36398229002952576, | |
| "step": 75, | |
| "step_time": 10.52198056280613 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.12916667088866235, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 242.6, | |
| "completions/mean_length": 142.55417175292968, | |
| "completions/mean_terminated_length": 126.68900451660156, | |
| "completions/min_length": 48.2, | |
| "completions/min_terminated_length": 48.2, | |
| "entropy": 1.3346869945526123, | |
| "epoch": 0.16, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.63394021987915, | |
| "kl": 0.19363965839147568, | |
| "learning_rate": 4.605000000000001e-06, | |
| "loss": 0.0898, | |
| "num_tokens": 1056308.0, | |
| "reward": 1.459042239189148, | |
| "reward_std": 0.2736098259687424, | |
| "rewards/politeness_reward_func/mean": 1.45904221534729, | |
| "rewards/politeness_reward_func/std": 0.3277650147676468, | |
| "step": 80, | |
| "step_time": 10.517034325748682 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.20416666716337203, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 241.2, | |
| "completions/mean_length": 137.8166717529297, | |
| "completions/mean_terminated_length": 107.75043640136718, | |
| "completions/min_length": 39.2, | |
| "completions/min_terminated_length": 39.2, | |
| "entropy": 1.0585298597812653, | |
| "epoch": 0.17, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.915429592132568, | |
| "kl": 0.2862268269062042, | |
| "learning_rate": 4.58e-06, | |
| "loss": 0.2369, | |
| "num_tokens": 1106584.0, | |
| "reward": 1.339914321899414, | |
| "reward_std": 0.42014909982681276, | |
| "rewards/politeness_reward_func/mean": 1.3399142742156982, | |
| "rewards/politeness_reward_func/std": 0.46948710083961487, | |
| "step": 85, | |
| "step_time": 10.558716806769372 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.26250001005828383, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 232.6, | |
| "completions/mean_length": 154.93750457763673, | |
| "completions/mean_terminated_length": 121.31997375488281, | |
| "completions/min_length": 47.0, | |
| "completions/min_terminated_length": 47.0, | |
| "entropy": 1.208097517490387, | |
| "epoch": 0.18, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.7058539390563965, | |
| "kl": 0.17084270417690278, | |
| "learning_rate": 4.5550000000000004e-06, | |
| "loss": 0.0854, | |
| "num_tokens": 1161209.0, | |
| "reward": 1.4180654048919679, | |
| "reward_std": 0.24261603355407715, | |
| "rewards/politeness_reward_func/mean": 1.4180653095245361, | |
| "rewards/politeness_reward_func/std": 0.3126596033573151, | |
| "step": 90, | |
| "step_time": 10.573557010293007 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.15000000707805156, | |
| "completions/max_length": 245.4, | |
| "completions/max_terminated_length": 235.8, | |
| "completions/mean_length": 145.20000457763672, | |
| "completions/mean_terminated_length": 128.7294189453125, | |
| "completions/min_length": 50.8, | |
| "completions/min_terminated_length": 50.8, | |
| "entropy": 1.2446547746658325, | |
| "epoch": 0.19, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.650216579437256, | |
| "kl": 0.19710368812084197, | |
| "learning_rate": 4.530000000000001e-06, | |
| "loss": 0.1361, | |
| "num_tokens": 1213177.0, | |
| "reward": 1.3565361022949218, | |
| "reward_std": 0.3620565414428711, | |
| "rewards/politeness_reward_func/mean": 1.356536078453064, | |
| "rewards/politeness_reward_func/std": 0.4186973810195923, | |
| "step": 95, | |
| "step_time": 10.10843729674816 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.15416666865348816, | |
| "completions/max_length": 253.2, | |
| "completions/max_terminated_length": 239.6, | |
| "completions/mean_length": 127.69167022705078, | |
| "completions/mean_terminated_length": 104.76427154541015, | |
| "completions/min_length": 33.6, | |
| "completions/min_terminated_length": 33.6, | |
| "entropy": 1.21082746386528, | |
| "epoch": 0.2, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.778512477874756, | |
| "kl": 0.23306063413619996, | |
| "learning_rate": 4.505e-06, | |
| "loss": 0.1579, | |
| "num_tokens": 1261199.0, | |
| "reward": 1.4624977350234984, | |
| "reward_std": 0.27724905014038087, | |
| "rewards/politeness_reward_func/mean": 1.4624976634979248, | |
| "rewards/politeness_reward_func/std": 0.353417181968689, | |
| "step": 100, | |
| "step_time": 10.490663215517998 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2916666746139526, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 239.8, | |
| "completions/mean_length": 164.0000030517578, | |
| "completions/mean_terminated_length": 131.73128814697264, | |
| "completions/min_length": 52.0, | |
| "completions/min_terminated_length": 52.0, | |
| "entropy": 1.227693748474121, | |
| "epoch": 0.21, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.480128765106201, | |
| "kl": 0.21182419136166572, | |
| "learning_rate": 4.48e-06, | |
| "loss": 0.1385, | |
| "num_tokens": 1317887.0, | |
| "reward": 1.4506232976913451, | |
| "reward_std": 0.38838809728622437, | |
| "rewards/politeness_reward_func/mean": 1.4506232500076295, | |
| "rewards/politeness_reward_func/std": 0.415321010351181, | |
| "step": 105, | |
| "step_time": 10.568055501580238 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2875000134110451, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.6, | |
| "completions/mean_length": 164.9791717529297, | |
| "completions/mean_terminated_length": 129.34453430175782, | |
| "completions/min_length": 50.8, | |
| "completions/min_terminated_length": 50.8, | |
| "entropy": 1.2591872215270996, | |
| "epoch": 0.22, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.477797985076904, | |
| "kl": 0.21972680687904358, | |
| "learning_rate": 4.4550000000000005e-06, | |
| "loss": 0.118, | |
| "num_tokens": 1374570.0, | |
| "reward": 1.3832386493682862, | |
| "reward_std": 0.36756529808044436, | |
| "rewards/politeness_reward_func/mean": 1.3832386016845704, | |
| "rewards/politeness_reward_func/std": 0.45387519598007203, | |
| "step": 110, | |
| "step_time": 10.580036842823029 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.4041666805744171, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 239.6, | |
| "completions/mean_length": 189.48334045410155, | |
| "completions/mean_terminated_length": 144.77314147949218, | |
| "completions/min_length": 55.6, | |
| "completions/min_terminated_length": 55.6, | |
| "entropy": 1.4673572778701782, | |
| "epoch": 0.23, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.5870041847229, | |
| "kl": 0.18002651631832123, | |
| "learning_rate": 4.430000000000001e-06, | |
| "loss": 0.0902, | |
| "num_tokens": 1437278.0, | |
| "reward": 1.463021945953369, | |
| "reward_std": 0.36559332013130186, | |
| "rewards/politeness_reward_func/mean": 1.4630219221115113, | |
| "rewards/politeness_reward_func/std": 0.4403323769569397, | |
| "step": 115, | |
| "step_time": 10.606720576435327 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.29166667759418485, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.4, | |
| "completions/mean_length": 168.28750610351562, | |
| "completions/mean_terminated_length": 134.1982864379883, | |
| "completions/min_length": 45.0, | |
| "completions/min_terminated_length": 45.0, | |
| "entropy": 1.439770805835724, | |
| "epoch": 0.24, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.188452243804932, | |
| "kl": 0.23004925400018691, | |
| "learning_rate": 4.405e-06, | |
| "loss": 0.1038, | |
| "num_tokens": 1495123.0, | |
| "reward": 1.6009358167648315, | |
| "reward_std": 0.3070066928863525, | |
| "rewards/politeness_reward_func/mean": 1.6009357929229737, | |
| "rewards/politeness_reward_func/std": 0.34360362887382506, | |
| "step": 120, | |
| "step_time": 10.64953521117568 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.1833333395421505, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 243.2, | |
| "completions/mean_length": 140.8666748046875, | |
| "completions/mean_terminated_length": 116.34450073242188, | |
| "completions/min_length": 37.8, | |
| "completions/min_terminated_length": 37.8, | |
| "entropy": 1.2707997798919677, | |
| "epoch": 0.25, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.1139140129089355, | |
| "kl": 0.2768221229314804, | |
| "learning_rate": 4.38e-06, | |
| "loss": 0.146, | |
| "num_tokens": 1546755.0, | |
| "reward": 1.528564214706421, | |
| "reward_std": 0.29817952811717985, | |
| "rewards/politeness_reward_func/mean": 1.528564167022705, | |
| "rewards/politeness_reward_func/std": 0.353902006149292, | |
| "step": 125, | |
| "step_time": 10.536190098524093 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.12500000298023223, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 228.6, | |
| "completions/mean_length": 126.03333587646485, | |
| "completions/mean_terminated_length": 108.7752197265625, | |
| "completions/min_length": 37.6, | |
| "completions/min_terminated_length": 37.6, | |
| "entropy": 1.239863795042038, | |
| "epoch": 0.26, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 20.54998207092285, | |
| "kl": 0.4258181616663933, | |
| "learning_rate": 4.355000000000001e-06, | |
| "loss": 0.205, | |
| "num_tokens": 1594475.0, | |
| "reward": 1.5162237882614136, | |
| "reward_std": 0.37349834442138674, | |
| "rewards/politeness_reward_func/mean": 1.5162237882614136, | |
| "rewards/politeness_reward_func/std": 0.4025087893009186, | |
| "step": 130, | |
| "step_time": 10.561264203488827 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.1375000063329935, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 235.4, | |
| "completions/mean_length": 121.34167175292968, | |
| "completions/mean_terminated_length": 100.33185272216797, | |
| "completions/min_length": 29.0, | |
| "completions/min_terminated_length": 29.0, | |
| "entropy": 1.44633287191391, | |
| "epoch": 0.27, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.6437859535217285, | |
| "kl": 0.2876953512430191, | |
| "learning_rate": 4.33e-06, | |
| "loss": 0.1474, | |
| "num_tokens": 1640909.0, | |
| "reward": 1.6570862054824829, | |
| "reward_std": 0.35202938318252563, | |
| "rewards/politeness_reward_func/mean": 1.657086157798767, | |
| "rewards/politeness_reward_func/std": 0.4086485207080841, | |
| "step": 135, | |
| "step_time": 10.50367113724351 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.25000000596046446, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 232.4, | |
| "completions/mean_length": 165.46250610351564, | |
| "completions/mean_terminated_length": 134.87125549316406, | |
| "completions/min_length": 50.2, | |
| "completions/min_terminated_length": 50.2, | |
| "entropy": 1.5645981192588807, | |
| "epoch": 0.28, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.930655479431152, | |
| "kl": 0.20734833329916, | |
| "learning_rate": 4.305e-06, | |
| "loss": 0.1052, | |
| "num_tokens": 1698396.0, | |
| "reward": 1.5796484470367431, | |
| "reward_std": 0.39563443660736086, | |
| "rewards/politeness_reward_func/mean": 1.5796483516693116, | |
| "rewards/politeness_reward_func/std": 0.43083653450012205, | |
| "step": 140, | |
| "step_time": 10.580496142059564 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3666666731238365, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 245.4, | |
| "completions/mean_length": 182.1541717529297, | |
| "completions/mean_terminated_length": 148.0070785522461, | |
| "completions/min_length": 67.8, | |
| "completions/min_terminated_length": 67.8, | |
| "entropy": 1.4628460884094239, | |
| "epoch": 0.29, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.1963322162628174, | |
| "kl": 0.20285871326923371, | |
| "learning_rate": 4.2800000000000005e-06, | |
| "loss": 0.0054, | |
| "num_tokens": 1759217.0, | |
| "reward": 1.5254708528518677, | |
| "reward_std": 0.3578204929828644, | |
| "rewards/politeness_reward_func/mean": 1.5254708051681518, | |
| "rewards/politeness_reward_func/std": 0.39698067903518675, | |
| "step": 145, | |
| "step_time": 10.641796179115772 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.325000011920929, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 243.8, | |
| "completions/mean_length": 185.4041717529297, | |
| "completions/mean_terminated_length": 151.72132263183593, | |
| "completions/min_length": 54.2, | |
| "completions/min_terminated_length": 54.2, | |
| "entropy": 1.550300669670105, | |
| "epoch": 0.3, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.5268337726593018, | |
| "kl": 0.26886349618434907, | |
| "learning_rate": 4.255e-06, | |
| "loss": 0.0341, | |
| "num_tokens": 1821250.0, | |
| "reward": 1.5432605028152466, | |
| "reward_std": 0.35273998975753784, | |
| "rewards/politeness_reward_func/mean": 1.5432604551315308, | |
| "rewards/politeness_reward_func/std": 0.39439594745635986, | |
| "step": 150, | |
| "step_time": 10.68586125895381 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.20416667610406875, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 241.8, | |
| "completions/mean_length": 162.42083740234375, | |
| "completions/mean_terminated_length": 139.60369567871095, | |
| "completions/min_length": 58.0, | |
| "completions/min_terminated_length": 58.0, | |
| "entropy": 1.665429162979126, | |
| "epoch": 0.31, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.6844170093536377, | |
| "kl": 0.22049252390861512, | |
| "learning_rate": 4.23e-06, | |
| "loss": 0.0256, | |
| "num_tokens": 1877383.0, | |
| "reward": 1.6982928276062013, | |
| "reward_std": 0.30497619807720183, | |
| "rewards/politeness_reward_func/mean": 1.6982927560806274, | |
| "rewards/politeness_reward_func/std": 0.3323679566383362, | |
| "step": 155, | |
| "step_time": 10.646523833274841 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.12083333805203438, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 238.4, | |
| "completions/mean_length": 141.9625045776367, | |
| "completions/mean_terminated_length": 126.35040893554688, | |
| "completions/min_length": 53.0, | |
| "completions/min_terminated_length": 53.0, | |
| "entropy": 1.493414855003357, | |
| "epoch": 0.32, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.583273410797119, | |
| "kl": 0.25222020596265793, | |
| "learning_rate": 4.205e-06, | |
| "loss": 0.0511, | |
| "num_tokens": 1928318.0, | |
| "reward": 1.6409826755523682, | |
| "reward_std": 0.2959387719631195, | |
| "rewards/politeness_reward_func/mean": 1.6409826040267945, | |
| "rewards/politeness_reward_func/std": 0.3353587478399277, | |
| "step": 160, | |
| "step_time": 10.597700411826372 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.22916667088866233, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.2, | |
| "completions/mean_length": 159.16250762939453, | |
| "completions/mean_terminated_length": 130.86475067138673, | |
| "completions/min_length": 52.4, | |
| "completions/min_terminated_length": 52.4, | |
| "entropy": 1.5048229694366455, | |
| "epoch": 0.33, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 8.112273216247559, | |
| "kl": 0.26160909086465833, | |
| "learning_rate": 4.18e-06, | |
| "loss": 0.1583, | |
| "num_tokens": 1983861.0, | |
| "reward": 1.5394394874572754, | |
| "reward_std": 0.4292898178100586, | |
| "rewards/politeness_reward_func/mean": 1.5394394636154174, | |
| "rewards/politeness_reward_func/std": 0.4557326793670654, | |
| "step": 165, | |
| "step_time": 10.650840406119823 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3583333373069763, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.8, | |
| "completions/mean_length": 181.42084045410155, | |
| "completions/mean_terminated_length": 142.14824829101562, | |
| "completions/min_length": 47.6, | |
| "completions/min_terminated_length": 47.6, | |
| "entropy": 1.6774910688400269, | |
| "epoch": 0.34, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.154572486877441, | |
| "kl": 0.21350446194410325, | |
| "learning_rate": 4.155e-06, | |
| "loss": 0.0845, | |
| "num_tokens": 2044922.0, | |
| "reward": 1.617796039581299, | |
| "reward_std": 0.42567009329795835, | |
| "rewards/politeness_reward_func/mean": 1.617796039581299, | |
| "rewards/politeness_reward_func/std": 0.5039380311965942, | |
| "step": 170, | |
| "step_time": 10.696013970673084 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.21666667759418487, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 235.8, | |
| "completions/mean_length": 150.75000610351563, | |
| "completions/mean_terminated_length": 122.61416320800781, | |
| "completions/min_length": 41.4, | |
| "completions/min_terminated_length": 41.4, | |
| "entropy": 1.7118729948997498, | |
| "epoch": 0.35, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.096772193908691, | |
| "kl": 0.24443377554416656, | |
| "learning_rate": 4.13e-06, | |
| "loss": 0.062, | |
| "num_tokens": 2098654.0, | |
| "reward": 1.7112194776535035, | |
| "reward_std": 0.41009202003479006, | |
| "rewards/politeness_reward_func/mean": 1.7112194299697876, | |
| "rewards/politeness_reward_func/std": 0.46850563287734986, | |
| "step": 175, | |
| "step_time": 10.681096732616425 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.24583333544433117, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.4, | |
| "completions/mean_length": 161.39583892822264, | |
| "completions/mean_terminated_length": 134.3897247314453, | |
| "completions/min_length": 56.0, | |
| "completions/min_terminated_length": 56.0, | |
| "entropy": 1.6281691670417786, | |
| "epoch": 0.36, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 39.266761779785156, | |
| "kl": 0.3168014049530029, | |
| "learning_rate": 4.1050000000000005e-06, | |
| "loss": 0.0628, | |
| "num_tokens": 2154557.0, | |
| "reward": 1.592983651161194, | |
| "reward_std": 0.4036149501800537, | |
| "rewards/politeness_reward_func/mean": 1.5929836750030517, | |
| "rewards/politeness_reward_func/std": 0.4594584465026855, | |
| "step": 180, | |
| "step_time": 10.576303614675998 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2041666716337204, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.0, | |
| "completions/mean_length": 154.63750305175782, | |
| "completions/mean_terminated_length": 128.9908935546875, | |
| "completions/min_length": 47.2, | |
| "completions/min_terminated_length": 47.2, | |
| "entropy": 1.5905902862548829, | |
| "epoch": 0.37, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.8977794647216797, | |
| "kl": 0.2617978900671005, | |
| "learning_rate": 4.08e-06, | |
| "loss": 0.0605, | |
| "num_tokens": 2208870.0, | |
| "reward": 1.608209228515625, | |
| "reward_std": 0.40417273342609406, | |
| "rewards/politeness_reward_func/mean": 1.608209204673767, | |
| "rewards/politeness_reward_func/std": 0.44712826013565066, | |
| "step": 185, | |
| "step_time": 10.657971718162297 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.25416666865348814, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 243.6, | |
| "completions/mean_length": 167.1416748046875, | |
| "completions/mean_terminated_length": 136.75882873535156, | |
| "completions/min_length": 46.6, | |
| "completions/min_terminated_length": 46.6, | |
| "entropy": 1.76783127784729, | |
| "epoch": 0.38, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.072940826416016, | |
| "kl": 0.29045032411813737, | |
| "learning_rate": 4.055000000000001e-06, | |
| "loss": 0.0313, | |
| "num_tokens": 2266296.0, | |
| "reward": 1.7399524927139283, | |
| "reward_std": 0.4326803207397461, | |
| "rewards/politeness_reward_func/mean": 1.7399524688720702, | |
| "rewards/politeness_reward_func/std": 0.44698241353034973, | |
| "step": 190, | |
| "step_time": 10.595029101520776 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.387500011920929, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.8, | |
| "completions/mean_length": 190.1125030517578, | |
| "completions/mean_terminated_length": 150.54798278808593, | |
| "completions/min_length": 57.8, | |
| "completions/min_terminated_length": 57.8, | |
| "entropy": 1.7596600532531739, | |
| "epoch": 0.39, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.128026008605957, | |
| "kl": 0.25340984016656876, | |
| "learning_rate": 4.03e-06, | |
| "loss": 0.0384, | |
| "num_tokens": 2329619.0, | |
| "reward": 1.6809997797012328, | |
| "reward_std": 0.4062549531459808, | |
| "rewards/politeness_reward_func/mean": 1.6809997320175172, | |
| "rewards/politeness_reward_func/std": 0.44869658946990965, | |
| "step": 195, | |
| "step_time": 10.63182978257537 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.320833346247673, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.8, | |
| "completions/mean_length": 182.68334045410157, | |
| "completions/mean_terminated_length": 149.52076721191406, | |
| "completions/min_length": 60.2, | |
| "completions/min_terminated_length": 60.2, | |
| "entropy": 1.7882241249084472, | |
| "epoch": 0.4, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.352699279785156, | |
| "kl": 0.2817923933267593, | |
| "learning_rate": 4.005000000000001e-06, | |
| "loss": -0.0338, | |
| "num_tokens": 2390743.0, | |
| "reward": 1.7947330713272094, | |
| "reward_std": 0.3602103054523468, | |
| "rewards/politeness_reward_func/mean": 1.7947329998016357, | |
| "rewards/politeness_reward_func/std": 0.37249717116355896, | |
| "step": 200, | |
| "step_time": 10.771137349307537 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.279166679084301, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 251.0, | |
| "completions/mean_length": 175.2791717529297, | |
| "completions/mean_terminated_length": 146.66289672851562, | |
| "completions/min_length": 54.2, | |
| "completions/min_terminated_length": 54.2, | |
| "entropy": 1.805497646331787, | |
| "epoch": 0.41, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.236753463745117, | |
| "kl": 0.4257870987057686, | |
| "learning_rate": 3.980000000000001e-06, | |
| "loss": 0.0117, | |
| "num_tokens": 2449866.0, | |
| "reward": 1.7829206705093383, | |
| "reward_std": 0.4116291403770447, | |
| "rewards/politeness_reward_func/mean": 1.7829206705093383, | |
| "rewards/politeness_reward_func/std": 0.43859742283821107, | |
| "step": 205, | |
| "step_time": 10.78743917644024 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.254166679084301, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.2, | |
| "completions/mean_length": 171.39584045410157, | |
| "completions/mean_terminated_length": 143.78868103027344, | |
| "completions/min_length": 52.0, | |
| "completions/min_terminated_length": 52.0, | |
| "entropy": 1.6819992780685424, | |
| "epoch": 0.42, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.016842842102051, | |
| "kl": 0.2606677010655403, | |
| "learning_rate": 3.955e-06, | |
| "loss": 0.0507, | |
| "num_tokens": 2507913.0, | |
| "reward": 1.6511780500411988, | |
| "reward_std": 0.4131308376789093, | |
| "rewards/politeness_reward_func/mean": 1.651178002357483, | |
| "rewards/politeness_reward_func/std": 0.43051646947860717, | |
| "step": 210, | |
| "step_time": 10.86044239550829 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2250000059604645, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.2, | |
| "completions/mean_length": 166.20833740234374, | |
| "completions/mean_terminated_length": 139.74808349609376, | |
| "completions/min_length": 58.8, | |
| "completions/min_terminated_length": 58.8, | |
| "entropy": 1.5043580889701844, | |
| "epoch": 0.43, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.5619537830352783, | |
| "kl": 0.24042100906372071, | |
| "learning_rate": 3.9300000000000005e-06, | |
| "loss": -0.0266, | |
| "num_tokens": 2565323.0, | |
| "reward": 1.6339158296585083, | |
| "reward_std": 0.33398547768592834, | |
| "rewards/politeness_reward_func/mean": 1.6339158535003662, | |
| "rewards/politeness_reward_func/std": 0.37013601660728457, | |
| "step": 215, | |
| "step_time": 10.860479059070348 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2291666731238365, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 238.6, | |
| "completions/mean_length": 162.5666717529297, | |
| "completions/mean_terminated_length": 137.25745391845703, | |
| "completions/min_length": 52.4, | |
| "completions/min_terminated_length": 52.4, | |
| "entropy": 1.707606041431427, | |
| "epoch": 0.44, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.356007099151611, | |
| "kl": 0.275185127556324, | |
| "learning_rate": 3.905000000000001e-06, | |
| "loss": 0.0385, | |
| "num_tokens": 2621347.0, | |
| "reward": 1.7018348932266236, | |
| "reward_std": 0.3872075915336609, | |
| "rewards/politeness_reward_func/mean": 1.7018348217010497, | |
| "rewards/politeness_reward_func/std": 0.43171402215957644, | |
| "step": 220, | |
| "step_time": 10.862962251901626 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.25000000447034837, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.6, | |
| "completions/mean_length": 170.00000610351563, | |
| "completions/mean_terminated_length": 142.47680053710937, | |
| "completions/min_length": 51.4, | |
| "completions/min_terminated_length": 51.4, | |
| "entropy": 1.7443210363388062, | |
| "epoch": 0.45, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.761515140533447, | |
| "kl": 0.2994952619075775, | |
| "learning_rate": 3.88e-06, | |
| "loss": 0.0254, | |
| "num_tokens": 2679443.0, | |
| "reward": 1.6937603950500488, | |
| "reward_std": 0.42128287851810453, | |
| "rewards/politeness_reward_func/mean": 1.6937604188919066, | |
| "rewards/politeness_reward_func/std": 0.4645624727010727, | |
| "step": 225, | |
| "step_time": 10.798750822246074 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2750000089406967, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 238.4, | |
| "completions/mean_length": 171.07500305175782, | |
| "completions/mean_terminated_length": 139.3609603881836, | |
| "completions/min_length": 51.4, | |
| "completions/min_terminated_length": 51.4, | |
| "entropy": 1.7655636191368103, | |
| "epoch": 0.46, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.219749450683594, | |
| "kl": 0.30960221886634826, | |
| "learning_rate": 3.855e-06, | |
| "loss": 0.0374, | |
| "num_tokens": 2738565.0, | |
| "reward": 1.74673752784729, | |
| "reward_std": 0.49648854732513426, | |
| "rewards/politeness_reward_func/mean": 1.7467374801635742, | |
| "rewards/politeness_reward_func/std": 0.5134409368038177, | |
| "step": 230, | |
| "step_time": 10.840561749786138 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.24166667759418486, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.4, | |
| "completions/mean_length": 170.58750610351564, | |
| "completions/mean_terminated_length": 144.16094665527345, | |
| "completions/min_length": 49.6, | |
| "completions/min_terminated_length": 49.6, | |
| "entropy": 1.5835796117782592, | |
| "epoch": 0.47, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 8.280583381652832, | |
| "kl": 0.28185769021511076, | |
| "learning_rate": 3.830000000000001e-06, | |
| "loss": 0.0433, | |
| "num_tokens": 2797282.0, | |
| "reward": 1.6517542362213136, | |
| "reward_std": 0.4238882720470428, | |
| "rewards/politeness_reward_func/mean": 1.6517542362213136, | |
| "rewards/politeness_reward_func/std": 0.46576651334762575, | |
| "step": 235, | |
| "step_time": 10.934168311953545 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.1833333358168602, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.4, | |
| "completions/mean_length": 149.59166870117187, | |
| "completions/mean_terminated_length": 125.1296600341797, | |
| "completions/min_length": 40.4, | |
| "completions/min_terminated_length": 40.4, | |
| "entropy": 1.5539511442184448, | |
| "epoch": 0.48, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.9952290058135986, | |
| "kl": 0.2962296575307846, | |
| "learning_rate": 3.8050000000000004e-06, | |
| "loss": 0.0772, | |
| "num_tokens": 2850656.0, | |
| "reward": 1.5875454187393188, | |
| "reward_std": 0.33082354068756104, | |
| "rewards/politeness_reward_func/mean": 1.5875454187393188, | |
| "rewards/politeness_reward_func/std": 0.4173043370246887, | |
| "step": 240, | |
| "step_time": 10.826450176537037 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.42500001192092896, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 252.4, | |
| "completions/mean_length": 203.55417175292968, | |
| "completions/mean_terminated_length": 167.60662841796875, | |
| "completions/min_length": 68.0, | |
| "completions/min_terminated_length": 68.0, | |
| "entropy": 1.732195258140564, | |
| "epoch": 0.49, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.070302963256836, | |
| "kl": 0.2232723578810692, | |
| "learning_rate": 3.7800000000000002e-06, | |
| "loss": 0.0646, | |
| "num_tokens": 2916917.0, | |
| "reward": 1.5489000082015991, | |
| "reward_std": 0.44896683692932127, | |
| "rewards/politeness_reward_func/mean": 1.5488999366760254, | |
| "rewards/politeness_reward_func/std": 0.4965748846530914, | |
| "step": 245, | |
| "step_time": 10.93318238928914 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3375000059604645, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.6, | |
| "completions/mean_length": 175.0666717529297, | |
| "completions/mean_terminated_length": 135.0446517944336, | |
| "completions/min_length": 47.4, | |
| "completions/min_terminated_length": 47.4, | |
| "entropy": 1.7236797571182252, | |
| "epoch": 0.5, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 15.248210906982422, | |
| "kl": 0.31031118631362914, | |
| "learning_rate": 3.7550000000000005e-06, | |
| "loss": 0.0402, | |
| "num_tokens": 2976245.0, | |
| "reward": 1.6219825506210328, | |
| "reward_std": 0.4284421443939209, | |
| "rewards/politeness_reward_func/mean": 1.6219825744628906, | |
| "rewards/politeness_reward_func/std": 0.4772031903266907, | |
| "step": 250, | |
| "step_time": 10.9129977889359 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.420833358168602, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 251.8, | |
| "completions/mean_length": 198.55000915527344, | |
| "completions/mean_terminated_length": 162.55760955810547, | |
| "completions/min_length": 71.8, | |
| "completions/min_terminated_length": 71.8, | |
| "entropy": 1.9296841025352478, | |
| "epoch": 0.51, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.592284679412842, | |
| "kl": 0.25315294712781905, | |
| "learning_rate": 3.7300000000000003e-06, | |
| "loss": 0.0132, | |
| "num_tokens": 3041337.0, | |
| "reward": 1.6332777500152589, | |
| "reward_std": 0.4870866537094116, | |
| "rewards/politeness_reward_func/mean": 1.6332777261734008, | |
| "rewards/politeness_reward_func/std": 0.5504487872123718, | |
| "step": 255, | |
| "step_time": 10.853592294454575 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.32083334028720856, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.0, | |
| "completions/mean_length": 184.1916717529297, | |
| "completions/mean_terminated_length": 150.5311309814453, | |
| "completions/min_length": 62.8, | |
| "completions/min_terminated_length": 62.8, | |
| "entropy": 1.8794168829917908, | |
| "epoch": 0.52, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.218177795410156, | |
| "kl": 0.24962413907051087, | |
| "learning_rate": 3.705e-06, | |
| "loss": 0.058, | |
| "num_tokens": 3102695.0, | |
| "reward": 1.7052141666412353, | |
| "reward_std": 0.4909538745880127, | |
| "rewards/politeness_reward_func/mean": 1.7052141666412353, | |
| "rewards/politeness_reward_func/std": 0.5124517560005188, | |
| "step": 260, | |
| "step_time": 10.939885137230158 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.39166667461395266, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 240.8, | |
| "completions/mean_length": 182.05833740234374, | |
| "completions/mean_terminated_length": 135.85120544433593, | |
| "completions/min_length": 47.8, | |
| "completions/min_terminated_length": 47.8, | |
| "entropy": 1.78580139875412, | |
| "epoch": 0.53, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.726942777633667, | |
| "kl": 0.26940242052078245, | |
| "learning_rate": 3.6800000000000003e-06, | |
| "loss": 0.0129, | |
| "num_tokens": 3163621.0, | |
| "reward": 1.6984041452407836, | |
| "reward_std": 0.4623105704784393, | |
| "rewards/politeness_reward_func/mean": 1.69840407371521, | |
| "rewards/politeness_reward_func/std": 0.5121756374835968, | |
| "step": 265, | |
| "step_time": 10.816243136674165 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.37500000596046446, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 242.6, | |
| "completions/mean_length": 196.69584045410156, | |
| "completions/mean_terminated_length": 160.64813537597655, | |
| "completions/min_length": 66.4, | |
| "completions/min_terminated_length": 66.4, | |
| "entropy": 1.7563302516937256, | |
| "epoch": 0.54, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.206515312194824, | |
| "kl": 0.3476646825671196, | |
| "learning_rate": 3.655e-06, | |
| "loss": 0.0463, | |
| "num_tokens": 3227836.0, | |
| "reward": 1.7140352249145507, | |
| "reward_std": 0.41706904768943787, | |
| "rewards/politeness_reward_func/mean": 1.714035153388977, | |
| "rewards/politeness_reward_func/std": 0.4722850799560547, | |
| "step": 270, | |
| "step_time": 10.934283661842347 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.43750002086162565, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.0, | |
| "completions/mean_length": 200.75416870117186, | |
| "completions/mean_terminated_length": 159.0064697265625, | |
| "completions/min_length": 59.6, | |
| "completions/min_terminated_length": 59.6, | |
| "entropy": 1.853849744796753, | |
| "epoch": 0.55, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3859314918518066, | |
| "kl": 0.2895158067345619, | |
| "learning_rate": 3.6300000000000004e-06, | |
| "loss": -0.0048, | |
| "num_tokens": 3292993.0, | |
| "reward": 1.8271270751953126, | |
| "reward_std": 0.42741702795028685, | |
| "rewards/politeness_reward_func/mean": 1.8271270513534545, | |
| "rewards/politeness_reward_func/std": 0.47546608448028566, | |
| "step": 275, | |
| "step_time": 10.752403935790062 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.35833334624767305, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 241.8, | |
| "completions/mean_length": 189.95000305175782, | |
| "completions/mean_terminated_length": 153.42260131835937, | |
| "completions/min_length": 57.8, | |
| "completions/min_terminated_length": 57.8, | |
| "entropy": 1.8662793278694152, | |
| "epoch": 0.56, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.938138484954834, | |
| "kl": 0.29017033874988557, | |
| "learning_rate": 3.6050000000000002e-06, | |
| "loss": -0.0162, | |
| "num_tokens": 3355893.0, | |
| "reward": 1.7445276975631714, | |
| "reward_std": 0.4672766387462616, | |
| "rewards/politeness_reward_func/mean": 1.744527554512024, | |
| "rewards/politeness_reward_func/std": 0.4982309639453888, | |
| "step": 280, | |
| "step_time": 10.624229270964861 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.31250000596046446, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.4, | |
| "completions/mean_length": 190.03333740234376, | |
| "completions/mean_terminated_length": 162.05103149414063, | |
| "completions/min_length": 65.8, | |
| "completions/min_terminated_length": 65.8, | |
| "entropy": 1.74795663356781, | |
| "epoch": 0.57, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.650832176208496, | |
| "kl": 0.21563028842210769, | |
| "learning_rate": 3.58e-06, | |
| "loss": 0.0183, | |
| "num_tokens": 3418413.0, | |
| "reward": 1.6855457305908204, | |
| "reward_std": 0.3864980161190033, | |
| "rewards/politeness_reward_func/mean": 1.6855456590652467, | |
| "rewards/politeness_reward_func/std": 0.4192918837070465, | |
| "step": 285, | |
| "step_time": 10.76889206841588 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.28750001192092894, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 240.4, | |
| "completions/mean_length": 174.78333740234376, | |
| "completions/mean_terminated_length": 144.00184173583983, | |
| "completions/min_length": 54.4, | |
| "completions/min_terminated_length": 54.4, | |
| "entropy": 1.5682233929634095, | |
| "epoch": 0.58, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.823276042938232, | |
| "kl": 0.25128297358751295, | |
| "learning_rate": 3.5550000000000003e-06, | |
| "loss": 0.0429, | |
| "num_tokens": 3477785.0, | |
| "reward": 1.6693106412887573, | |
| "reward_std": 0.3742055296897888, | |
| "rewards/politeness_reward_func/mean": 1.6693106412887573, | |
| "rewards/politeness_reward_func/std": 0.4070769190788269, | |
| "step": 290, | |
| "step_time": 10.749481346458197 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.27083333730697634, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 240.8, | |
| "completions/mean_length": 174.8166717529297, | |
| "completions/mean_terminated_length": 145.73609619140626, | |
| "completions/min_length": 56.6, | |
| "completions/min_terminated_length": 56.6, | |
| "entropy": 1.7244809031486512, | |
| "epoch": 0.59, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.0669989585876465, | |
| "kl": 0.28962416797876356, | |
| "learning_rate": 3.53e-06, | |
| "loss": 0.0175, | |
| "num_tokens": 3537101.0, | |
| "reward": 1.6176325321197509, | |
| "reward_std": 0.4998854100704193, | |
| "rewards/politeness_reward_func/mean": 1.6176324844360352, | |
| "rewards/politeness_reward_func/std": 0.5255676746368408, | |
| "step": 295, | |
| "step_time": 10.83822975307703 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.29166668355464936, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.2, | |
| "completions/mean_length": 176.39166870117188, | |
| "completions/mean_terminated_length": 145.56358642578124, | |
| "completions/min_length": 58.0, | |
| "completions/min_terminated_length": 58.0, | |
| "entropy": 1.7615143656730652, | |
| "epoch": 0.6, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 9.959527015686035, | |
| "kl": 0.30573274195194244, | |
| "learning_rate": 3.505e-06, | |
| "loss": 0.0663, | |
| "num_tokens": 3597051.0, | |
| "reward": 1.7090127229690553, | |
| "reward_std": 0.4356145679950714, | |
| "rewards/politeness_reward_func/mean": 1.7090126276016235, | |
| "rewards/politeness_reward_func/std": 0.458771163225174, | |
| "step": 300, | |
| "step_time": 10.85788278505206 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.325000011920929, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.2, | |
| "completions/mean_length": 181.22083740234376, | |
| "completions/mean_terminated_length": 146.1617431640625, | |
| "completions/min_length": 55.0, | |
| "completions/min_terminated_length": 55.0, | |
| "entropy": 1.9170334219932557, | |
| "epoch": 0.61, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.597347259521484, | |
| "kl": 0.2534967973828316, | |
| "learning_rate": 3.48e-06, | |
| "loss": -0.019, | |
| "num_tokens": 3657680.0, | |
| "reward": 1.8059730291366578, | |
| "reward_std": 0.4023502767086029, | |
| "rewards/politeness_reward_func/mean": 1.8059730291366578, | |
| "rewards/politeness_reward_func/std": 0.42335187196731566, | |
| "step": 305, | |
| "step_time": 10.813059192150831 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3458333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.6, | |
| "completions/mean_length": 192.33333435058594, | |
| "completions/mean_terminated_length": 158.2584686279297, | |
| "completions/min_length": 61.0, | |
| "completions/min_terminated_length": 61.0, | |
| "entropy": 1.7168800473213195, | |
| "epoch": 0.62, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.3291268348693848, | |
| "kl": 0.30181923806667327, | |
| "learning_rate": 3.455e-06, | |
| "loss": 0.0438, | |
| "num_tokens": 3721168.0, | |
| "reward": 1.6650686740875245, | |
| "reward_std": 0.4736558377742767, | |
| "rewards/politeness_reward_func/mean": 1.6650686979293823, | |
| "rewards/politeness_reward_func/std": 0.5267638087272644, | |
| "step": 310, | |
| "step_time": 10.904363192617893 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2708333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.4, | |
| "completions/mean_length": 178.20833740234374, | |
| "completions/mean_terminated_length": 150.5656768798828, | |
| "completions/min_length": 52.8, | |
| "completions/min_terminated_length": 52.8, | |
| "entropy": 1.6762184143066405, | |
| "epoch": 0.63, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.4125494956970215, | |
| "kl": 0.3411761596798897, | |
| "learning_rate": 3.4300000000000006e-06, | |
| "loss": 0.0739, | |
| "num_tokens": 3781170.0, | |
| "reward": 1.698883295059204, | |
| "reward_std": 0.5051921904087067, | |
| "rewards/politeness_reward_func/mean": 1.6988832235336304, | |
| "rewards/politeness_reward_func/std": 0.5214821577072144, | |
| "step": 315, | |
| "step_time": 10.870812387019395 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3333333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 243.0, | |
| "completions/mean_length": 174.2375061035156, | |
| "completions/mean_terminated_length": 133.60752563476564, | |
| "completions/min_length": 38.8, | |
| "completions/min_terminated_length": 38.8, | |
| "entropy": 1.608705174922943, | |
| "epoch": 0.64, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.398674011230469, | |
| "kl": 0.3067095875740051, | |
| "learning_rate": 3.4050000000000004e-06, | |
| "loss": 0.0628, | |
| "num_tokens": 3840075.0, | |
| "reward": 1.6573886156082154, | |
| "reward_std": 0.4343807339668274, | |
| "rewards/politeness_reward_func/mean": 1.6573885917663573, | |
| "rewards/politeness_reward_func/std": 0.4526840627193451, | |
| "step": 320, | |
| "step_time": 10.640466004610062 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2666666716337204, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.8, | |
| "completions/mean_length": 177.06666870117186, | |
| "completions/mean_terminated_length": 148.47830200195312, | |
| "completions/min_length": 59.6, | |
| "completions/min_terminated_length": 59.6, | |
| "entropy": 1.832581627368927, | |
| "epoch": 0.65, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.723278522491455, | |
| "kl": 0.2649755135178566, | |
| "learning_rate": 3.3800000000000007e-06, | |
| "loss": -0.026, | |
| "num_tokens": 3899579.0, | |
| "reward": 1.81795072555542, | |
| "reward_std": 0.39153199791908266, | |
| "rewards/politeness_reward_func/mean": 1.8179506063461304, | |
| "rewards/politeness_reward_func/std": 0.41539669036865234, | |
| "step": 325, | |
| "step_time": 10.751856955885888 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.24583333730697632, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 238.2, | |
| "completions/mean_length": 164.57083740234376, | |
| "completions/mean_terminated_length": 135.28192138671875, | |
| "completions/min_length": 47.6, | |
| "completions/min_terminated_length": 47.6, | |
| "entropy": 1.6712656617164612, | |
| "epoch": 0.66, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.251152992248535, | |
| "kl": 0.2780707836151123, | |
| "learning_rate": 3.3550000000000005e-06, | |
| "loss": 0.0515, | |
| "num_tokens": 3956132.0, | |
| "reward": 1.6795947074890136, | |
| "reward_std": 0.4304037630558014, | |
| "rewards/politeness_reward_func/mean": 1.6795947313308717, | |
| "rewards/politeness_reward_func/std": 0.456265515089035, | |
| "step": 330, | |
| "step_time": 10.65747187435627 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.27083334028720857, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.8, | |
| "completions/mean_length": 170.53750610351562, | |
| "completions/mean_terminated_length": 139.22777709960937, | |
| "completions/min_length": 48.4, | |
| "completions/min_terminated_length": 48.4, | |
| "entropy": 1.6737295269966126, | |
| "epoch": 0.67, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.981579065322876, | |
| "kl": 0.27397735267877577, | |
| "learning_rate": 3.3300000000000003e-06, | |
| "loss": 0.0586, | |
| "num_tokens": 4014149.0, | |
| "reward": 1.639498805999756, | |
| "reward_std": 0.3987973630428314, | |
| "rewards/politeness_reward_func/mean": 1.639498782157898, | |
| "rewards/politeness_reward_func/std": 0.4488051861524582, | |
| "step": 335, | |
| "step_time": 10.64305683746934 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.24583334028720855, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.6, | |
| "completions/mean_length": 170.57083435058593, | |
| "completions/mean_terminated_length": 143.0279296875, | |
| "completions/min_length": 53.6, | |
| "completions/min_terminated_length": 53.6, | |
| "entropy": 1.8374268651008605, | |
| "epoch": 0.68, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.304769515991211, | |
| "kl": 0.2890111759305, | |
| "learning_rate": 3.3050000000000005e-06, | |
| "loss": 0.0539, | |
| "num_tokens": 4072318.0, | |
| "reward": 1.827410674095154, | |
| "reward_std": 0.34623334407806394, | |
| "rewards/politeness_reward_func/mean": 1.8274105072021485, | |
| "rewards/politeness_reward_func/std": 0.3635041773319244, | |
| "step": 340, | |
| "step_time": 10.67860155031085 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3791666865348816, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.8, | |
| "completions/mean_length": 182.3541717529297, | |
| "completions/mean_terminated_length": 138.33062896728515, | |
| "completions/min_length": 54.4, | |
| "completions/min_terminated_length": 54.4, | |
| "entropy": 1.8917979717254638, | |
| "epoch": 0.69, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.783524751663208, | |
| "kl": 0.3112198129296303, | |
| "learning_rate": 3.2800000000000004e-06, | |
| "loss": -0.0079, | |
| "num_tokens": 4133395.0, | |
| "reward": 1.8497365951538085, | |
| "reward_std": 0.41627883315086367, | |
| "rewards/politeness_reward_func/mean": 1.8497365236282348, | |
| "rewards/politeness_reward_func/std": 0.4410283327102661, | |
| "step": 345, | |
| "step_time": 10.664074825495481 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.43750001192092897, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.8, | |
| "completions/mean_length": 192.87083740234374, | |
| "completions/mean_terminated_length": 144.26836700439452, | |
| "completions/min_length": 62.4, | |
| "completions/min_terminated_length": 62.4, | |
| "entropy": 1.9642783045768737, | |
| "epoch": 0.7, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.8300986289978027, | |
| "kl": 0.3175328865647316, | |
| "learning_rate": 3.255e-06, | |
| "loss": 0.033, | |
| "num_tokens": 4197028.0, | |
| "reward": 1.8747668027877809, | |
| "reward_std": 0.485896635055542, | |
| "rewards/politeness_reward_func/mean": 1.8747668027877809, | |
| "rewards/politeness_reward_func/std": 0.5172119975090027, | |
| "step": 350, | |
| "step_time": 10.599913079291582 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.30416667759418486, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 245.4, | |
| "completions/mean_length": 173.94584045410156, | |
| "completions/mean_terminated_length": 138.12840881347657, | |
| "completions/min_length": 50.4, | |
| "completions/min_terminated_length": 50.4, | |
| "entropy": 1.6719454765319823, | |
| "epoch": 0.71, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.62943172454834, | |
| "kl": 0.354923115670681, | |
| "learning_rate": 3.2300000000000004e-06, | |
| "loss": 0.04, | |
| "num_tokens": 4256759.0, | |
| "reward": 1.746033239364624, | |
| "reward_std": 0.46536014080047605, | |
| "rewards/politeness_reward_func/mean": 1.7460331916809082, | |
| "rewards/politeness_reward_func/std": 0.513632845878601, | |
| "step": 355, | |
| "step_time": 10.574684323370457 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2166666716337204, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 246.4, | |
| "completions/mean_length": 164.38750610351562, | |
| "completions/mean_terminated_length": 139.19703826904296, | |
| "completions/min_length": 58.2, | |
| "completions/min_terminated_length": 58.2, | |
| "entropy": 1.7568098187446595, | |
| "epoch": 0.72, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.490025997161865, | |
| "kl": 0.2866540029644966, | |
| "learning_rate": 3.2050000000000002e-06, | |
| "loss": -0.0106, | |
| "num_tokens": 4313620.0, | |
| "reward": 1.7767923593521118, | |
| "reward_std": 0.4481996238231659, | |
| "rewards/politeness_reward_func/mean": 1.776792311668396, | |
| "rewards/politeness_reward_func/std": 0.4569097697734833, | |
| "step": 360, | |
| "step_time": 10.563734823465348 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3833333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 249.8, | |
| "completions/mean_length": 192.51250305175782, | |
| "completions/mean_terminated_length": 152.48777465820314, | |
| "completions/min_length": 54.8, | |
| "completions/min_terminated_length": 54.8, | |
| "entropy": 1.7603564739227295, | |
| "epoch": 0.73, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.731227159500122, | |
| "kl": 0.2559724062681198, | |
| "learning_rate": 3.1800000000000005e-06, | |
| "loss": 0.0058, | |
| "num_tokens": 4377423.0, | |
| "reward": 1.7886356353759765, | |
| "reward_std": 0.43659440279006956, | |
| "rewards/politeness_reward_func/mean": 1.7886356592178345, | |
| "rewards/politeness_reward_func/std": 0.4865987479686737, | |
| "step": 365, | |
| "step_time": 10.596465566009282 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3208333432674408, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 253.0, | |
| "completions/mean_length": 183.37500610351563, | |
| "completions/mean_terminated_length": 150.60201873779297, | |
| "completions/min_length": 56.2, | |
| "completions/min_terminated_length": 56.2, | |
| "entropy": 1.599543821811676, | |
| "epoch": 0.74, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.495799541473389, | |
| "kl": 0.26432290226221083, | |
| "learning_rate": 3.1550000000000003e-06, | |
| "loss": 0.0365, | |
| "num_tokens": 4438841.0, | |
| "reward": 1.663888192176819, | |
| "reward_std": 0.4724281966686249, | |
| "rewards/politeness_reward_func/mean": 1.663888168334961, | |
| "rewards/politeness_reward_func/std": 0.49500845074653627, | |
| "step": 370, | |
| "step_time": 10.550004740059375 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2833333402872086, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 245.8, | |
| "completions/mean_length": 176.43333740234374, | |
| "completions/mean_terminated_length": 145.9316650390625, | |
| "completions/min_length": 46.8, | |
| "completions/min_terminated_length": 46.8, | |
| "entropy": 1.649828588962555, | |
| "epoch": 0.75, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.405642509460449, | |
| "kl": 0.3023358851671219, | |
| "learning_rate": 3.13e-06, | |
| "loss": 0.036, | |
| "num_tokens": 4498209.0, | |
| "reward": 1.7260100841522217, | |
| "reward_std": 0.4888251006603241, | |
| "rewards/politeness_reward_func/mean": 1.726010012626648, | |
| "rewards/politeness_reward_func/std": 0.5257126033306122, | |
| "step": 375, | |
| "step_time": 10.784504148364068 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.2750000059604645, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 251.0, | |
| "completions/mean_length": 177.4791717529297, | |
| "completions/mean_terminated_length": 149.0453887939453, | |
| "completions/min_length": 52.6, | |
| "completions/min_terminated_length": 52.6, | |
| "entropy": 1.669038712978363, | |
| "epoch": 0.76, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.453659534454346, | |
| "kl": 0.26074831187725067, | |
| "learning_rate": 3.1050000000000003e-06, | |
| "loss": 0.0133, | |
| "num_tokens": 4558116.0, | |
| "reward": 1.7338483333587646, | |
| "reward_std": 0.40128968358039857, | |
| "rewards/politeness_reward_func/mean": 1.7338482856750488, | |
| "rewards/politeness_reward_func/std": 0.4201090157032013, | |
| "step": 380, | |
| "step_time": 10.85576168820262 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3166666775941849, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 248.4, | |
| "completions/mean_length": 175.9791687011719, | |
| "completions/mean_terminated_length": 141.09287872314454, | |
| "completions/min_length": 60.8, | |
| "completions/min_terminated_length": 60.8, | |
| "entropy": 1.70971120595932, | |
| "epoch": 0.77, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 4.21511173248291, | |
| "kl": 0.33568228632211683, | |
| "learning_rate": 3.08e-06, | |
| "loss": 0.0539, | |
| "num_tokens": 4617679.0, | |
| "reward": 1.8021876573562623, | |
| "reward_std": 0.39429293274879457, | |
| "rewards/politeness_reward_func/mean": 1.8021875858306884, | |
| "rewards/politeness_reward_func/std": 0.4176251709461212, | |
| "step": 385, | |
| "step_time": 10.889974600821734 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.24583333879709243, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.4, | |
| "completions/mean_length": 162.72084045410156, | |
| "completions/mean_terminated_length": 133.57429962158204, | |
| "completions/min_length": 56.8, | |
| "completions/min_terminated_length": 56.8, | |
| "entropy": 1.5501957058906555, | |
| "epoch": 0.78, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 5.835730075836182, | |
| "kl": 0.30908732712268827, | |
| "learning_rate": 3.0550000000000004e-06, | |
| "loss": 0.0817, | |
| "num_tokens": 4674108.0, | |
| "reward": 1.7301536321640014, | |
| "reward_std": 0.44032427072525027, | |
| "rewards/politeness_reward_func/mean": 1.7301535844802856, | |
| "rewards/politeness_reward_func/std": 0.4708732008934021, | |
| "step": 390, | |
| "step_time": 10.865944185107946 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.1791666716337204, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 247.8, | |
| "completions/mean_length": 157.25000915527343, | |
| "completions/mean_terminated_length": 135.5115005493164, | |
| "completions/min_length": 51.2, | |
| "completions/min_terminated_length": 51.2, | |
| "entropy": 1.6409943580627442, | |
| "epoch": 0.79, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 6.950301647186279, | |
| "kl": 0.36195332258939744, | |
| "learning_rate": 3.0300000000000002e-06, | |
| "loss": 0.0805, | |
| "num_tokens": 4729176.0, | |
| "reward": 1.737137007713318, | |
| "reward_std": 0.4313428819179535, | |
| "rewards/politeness_reward_func/mean": 1.737137007713318, | |
| "rewards/politeness_reward_func/std": 0.4497229218482971, | |
| "step": 395, | |
| "step_time": 10.928832749277353 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.3291666775941849, | |
| "completions/max_length": 256.0, | |
| "completions/max_terminated_length": 250.4, | |
| "completions/mean_length": 180.15833740234376, | |
| "completions/mean_terminated_length": 143.84142608642577, | |
| "completions/min_length": 58.0, | |
| "completions/min_terminated_length": 58.0, | |
| "entropy": 1.7865488886833192, | |
| "epoch": 0.8, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 3.5700058937072754, | |
| "kl": 0.2882836848497391, | |
| "learning_rate": 3.005e-06, | |
| "loss": 0.0432, | |
| "num_tokens": 4790094.0, | |
| "reward": 1.7929600477218628, | |
| "reward_std": 0.4272151470184326, | |
| "rewards/politeness_reward_func/mean": 1.792960000038147, | |
| "rewards/politeness_reward_func/std": 0.4529218733310699, | |
| "step": 400, | |
| "step_time": 10.825692503154277 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 1000, | |
| "num_input_tokens_seen": 4790094, | |
| "num_train_epochs": 2, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |