{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.8, "eval_steps": 500, "global_step": 400, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6791666865348815, "completions/max_length": 256.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 236.5416717529297, "completions/mean_terminated_length": 200.61068420410157, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.070425021648407, "epoch": 0.01, "frac_reward_zero_std": 0.0, "grad_norm": 3.5985119342803955, "kl": 0.0035211448557674885, "learning_rate": 4.980000000000001e-06, "loss": 0.0011, "num_tokens": 73826.0, "reward": 0.2776068687438965, "reward_std": 0.5232907950878143, "rewards/politeness_reward_func/mean": 0.27760685607790947, "rewards/politeness_reward_func/std": 0.5940477013587951, "step": 5, "step_time": 11.145096338540316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6208333492279052, "completions/max_length": 256.0, "completions/max_terminated_length": 246.8, "completions/mean_length": 232.80417175292968, "completions/mean_terminated_length": 204.57913513183593, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.070945155620575, "epoch": 0.02, "frac_reward_zero_std": 0.0, "grad_norm": 3.3468329906463623, "kl": 0.02206921111792326, "learning_rate": 4.955e-06, "loss": 0.0224, "num_tokens": 147283.0, "reward": 0.6233123302459717, "reward_std": 0.4648557245731354, "rewards/politeness_reward_func/mean": 0.6233122944831848, "rewards/politeness_reward_func/std": 0.553675526380539, "step": 10, "step_time": 10.835208699852227 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5000000119209289, "completions/max_length": 256.0, "completions/max_terminated_length": 249.6, "completions/mean_length": 221.8000030517578, "completions/mean_terminated_length": 188.9316619873047, "completions/min_length": 110.8, "completions/min_terminated_length": 110.8, "entropy": 1.1163462162017823, "epoch": 0.03, "frac_reward_zero_std": 0.0, "grad_norm": 3.3379204273223877, "kl": 0.04084419272840023, "learning_rate": 4.93e-06, "loss": -0.0052, "num_tokens": 218131.0, "reward": 0.8122925400733948, "reward_std": 0.41203114986419676, "rewards/politeness_reward_func/mean": 0.8122925400733948, "rewards/politeness_reward_func/std": 0.4989388585090637, "step": 15, "step_time": 10.806195054203272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6791666865348815, "completions/max_length": 256.0, "completions/max_terminated_length": 251.6, "completions/mean_length": 238.19584045410156, "completions/mean_terminated_length": 201.80428771972657, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.1251786351203918, "epoch": 0.04, "frac_reward_zero_std": 0.0, "grad_norm": 3.3837928771972656, "kl": 0.0385904410853982, "learning_rate": 4.9050000000000005e-06, "loss": -0.0031, "num_tokens": 292306.0, "reward": 0.6734941720962524, "reward_std": 0.448628181219101, "rewards/politeness_reward_func/mean": 0.6734941720962524, "rewards/politeness_reward_func/std": 0.509108817577362, "step": 20, "step_time": 10.808321141451597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.7333333551883697, "completions/max_length": 256.0, "completions/max_terminated_length": 246.6, "completions/mean_length": 239.7041748046875, "completions/mean_terminated_length": 201.94395141601564, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0987902224063872, "epoch": 0.05, "frac_reward_zero_std": 0.0, "grad_norm": 3.3908112049102783, "kl": 0.04714170750230551, "learning_rate": 4.880000000000001e-06, "loss": 0.013, "num_tokens": 366731.0, "reward": 0.7238378047943115, "reward_std": 0.5012153267860413, "rewards/politeness_reward_func/mean": 0.7238377809524537, "rewards/politeness_reward_func/std": 0.5635832965373992, "step": 25, "step_time": 10.778460966795683 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.7041666984558106, "completions/max_length": 256.0, "completions/max_terminated_length": 251.4, "completions/mean_length": 238.33334045410157, "completions/mean_terminated_length": 209.13802490234374, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.0942281484603882, "epoch": 0.06, "frac_reward_zero_std": 0.0, "grad_norm": 3.272942304611206, "kl": 0.04730561040341854, "learning_rate": 4.855e-06, "loss": -0.0003, "num_tokens": 441211.0, "reward": 0.7387123763561249, "reward_std": 0.3938853979110718, "rewards/politeness_reward_func/mean": 0.7387123882770539, "rewards/politeness_reward_func/std": 0.5152743279933929, "step": 30, "step_time": 10.791689745336772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6000000178813935, "completions/max_length": 256.0, "completions/max_terminated_length": 250.8, "completions/mean_length": 232.95417175292968, "completions/mean_terminated_length": 200.11821594238282, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1196449041366576, "epoch": 0.07, "frac_reward_zero_std": 0.0, "grad_norm": 3.515143394470215, "kl": 0.055315449833869934, "learning_rate": 4.83e-06, "loss": 0.0018, "num_tokens": 514144.0, "reward": 0.9808868050575257, "reward_std": 0.3419617712497711, "rewards/politeness_reward_func/mean": 0.9808867692947387, "rewards/politeness_reward_func/std": 0.4772939085960388, "step": 35, "step_time": 10.767037071287632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4333333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 253.2, "completions/mean_length": 216.89583740234374, "completions/mean_terminated_length": 186.88653869628905, "completions/min_length": 90.4, "completions/min_terminated_length": 90.4, "entropy": 1.062067198753357, "epoch": 0.08, "frac_reward_zero_std": 0.0, "grad_norm": 3.2335519790649414, "kl": 0.06222313120961189, "learning_rate": 4.805000000000001e-06, "loss": 0.0425, "num_tokens": 583095.0, "reward": 0.873325777053833, "reward_std": 0.4448536515235901, "rewards/politeness_reward_func/mean": 0.873325777053833, "rewards/politeness_reward_func/std": 0.49484293460845946, "step": 40, "step_time": 10.696332201361656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5666666746139526, "completions/max_length": 256.0, "completions/max_terminated_length": 248.6, "completions/mean_length": 220.82500610351562, "completions/mean_terminated_length": 175.39512634277344, "completions/min_length": 75.6, "completions/min_terminated_length": 75.6, "entropy": 1.0480997025966645, "epoch": 0.09, "frac_reward_zero_std": 0.0, "grad_norm": 3.452233076095581, "kl": 0.07400040999054909, "learning_rate": 4.78e-06, "loss": 0.0107, "num_tokens": 653373.0, "reward": 0.9472709178924561, "reward_std": 0.40665341913700104, "rewards/politeness_reward_func/mean": 0.9472708940505982, "rewards/politeness_reward_func/std": 0.4510787308216095, "step": 45, "step_time": 10.80831560343504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.5416666865348816, "completions/max_length": 256.0, "completions/max_terminated_length": 249.8, "completions/mean_length": 208.1916717529297, "completions/mean_terminated_length": 153.92529907226563, "completions/min_length": 61.8, "completions/min_terminated_length": 61.8, "entropy": 1.1236772537231445, "epoch": 0.1, "frac_reward_zero_std": 0.0, "grad_norm": 3.160369634628296, "kl": 0.08055943846702576, "learning_rate": 4.755e-06, "loss": 0.0611, "num_tokens": 720619.0, "reward": 0.8770254850387573, "reward_std": 0.5482801616191864, "rewards/politeness_reward_func/mean": 0.8770254492759705, "rewards/politeness_reward_func/std": 0.5923300921916962, "step": 50, "step_time": 10.79710350483656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4083333373069763, "completions/max_length": 256.0, "completions/max_terminated_length": 245.4, "completions/mean_length": 180.80833740234374, "completions/mean_terminated_length": 134.77063598632813, "completions/min_length": 46.0, "completions/min_terminated_length": 46.0, "entropy": 1.2555986642837524, "epoch": 0.11, "frac_reward_zero_std": 0.0, "grad_norm": 5.3433427810668945, "kl": 0.09562707468867301, "learning_rate": 4.7300000000000005e-06, "loss": 0.0773, "num_tokens": 781261.0, "reward": 1.092347764968872, "reward_std": 0.4649295687675476, "rewards/politeness_reward_func/mean": 1.0923476934432983, "rewards/politeness_reward_func/std": 0.5502241551876068, "step": 55, "step_time": 10.756779878586531 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2875000063329935, "completions/max_length": 256.0, "completions/max_terminated_length": 238.8, "completions/mean_length": 154.1916702270508, "completions/mean_terminated_length": 119.38919677734376, "completions/min_length": 42.8, "completions/min_terminated_length": 42.8, "entropy": 1.3319262266159058, "epoch": 0.12, "frac_reward_zero_std": 0.0, "grad_norm": 3.3366644382476807, "kl": 0.14686549603939056, "learning_rate": 4.705e-06, "loss": 0.0456, "num_tokens": 835835.0, "reward": 1.4423248767852783, "reward_std": 0.36013287007808686, "rewards/politeness_reward_func/mean": 1.4423248767852783, "rewards/politeness_reward_func/std": 0.40978127121925356, "step": 60, "step_time": 10.825969719141721 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41250001192092894, "completions/max_length": 256.0, "completions/max_terminated_length": 247.4, "completions/mean_length": 183.07083740234376, "completions/mean_terminated_length": 133.48724670410155, "completions/min_length": 46.8, "completions/min_terminated_length": 46.8, "entropy": 1.3182121157646178, "epoch": 0.13, "frac_reward_zero_std": 0.0, "grad_norm": 25.555892944335938, "kl": 0.17301606237888337, "learning_rate": 4.680000000000001e-06, "loss": 0.0985, "num_tokens": 896732.0, "reward": 1.3051372289657592, "reward_std": 0.43976340293884275, "rewards/politeness_reward_func/mean": 1.3051372170448303, "rewards/politeness_reward_func/std": 0.5006832242012024, "step": 65, "step_time": 10.645895641297102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2708333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 246.4, "completions/mean_length": 157.8791717529297, "completions/mean_terminated_length": 123.51410827636718, "completions/min_length": 41.4, "completions/min_terminated_length": 41.4, "entropy": 1.2897038578987121, "epoch": 0.14, "frac_reward_zero_std": 0.0, "grad_norm": 3.8760225772857666, "kl": 0.19847530573606492, "learning_rate": 4.655e-06, "loss": 0.1906, "num_tokens": 951999.0, "reward": 1.420637583732605, "reward_std": 0.430054372549057, "rewards/politeness_reward_func/mean": 1.420637583732605, "rewards/politeness_reward_func/std": 0.48181390166282656, "step": 70, "step_time": 10.596996573358775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2083333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 240.4, "completions/mean_length": 146.60000610351562, "completions/mean_terminated_length": 118.93135223388671, "completions/min_length": 43.2, "completions/min_terminated_length": 43.2, "entropy": 1.4299420952796935, "epoch": 0.15, "frac_reward_zero_std": 0.0, "grad_norm": 4.174029350280762, "kl": 0.20222627967596055, "learning_rate": 4.6300000000000006e-06, "loss": 0.1187, "num_tokens": 1004287.0, "reward": 1.5081899881362915, "reward_std": 0.33264915347099305, "rewards/politeness_reward_func/mean": 1.508189868927002, "rewards/politeness_reward_func/std": 0.36398229002952576, "step": 75, "step_time": 10.52198056280613 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12916667088866235, "completions/max_length": 256.0, "completions/max_terminated_length": 242.6, "completions/mean_length": 142.55417175292968, "completions/mean_terminated_length": 126.68900451660156, "completions/min_length": 48.2, "completions/min_terminated_length": 48.2, "entropy": 1.3346869945526123, "epoch": 0.16, "frac_reward_zero_std": 0.0, "grad_norm": 6.63394021987915, "kl": 0.19363965839147568, "learning_rate": 4.605000000000001e-06, "loss": 0.0898, "num_tokens": 1056308.0, "reward": 1.459042239189148, "reward_std": 0.2736098259687424, "rewards/politeness_reward_func/mean": 1.45904221534729, "rewards/politeness_reward_func/std": 0.3277650147676468, "step": 80, "step_time": 10.517034325748682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.20416666716337203, "completions/max_length": 256.0, "completions/max_terminated_length": 241.2, "completions/mean_length": 137.8166717529297, "completions/mean_terminated_length": 107.75043640136718, "completions/min_length": 39.2, "completions/min_terminated_length": 39.2, "entropy": 1.0585298597812653, "epoch": 0.17, "frac_reward_zero_std": 0.0, "grad_norm": 6.915429592132568, "kl": 0.2862268269062042, "learning_rate": 4.58e-06, "loss": 0.2369, "num_tokens": 1106584.0, "reward": 1.339914321899414, "reward_std": 0.42014909982681276, "rewards/politeness_reward_func/mean": 1.3399142742156982, "rewards/politeness_reward_func/std": 0.46948710083961487, "step": 85, "step_time": 10.558716806769372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.26250001005828383, "completions/max_length": 256.0, "completions/max_terminated_length": 232.6, "completions/mean_length": 154.93750457763673, "completions/mean_terminated_length": 121.31997375488281, "completions/min_length": 47.0, "completions/min_terminated_length": 47.0, "entropy": 1.208097517490387, "epoch": 0.18, "frac_reward_zero_std": 0.0, "grad_norm": 4.7058539390563965, "kl": 0.17084270417690278, "learning_rate": 4.5550000000000004e-06, "loss": 0.0854, "num_tokens": 1161209.0, "reward": 1.4180654048919679, "reward_std": 0.24261603355407715, "rewards/politeness_reward_func/mean": 1.4180653095245361, "rewards/politeness_reward_func/std": 0.3126596033573151, "step": 90, "step_time": 10.573557010293007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15000000707805156, "completions/max_length": 245.4, "completions/max_terminated_length": 235.8, "completions/mean_length": 145.20000457763672, "completions/mean_terminated_length": 128.7294189453125, "completions/min_length": 50.8, "completions/min_terminated_length": 50.8, "entropy": 1.2446547746658325, "epoch": 0.19, "frac_reward_zero_std": 0.0, "grad_norm": 3.650216579437256, "kl": 0.19710368812084197, "learning_rate": 4.530000000000001e-06, "loss": 0.1361, "num_tokens": 1213177.0, "reward": 1.3565361022949218, "reward_std": 0.3620565414428711, "rewards/politeness_reward_func/mean": 1.356536078453064, "rewards/politeness_reward_func/std": 0.4186973810195923, "step": 95, "step_time": 10.10843729674816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15416666865348816, "completions/max_length": 253.2, "completions/max_terminated_length": 239.6, "completions/mean_length": 127.69167022705078, "completions/mean_terminated_length": 104.76427154541015, "completions/min_length": 33.6, "completions/min_terminated_length": 33.6, "entropy": 1.21082746386528, "epoch": 0.2, "frac_reward_zero_std": 0.0, "grad_norm": 4.778512477874756, "kl": 0.23306063413619996, "learning_rate": 4.505e-06, "loss": 0.1579, "num_tokens": 1261199.0, "reward": 1.4624977350234984, "reward_std": 0.27724905014038087, "rewards/politeness_reward_func/mean": 1.4624976634979248, "rewards/politeness_reward_func/std": 0.353417181968689, "step": 100, "step_time": 10.490663215517998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2916666746139526, "completions/max_length": 256.0, "completions/max_terminated_length": 239.8, "completions/mean_length": 164.0000030517578, "completions/mean_terminated_length": 131.73128814697264, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "entropy": 1.227693748474121, "epoch": 0.21, "frac_reward_zero_std": 0.0, "grad_norm": 3.480128765106201, "kl": 0.21182419136166572, "learning_rate": 4.48e-06, "loss": 0.1385, "num_tokens": 1317887.0, "reward": 1.4506232976913451, "reward_std": 0.38838809728622437, "rewards/politeness_reward_func/mean": 1.4506232500076295, "rewards/politeness_reward_func/std": 0.415321010351181, "step": 105, "step_time": 10.568055501580238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2875000134110451, "completions/max_length": 256.0, "completions/max_terminated_length": 246.6, "completions/mean_length": 164.9791717529297, "completions/mean_terminated_length": 129.34453430175782, "completions/min_length": 50.8, "completions/min_terminated_length": 50.8, "entropy": 1.2591872215270996, "epoch": 0.22, "frac_reward_zero_std": 0.0, "grad_norm": 5.477797985076904, "kl": 0.21972680687904358, "learning_rate": 4.4550000000000005e-06, "loss": 0.118, "num_tokens": 1374570.0, "reward": 1.3832386493682862, "reward_std": 0.36756529808044436, "rewards/politeness_reward_func/mean": 1.3832386016845704, "rewards/politeness_reward_func/std": 0.45387519598007203, "step": 110, "step_time": 10.580036842823029 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4041666805744171, "completions/max_length": 256.0, "completions/max_terminated_length": 239.6, "completions/mean_length": 189.48334045410155, "completions/mean_terminated_length": 144.77314147949218, "completions/min_length": 55.6, "completions/min_terminated_length": 55.6, "entropy": 1.4673572778701782, "epoch": 0.23, "frac_reward_zero_std": 0.0, "grad_norm": 4.5870041847229, "kl": 0.18002651631832123, "learning_rate": 4.430000000000001e-06, "loss": 0.0902, "num_tokens": 1437278.0, "reward": 1.463021945953369, "reward_std": 0.36559332013130186, "rewards/politeness_reward_func/mean": 1.4630219221115113, "rewards/politeness_reward_func/std": 0.4403323769569397, "step": 115, "step_time": 10.606720576435327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.29166667759418485, "completions/max_length": 256.0, "completions/max_terminated_length": 250.4, "completions/mean_length": 168.28750610351562, "completions/mean_terminated_length": 134.1982864379883, "completions/min_length": 45.0, "completions/min_terminated_length": 45.0, "entropy": 1.439770805835724, "epoch": 0.24, "frac_reward_zero_std": 0.0, "grad_norm": 4.188452243804932, "kl": 0.23004925400018691, "learning_rate": 4.405e-06, "loss": 0.1038, "num_tokens": 1495123.0, "reward": 1.6009358167648315, "reward_std": 0.3070066928863525, "rewards/politeness_reward_func/mean": 1.6009357929229737, "rewards/politeness_reward_func/std": 0.34360362887382506, "step": 120, "step_time": 10.64953521117568 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1833333395421505, "completions/max_length": 256.0, "completions/max_terminated_length": 243.2, "completions/mean_length": 140.8666748046875, "completions/mean_terminated_length": 116.34450073242188, "completions/min_length": 37.8, "completions/min_terminated_length": 37.8, "entropy": 1.2707997798919677, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 4.1139140129089355, "kl": 0.2768221229314804, "learning_rate": 4.38e-06, "loss": 0.146, "num_tokens": 1546755.0, "reward": 1.528564214706421, "reward_std": 0.29817952811717985, "rewards/politeness_reward_func/mean": 1.528564167022705, "rewards/politeness_reward_func/std": 0.353902006149292, "step": 125, "step_time": 10.536190098524093 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12500000298023223, "completions/max_length": 256.0, "completions/max_terminated_length": 228.6, "completions/mean_length": 126.03333587646485, "completions/mean_terminated_length": 108.7752197265625, "completions/min_length": 37.6, "completions/min_terminated_length": 37.6, "entropy": 1.239863795042038, "epoch": 0.26, "frac_reward_zero_std": 0.0, "grad_norm": 20.54998207092285, "kl": 0.4258181616663933, "learning_rate": 4.355000000000001e-06, "loss": 0.205, "num_tokens": 1594475.0, "reward": 1.5162237882614136, "reward_std": 0.37349834442138674, "rewards/politeness_reward_func/mean": 1.5162237882614136, "rewards/politeness_reward_func/std": 0.4025087893009186, "step": 130, "step_time": 10.561264203488827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1375000063329935, "completions/max_length": 256.0, "completions/max_terminated_length": 235.4, "completions/mean_length": 121.34167175292968, "completions/mean_terminated_length": 100.33185272216797, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.44633287191391, "epoch": 0.27, "frac_reward_zero_std": 0.0, "grad_norm": 4.6437859535217285, "kl": 0.2876953512430191, "learning_rate": 4.33e-06, "loss": 0.1474, "num_tokens": 1640909.0, "reward": 1.6570862054824829, "reward_std": 0.35202938318252563, "rewards/politeness_reward_func/mean": 1.657086157798767, "rewards/politeness_reward_func/std": 0.4086485207080841, "step": 135, "step_time": 10.50367113724351 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25000000596046446, "completions/max_length": 256.0, "completions/max_terminated_length": 232.4, "completions/mean_length": 165.46250610351564, "completions/mean_terminated_length": 134.87125549316406, "completions/min_length": 50.2, "completions/min_terminated_length": 50.2, "entropy": 1.5645981192588807, "epoch": 0.28, "frac_reward_zero_std": 0.0, "grad_norm": 4.930655479431152, "kl": 0.20734833329916, "learning_rate": 4.305e-06, "loss": 0.1052, "num_tokens": 1698396.0, "reward": 1.5796484470367431, "reward_std": 0.39563443660736086, "rewards/politeness_reward_func/mean": 1.5796483516693116, "rewards/politeness_reward_func/std": 0.43083653450012205, "step": 140, "step_time": 10.580496142059564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3666666731238365, "completions/max_length": 256.0, "completions/max_terminated_length": 245.4, "completions/mean_length": 182.1541717529297, "completions/mean_terminated_length": 148.0070785522461, "completions/min_length": 67.8, "completions/min_terminated_length": 67.8, "entropy": 1.4628460884094239, "epoch": 0.29, "frac_reward_zero_std": 0.0, "grad_norm": 3.1963322162628174, "kl": 0.20285871326923371, "learning_rate": 4.2800000000000005e-06, "loss": 0.0054, "num_tokens": 1759217.0, "reward": 1.5254708528518677, "reward_std": 0.3578204929828644, "rewards/politeness_reward_func/mean": 1.5254708051681518, "rewards/politeness_reward_func/std": 0.39698067903518675, "step": 145, "step_time": 10.641796179115772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.325000011920929, "completions/max_length": 256.0, "completions/max_terminated_length": 243.8, "completions/mean_length": 185.4041717529297, "completions/mean_terminated_length": 151.72132263183593, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 1.550300669670105, "epoch": 0.3, "frac_reward_zero_std": 0.0, "grad_norm": 3.5268337726593018, "kl": 0.26886349618434907, "learning_rate": 4.255e-06, "loss": 0.0341, "num_tokens": 1821250.0, "reward": 1.5432605028152466, "reward_std": 0.35273998975753784, "rewards/politeness_reward_func/mean": 1.5432604551315308, "rewards/politeness_reward_func/std": 0.39439594745635986, "step": 150, "step_time": 10.68586125895381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.20416667610406875, "completions/max_length": 256.0, "completions/max_terminated_length": 241.8, "completions/mean_length": 162.42083740234375, "completions/mean_terminated_length": 139.60369567871095, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 1.665429162979126, "epoch": 0.31, "frac_reward_zero_std": 0.0, "grad_norm": 3.6844170093536377, "kl": 0.22049252390861512, "learning_rate": 4.23e-06, "loss": 0.0256, "num_tokens": 1877383.0, "reward": 1.6982928276062013, "reward_std": 0.30497619807720183, "rewards/politeness_reward_func/mean": 1.6982927560806274, "rewards/politeness_reward_func/std": 0.3323679566383362, "step": 155, "step_time": 10.646523833274841 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12083333805203438, "completions/max_length": 256.0, "completions/max_terminated_length": 238.4, "completions/mean_length": 141.9625045776367, "completions/mean_terminated_length": 126.35040893554688, "completions/min_length": 53.0, "completions/min_terminated_length": 53.0, "entropy": 1.493414855003357, "epoch": 0.32, "frac_reward_zero_std": 0.0, "grad_norm": 5.583273410797119, "kl": 0.25222020596265793, "learning_rate": 4.205e-06, "loss": 0.0511, "num_tokens": 1928318.0, "reward": 1.6409826755523682, "reward_std": 0.2959387719631195, "rewards/politeness_reward_func/mean": 1.6409826040267945, "rewards/politeness_reward_func/std": 0.3353587478399277, "step": 160, "step_time": 10.597700411826372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.22916667088866233, "completions/max_length": 256.0, "completions/max_terminated_length": 248.2, "completions/mean_length": 159.16250762939453, "completions/mean_terminated_length": 130.86475067138673, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 1.5048229694366455, "epoch": 0.33, "frac_reward_zero_std": 0.0, "grad_norm": 8.112273216247559, "kl": 0.26160909086465833, "learning_rate": 4.18e-06, "loss": 0.1583, "num_tokens": 1983861.0, "reward": 1.5394394874572754, "reward_std": 0.4292898178100586, "rewards/politeness_reward_func/mean": 1.5394394636154174, "rewards/politeness_reward_func/std": 0.4557326793670654, "step": 165, "step_time": 10.650840406119823 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3583333373069763, "completions/max_length": 256.0, "completions/max_terminated_length": 249.8, "completions/mean_length": 181.42084045410155, "completions/mean_terminated_length": 142.14824829101562, "completions/min_length": 47.6, "completions/min_terminated_length": 47.6, "entropy": 1.6774910688400269, "epoch": 0.34, "frac_reward_zero_std": 0.0, "grad_norm": 5.154572486877441, "kl": 0.21350446194410325, "learning_rate": 4.155e-06, "loss": 0.0845, "num_tokens": 2044922.0, "reward": 1.617796039581299, "reward_std": 0.42567009329795835, "rewards/politeness_reward_func/mean": 1.617796039581299, "rewards/politeness_reward_func/std": 0.5039380311965942, "step": 170, "step_time": 10.696013970673084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21666667759418487, "completions/max_length": 256.0, "completions/max_terminated_length": 235.8, "completions/mean_length": 150.75000610351563, "completions/mean_terminated_length": 122.61416320800781, "completions/min_length": 41.4, "completions/min_terminated_length": 41.4, "entropy": 1.7118729948997498, "epoch": 0.35, "frac_reward_zero_std": 0.0, "grad_norm": 4.096772193908691, "kl": 0.24443377554416656, "learning_rate": 4.13e-06, "loss": 0.062, "num_tokens": 2098654.0, "reward": 1.7112194776535035, "reward_std": 0.41009202003479006, "rewards/politeness_reward_func/mean": 1.7112194299697876, "rewards/politeness_reward_func/std": 0.46850563287734986, "step": 175, "step_time": 10.681096732616425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24583333544433117, "completions/max_length": 256.0, "completions/max_terminated_length": 250.4, "completions/mean_length": 161.39583892822264, "completions/mean_terminated_length": 134.3897247314453, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "entropy": 1.6281691670417786, "epoch": 0.36, "frac_reward_zero_std": 0.0, "grad_norm": 39.266761779785156, "kl": 0.3168014049530029, "learning_rate": 4.1050000000000005e-06, "loss": 0.0628, "num_tokens": 2154557.0, "reward": 1.592983651161194, "reward_std": 0.4036149501800537, "rewards/politeness_reward_func/mean": 1.5929836750030517, "rewards/politeness_reward_func/std": 0.4594584465026855, "step": 180, "step_time": 10.576303614675998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2041666716337204, "completions/max_length": 256.0, "completions/max_terminated_length": 248.0, "completions/mean_length": 154.63750305175782, "completions/mean_terminated_length": 128.9908935546875, "completions/min_length": 47.2, "completions/min_terminated_length": 47.2, "entropy": 1.5905902862548829, "epoch": 0.37, "frac_reward_zero_std": 0.0, "grad_norm": 3.8977794647216797, "kl": 0.2617978900671005, "learning_rate": 4.08e-06, "loss": 0.0605, "num_tokens": 2208870.0, "reward": 1.608209228515625, "reward_std": 0.40417273342609406, "rewards/politeness_reward_func/mean": 1.608209204673767, "rewards/politeness_reward_func/std": 0.44712826013565066, "step": 185, "step_time": 10.657971718162297 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25416666865348814, "completions/max_length": 256.0, "completions/max_terminated_length": 243.6, "completions/mean_length": 167.1416748046875, "completions/mean_terminated_length": 136.75882873535156, "completions/min_length": 46.6, "completions/min_terminated_length": 46.6, "entropy": 1.76783127784729, "epoch": 0.38, "frac_reward_zero_std": 0.0, "grad_norm": 5.072940826416016, "kl": 0.29045032411813737, "learning_rate": 4.055000000000001e-06, "loss": 0.0313, "num_tokens": 2266296.0, "reward": 1.7399524927139283, "reward_std": 0.4326803207397461, "rewards/politeness_reward_func/mean": 1.7399524688720702, "rewards/politeness_reward_func/std": 0.44698241353034973, "step": 190, "step_time": 10.595029101520776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.387500011920929, "completions/max_length": 256.0, "completions/max_terminated_length": 249.8, "completions/mean_length": 190.1125030517578, "completions/mean_terminated_length": 150.54798278808593, "completions/min_length": 57.8, "completions/min_terminated_length": 57.8, "entropy": 1.7596600532531739, "epoch": 0.39, "frac_reward_zero_std": 0.0, "grad_norm": 4.128026008605957, "kl": 0.25340984016656876, "learning_rate": 4.03e-06, "loss": 0.0384, "num_tokens": 2329619.0, "reward": 1.6809997797012328, "reward_std": 0.4062549531459808, "rewards/politeness_reward_func/mean": 1.6809997320175172, "rewards/politeness_reward_func/std": 0.44869658946990965, "step": 195, "step_time": 10.63182978257537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.320833346247673, "completions/max_length": 256.0, "completions/max_terminated_length": 250.8, "completions/mean_length": 182.68334045410157, "completions/mean_terminated_length": 149.52076721191406, "completions/min_length": 60.2, "completions/min_terminated_length": 60.2, "entropy": 1.7882241249084472, "epoch": 0.4, "frac_reward_zero_std": 0.0, "grad_norm": 4.352699279785156, "kl": 0.2817923933267593, "learning_rate": 4.005000000000001e-06, "loss": -0.0338, "num_tokens": 2390743.0, "reward": 1.7947330713272094, "reward_std": 0.3602103054523468, "rewards/politeness_reward_func/mean": 1.7947329998016357, "rewards/politeness_reward_func/std": 0.37249717116355896, "step": 200, "step_time": 10.771137349307537 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.279166679084301, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 175.2791717529297, "completions/mean_terminated_length": 146.66289672851562, "completions/min_length": 54.2, "completions/min_terminated_length": 54.2, "entropy": 1.805497646331787, "epoch": 0.41, "frac_reward_zero_std": 0.0, "grad_norm": 6.236753463745117, "kl": 0.4257870987057686, "learning_rate": 3.980000000000001e-06, "loss": 0.0117, "num_tokens": 2449866.0, "reward": 1.7829206705093383, "reward_std": 0.4116291403770447, "rewards/politeness_reward_func/mean": 1.7829206705093383, "rewards/politeness_reward_func/std": 0.43859742283821107, "step": 205, "step_time": 10.78743917644024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.254166679084301, "completions/max_length": 256.0, "completions/max_terminated_length": 248.2, "completions/mean_length": 171.39584045410157, "completions/mean_terminated_length": 143.78868103027344, "completions/min_length": 52.0, "completions/min_terminated_length": 52.0, "entropy": 1.6819992780685424, "epoch": 0.42, "frac_reward_zero_std": 0.0, "grad_norm": 4.016842842102051, "kl": 0.2606677010655403, "learning_rate": 3.955e-06, "loss": 0.0507, "num_tokens": 2507913.0, "reward": 1.6511780500411988, "reward_std": 0.4131308376789093, "rewards/politeness_reward_func/mean": 1.651178002357483, "rewards/politeness_reward_func/std": 0.43051646947860717, "step": 210, "step_time": 10.86044239550829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2250000059604645, "completions/max_length": 256.0, "completions/max_terminated_length": 249.2, "completions/mean_length": 166.20833740234374, "completions/mean_terminated_length": 139.74808349609376, "completions/min_length": 58.8, "completions/min_terminated_length": 58.8, "entropy": 1.5043580889701844, "epoch": 0.43, "frac_reward_zero_std": 0.0, "grad_norm": 3.5619537830352783, "kl": 0.24042100906372071, "learning_rate": 3.9300000000000005e-06, "loss": -0.0266, "num_tokens": 2565323.0, "reward": 1.6339158296585083, "reward_std": 0.33398547768592834, "rewards/politeness_reward_func/mean": 1.6339158535003662, "rewards/politeness_reward_func/std": 0.37013601660728457, "step": 215, "step_time": 10.860479059070348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2291666731238365, "completions/max_length": 256.0, "completions/max_terminated_length": 238.6, "completions/mean_length": 162.5666717529297, "completions/mean_terminated_length": 137.25745391845703, "completions/min_length": 52.4, "completions/min_terminated_length": 52.4, "entropy": 1.707606041431427, "epoch": 0.44, "frac_reward_zero_std": 0.0, "grad_norm": 4.356007099151611, "kl": 0.275185127556324, "learning_rate": 3.905000000000001e-06, "loss": 0.0385, "num_tokens": 2621347.0, "reward": 1.7018348932266236, "reward_std": 0.3872075915336609, "rewards/politeness_reward_func/mean": 1.7018348217010497, "rewards/politeness_reward_func/std": 0.43171402215957644, "step": 220, "step_time": 10.862962251901626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25000000447034837, "completions/max_length": 256.0, "completions/max_terminated_length": 247.6, "completions/mean_length": 170.00000610351563, "completions/mean_terminated_length": 142.47680053710937, "completions/min_length": 51.4, "completions/min_terminated_length": 51.4, "entropy": 1.7443210363388062, "epoch": 0.45, "frac_reward_zero_std": 0.0, "grad_norm": 4.761515140533447, "kl": 0.2994952619075775, "learning_rate": 3.88e-06, "loss": 0.0254, "num_tokens": 2679443.0, "reward": 1.6937603950500488, "reward_std": 0.42128287851810453, "rewards/politeness_reward_func/mean": 1.6937604188919066, "rewards/politeness_reward_func/std": 0.4645624727010727, "step": 225, "step_time": 10.798750822246074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2750000089406967, "completions/max_length": 256.0, "completions/max_terminated_length": 238.4, "completions/mean_length": 171.07500305175782, "completions/mean_terminated_length": 139.3609603881836, "completions/min_length": 51.4, "completions/min_terminated_length": 51.4, "entropy": 1.7655636191368103, "epoch": 0.46, "frac_reward_zero_std": 0.0, "grad_norm": 4.219749450683594, "kl": 0.30960221886634826, "learning_rate": 3.855e-06, "loss": 0.0374, "num_tokens": 2738565.0, "reward": 1.74673752784729, "reward_std": 0.49648854732513426, "rewards/politeness_reward_func/mean": 1.7467374801635742, "rewards/politeness_reward_func/std": 0.5134409368038177, "step": 230, "step_time": 10.840561749786138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24166667759418486, "completions/max_length": 256.0, "completions/max_terminated_length": 249.4, "completions/mean_length": 170.58750610351564, "completions/mean_terminated_length": 144.16094665527345, "completions/min_length": 49.6, "completions/min_terminated_length": 49.6, "entropy": 1.5835796117782592, "epoch": 0.47, "frac_reward_zero_std": 0.0, "grad_norm": 8.280583381652832, "kl": 0.28185769021511076, "learning_rate": 3.830000000000001e-06, "loss": 0.0433, "num_tokens": 2797282.0, "reward": 1.6517542362213136, "reward_std": 0.4238882720470428, "rewards/politeness_reward_func/mean": 1.6517542362213136, "rewards/politeness_reward_func/std": 0.46576651334762575, "step": 235, "step_time": 10.934168311953545 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1833333358168602, "completions/max_length": 256.0, "completions/max_terminated_length": 248.4, "completions/mean_length": 149.59166870117187, "completions/mean_terminated_length": 125.1296600341797, "completions/min_length": 40.4, "completions/min_terminated_length": 40.4, "entropy": 1.5539511442184448, "epoch": 0.48, "frac_reward_zero_std": 0.0, "grad_norm": 3.9952290058135986, "kl": 0.2962296575307846, "learning_rate": 3.8050000000000004e-06, "loss": 0.0772, "num_tokens": 2850656.0, "reward": 1.5875454187393188, "reward_std": 0.33082354068756104, "rewards/politeness_reward_func/mean": 1.5875454187393188, "rewards/politeness_reward_func/std": 0.4173043370246887, "step": 240, "step_time": 10.826450176537037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.42500001192092896, "completions/max_length": 256.0, "completions/max_terminated_length": 252.4, "completions/mean_length": 203.55417175292968, "completions/mean_terminated_length": 167.60662841796875, "completions/min_length": 68.0, "completions/min_terminated_length": 68.0, "entropy": 1.732195258140564, "epoch": 0.49, "frac_reward_zero_std": 0.0, "grad_norm": 4.070302963256836, "kl": 0.2232723578810692, "learning_rate": 3.7800000000000002e-06, "loss": 0.0646, "num_tokens": 2916917.0, "reward": 1.5489000082015991, "reward_std": 0.44896683692932127, "rewards/politeness_reward_func/mean": 1.5488999366760254, "rewards/politeness_reward_func/std": 0.4965748846530914, "step": 245, "step_time": 10.93318238928914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3375000059604645, "completions/max_length": 256.0, "completions/max_terminated_length": 250.6, "completions/mean_length": 175.0666717529297, "completions/mean_terminated_length": 135.0446517944336, "completions/min_length": 47.4, "completions/min_terminated_length": 47.4, "entropy": 1.7236797571182252, "epoch": 0.5, "frac_reward_zero_std": 0.0, "grad_norm": 15.248210906982422, "kl": 0.31031118631362914, "learning_rate": 3.7550000000000005e-06, "loss": 0.0402, "num_tokens": 2976245.0, "reward": 1.6219825506210328, "reward_std": 0.4284421443939209, "rewards/politeness_reward_func/mean": 1.6219825744628906, "rewards/politeness_reward_func/std": 0.4772031903266907, "step": 250, "step_time": 10.9129977889359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.420833358168602, "completions/max_length": 256.0, "completions/max_terminated_length": 251.8, "completions/mean_length": 198.55000915527344, "completions/mean_terminated_length": 162.55760955810547, "completions/min_length": 71.8, "completions/min_terminated_length": 71.8, "entropy": 1.9296841025352478, "epoch": 0.51, "frac_reward_zero_std": 0.0, "grad_norm": 3.592284679412842, "kl": 0.25315294712781905, "learning_rate": 3.7300000000000003e-06, "loss": 0.0132, "num_tokens": 3041337.0, "reward": 1.6332777500152589, "reward_std": 0.4870866537094116, "rewards/politeness_reward_func/mean": 1.6332777261734008, "rewards/politeness_reward_func/std": 0.5504487872123718, "step": 255, "step_time": 10.853592294454575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.32083334028720856, "completions/max_length": 256.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 184.1916717529297, "completions/mean_terminated_length": 150.5311309814453, "completions/min_length": 62.8, "completions/min_terminated_length": 62.8, "entropy": 1.8794168829917908, "epoch": 0.52, "frac_reward_zero_std": 0.0, "grad_norm": 4.218177795410156, "kl": 0.24962413907051087, "learning_rate": 3.705e-06, "loss": 0.058, "num_tokens": 3102695.0, "reward": 1.7052141666412353, "reward_std": 0.4909538745880127, "rewards/politeness_reward_func/mean": 1.7052141666412353, "rewards/politeness_reward_func/std": 0.5124517560005188, "step": 260, "step_time": 10.939885137230158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.39166667461395266, "completions/max_length": 256.0, "completions/max_terminated_length": 240.8, "completions/mean_length": 182.05833740234374, "completions/mean_terminated_length": 135.85120544433593, "completions/min_length": 47.8, "completions/min_terminated_length": 47.8, "entropy": 1.78580139875412, "epoch": 0.53, "frac_reward_zero_std": 0.0, "grad_norm": 3.726942777633667, "kl": 0.26940242052078245, "learning_rate": 3.6800000000000003e-06, "loss": 0.0129, "num_tokens": 3163621.0, "reward": 1.6984041452407836, "reward_std": 0.4623105704784393, "rewards/politeness_reward_func/mean": 1.69840407371521, "rewards/politeness_reward_func/std": 0.5121756374835968, "step": 265, "step_time": 10.816243136674165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.37500000596046446, "completions/max_length": 256.0, "completions/max_terminated_length": 242.6, "completions/mean_length": 196.69584045410156, "completions/mean_terminated_length": 160.64813537597655, "completions/min_length": 66.4, "completions/min_terminated_length": 66.4, "entropy": 1.7563302516937256, "epoch": 0.54, "frac_reward_zero_std": 0.0, "grad_norm": 4.206515312194824, "kl": 0.3476646825671196, "learning_rate": 3.655e-06, "loss": 0.0463, "num_tokens": 3227836.0, "reward": 1.7140352249145507, "reward_std": 0.41706904768943787, "rewards/politeness_reward_func/mean": 1.714035153388977, "rewards/politeness_reward_func/std": 0.4722850799560547, "step": 270, "step_time": 10.934283661842347 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.43750002086162565, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 200.75416870117186, "completions/mean_terminated_length": 159.0064697265625, "completions/min_length": 59.6, "completions/min_terminated_length": 59.6, "entropy": 1.853849744796753, "epoch": 0.55, "frac_reward_zero_std": 0.0, "grad_norm": 3.3859314918518066, "kl": 0.2895158067345619, "learning_rate": 3.6300000000000004e-06, "loss": -0.0048, "num_tokens": 3292993.0, "reward": 1.8271270751953126, "reward_std": 0.42741702795028685, "rewards/politeness_reward_func/mean": 1.8271270513534545, "rewards/politeness_reward_func/std": 0.47546608448028566, "step": 275, "step_time": 10.752403935790062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.35833334624767305, "completions/max_length": 256.0, "completions/max_terminated_length": 241.8, "completions/mean_length": 189.95000305175782, "completions/mean_terminated_length": 153.42260131835937, "completions/min_length": 57.8, "completions/min_terminated_length": 57.8, "entropy": 1.8662793278694152, "epoch": 0.56, "frac_reward_zero_std": 0.0, "grad_norm": 3.938138484954834, "kl": 0.29017033874988557, "learning_rate": 3.6050000000000002e-06, "loss": -0.0162, "num_tokens": 3355893.0, "reward": 1.7445276975631714, "reward_std": 0.4672766387462616, "rewards/politeness_reward_func/mean": 1.744527554512024, "rewards/politeness_reward_func/std": 0.4982309639453888, "step": 280, "step_time": 10.624229270964861 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.31250000596046446, "completions/max_length": 256.0, "completions/max_terminated_length": 249.4, "completions/mean_length": 190.03333740234376, "completions/mean_terminated_length": 162.05103149414063, "completions/min_length": 65.8, "completions/min_terminated_length": 65.8, "entropy": 1.74795663356781, "epoch": 0.57, "frac_reward_zero_std": 0.0, "grad_norm": 3.650832176208496, "kl": 0.21563028842210769, "learning_rate": 3.58e-06, "loss": 0.0183, "num_tokens": 3418413.0, "reward": 1.6855457305908204, "reward_std": 0.3864980161190033, "rewards/politeness_reward_func/mean": 1.6855456590652467, "rewards/politeness_reward_func/std": 0.4192918837070465, "step": 285, "step_time": 10.76889206841588 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28750001192092894, "completions/max_length": 256.0, "completions/max_terminated_length": 240.4, "completions/mean_length": 174.78333740234376, "completions/mean_terminated_length": 144.00184173583983, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 1.5682233929634095, "epoch": 0.58, "frac_reward_zero_std": 0.0, "grad_norm": 4.823276042938232, "kl": 0.25128297358751295, "learning_rate": 3.5550000000000003e-06, "loss": 0.0429, "num_tokens": 3477785.0, "reward": 1.6693106412887573, "reward_std": 0.3742055296897888, "rewards/politeness_reward_func/mean": 1.6693106412887573, "rewards/politeness_reward_func/std": 0.4070769190788269, "step": 290, "step_time": 10.749481346458197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27083333730697634, "completions/max_length": 256.0, "completions/max_terminated_length": 240.8, "completions/mean_length": 174.8166717529297, "completions/mean_terminated_length": 145.73609619140626, "completions/min_length": 56.6, "completions/min_terminated_length": 56.6, "entropy": 1.7244809031486512, "epoch": 0.59, "frac_reward_zero_std": 0.0, "grad_norm": 5.0669989585876465, "kl": 0.28962416797876356, "learning_rate": 3.53e-06, "loss": 0.0175, "num_tokens": 3537101.0, "reward": 1.6176325321197509, "reward_std": 0.4998854100704193, "rewards/politeness_reward_func/mean": 1.6176324844360352, "rewards/politeness_reward_func/std": 0.5255676746368408, "step": 295, "step_time": 10.83822975307703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.29166668355464936, "completions/max_length": 256.0, "completions/max_terminated_length": 249.2, "completions/mean_length": 176.39166870117188, "completions/mean_terminated_length": 145.56358642578124, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 1.7615143656730652, "epoch": 0.6, "frac_reward_zero_std": 0.0, "grad_norm": 9.959527015686035, "kl": 0.30573274195194244, "learning_rate": 3.505e-06, "loss": 0.0663, "num_tokens": 3597051.0, "reward": 1.7090127229690553, "reward_std": 0.4356145679950714, "rewards/politeness_reward_func/mean": 1.7090126276016235, "rewards/politeness_reward_func/std": 0.458771163225174, "step": 300, "step_time": 10.85788278505206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.325000011920929, "completions/max_length": 256.0, "completions/max_terminated_length": 250.2, "completions/mean_length": 181.22083740234376, "completions/mean_terminated_length": 146.1617431640625, "completions/min_length": 55.0, "completions/min_terminated_length": 55.0, "entropy": 1.9170334219932557, "epoch": 0.61, "frac_reward_zero_std": 0.0, "grad_norm": 4.597347259521484, "kl": 0.2534967973828316, "learning_rate": 3.48e-06, "loss": -0.019, "num_tokens": 3657680.0, "reward": 1.8059730291366578, "reward_std": 0.4023502767086029, "rewards/politeness_reward_func/mean": 1.8059730291366578, "rewards/politeness_reward_func/std": 0.42335187196731566, "step": 305, "step_time": 10.813059192150831 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3458333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 247.6, "completions/mean_length": 192.33333435058594, "completions/mean_terminated_length": 158.2584686279297, "completions/min_length": 61.0, "completions/min_terminated_length": 61.0, "entropy": 1.7168800473213195, "epoch": 0.62, "frac_reward_zero_std": 0.0, "grad_norm": 3.3291268348693848, "kl": 0.30181923806667327, "learning_rate": 3.455e-06, "loss": 0.0438, "num_tokens": 3721168.0, "reward": 1.6650686740875245, "reward_std": 0.4736558377742767, "rewards/politeness_reward_func/mean": 1.6650686979293823, "rewards/politeness_reward_func/std": 0.5267638087272644, "step": 310, "step_time": 10.904363192617893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2708333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 248.4, "completions/mean_length": 178.20833740234374, "completions/mean_terminated_length": 150.5656768798828, "completions/min_length": 52.8, "completions/min_terminated_length": 52.8, "entropy": 1.6762184143066405, "epoch": 0.63, "frac_reward_zero_std": 0.0, "grad_norm": 5.4125494956970215, "kl": 0.3411761596798897, "learning_rate": 3.4300000000000006e-06, "loss": 0.0739, "num_tokens": 3781170.0, "reward": 1.698883295059204, "reward_std": 0.5051921904087067, "rewards/politeness_reward_func/mean": 1.6988832235336304, "rewards/politeness_reward_func/std": 0.5214821577072144, "step": 315, "step_time": 10.870812387019395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3333333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 174.2375061035156, "completions/mean_terminated_length": 133.60752563476564, "completions/min_length": 38.8, "completions/min_terminated_length": 38.8, "entropy": 1.608705174922943, "epoch": 0.64, "frac_reward_zero_std": 0.0, "grad_norm": 4.398674011230469, "kl": 0.3067095875740051, "learning_rate": 3.4050000000000004e-06, "loss": 0.0628, "num_tokens": 3840075.0, "reward": 1.6573886156082154, "reward_std": 0.4343807339668274, "rewards/politeness_reward_func/mean": 1.6573885917663573, "rewards/politeness_reward_func/std": 0.4526840627193451, "step": 320, "step_time": 10.640466004610062 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2666666716337204, "completions/max_length": 256.0, "completions/max_terminated_length": 248.8, "completions/mean_length": 177.06666870117186, "completions/mean_terminated_length": 148.47830200195312, "completions/min_length": 59.6, "completions/min_terminated_length": 59.6, "entropy": 1.832581627368927, "epoch": 0.65, "frac_reward_zero_std": 0.0, "grad_norm": 6.723278522491455, "kl": 0.2649755135178566, "learning_rate": 3.3800000000000007e-06, "loss": -0.026, "num_tokens": 3899579.0, "reward": 1.81795072555542, "reward_std": 0.39153199791908266, "rewards/politeness_reward_func/mean": 1.8179506063461304, "rewards/politeness_reward_func/std": 0.41539669036865234, "step": 325, "step_time": 10.751856955885888 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24583333730697632, "completions/max_length": 256.0, "completions/max_terminated_length": 238.2, "completions/mean_length": 164.57083740234376, "completions/mean_terminated_length": 135.28192138671875, "completions/min_length": 47.6, "completions/min_terminated_length": 47.6, "entropy": 1.6712656617164612, "epoch": 0.66, "frac_reward_zero_std": 0.0, "grad_norm": 4.251152992248535, "kl": 0.2780707836151123, "learning_rate": 3.3550000000000005e-06, "loss": 0.0515, "num_tokens": 3956132.0, "reward": 1.6795947074890136, "reward_std": 0.4304037630558014, "rewards/politeness_reward_func/mean": 1.6795947313308717, "rewards/politeness_reward_func/std": 0.456265515089035, "step": 330, "step_time": 10.65747187435627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27083334028720857, "completions/max_length": 256.0, "completions/max_terminated_length": 250.8, "completions/mean_length": 170.53750610351562, "completions/mean_terminated_length": 139.22777709960937, "completions/min_length": 48.4, "completions/min_terminated_length": 48.4, "entropy": 1.6737295269966126, "epoch": 0.67, "frac_reward_zero_std": 0.0, "grad_norm": 3.981579065322876, "kl": 0.27397735267877577, "learning_rate": 3.3300000000000003e-06, "loss": 0.0586, "num_tokens": 4014149.0, "reward": 1.639498805999756, "reward_std": 0.3987973630428314, "rewards/politeness_reward_func/mean": 1.639498782157898, "rewards/politeness_reward_func/std": 0.4488051861524582, "step": 335, "step_time": 10.64305683746934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24583334028720855, "completions/max_length": 256.0, "completions/max_terminated_length": 247.6, "completions/mean_length": 170.57083435058593, "completions/mean_terminated_length": 143.0279296875, "completions/min_length": 53.6, "completions/min_terminated_length": 53.6, "entropy": 1.8374268651008605, "epoch": 0.68, "frac_reward_zero_std": 0.0, "grad_norm": 5.304769515991211, "kl": 0.2890111759305, "learning_rate": 3.3050000000000005e-06, "loss": 0.0539, "num_tokens": 4072318.0, "reward": 1.827410674095154, "reward_std": 0.34623334407806394, "rewards/politeness_reward_func/mean": 1.8274105072021485, "rewards/politeness_reward_func/std": 0.3635041773319244, "step": 340, "step_time": 10.67860155031085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3791666865348816, "completions/max_length": 256.0, "completions/max_terminated_length": 246.8, "completions/mean_length": 182.3541717529297, "completions/mean_terminated_length": 138.33062896728515, "completions/min_length": 54.4, "completions/min_terminated_length": 54.4, "entropy": 1.8917979717254638, "epoch": 0.69, "frac_reward_zero_std": 0.0, "grad_norm": 3.783524751663208, "kl": 0.3112198129296303, "learning_rate": 3.2800000000000004e-06, "loss": -0.0079, "num_tokens": 4133395.0, "reward": 1.8497365951538085, "reward_std": 0.41627883315086367, "rewards/politeness_reward_func/mean": 1.8497365236282348, "rewards/politeness_reward_func/std": 0.4410283327102661, "step": 345, "step_time": 10.664074825495481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.43750001192092897, "completions/max_length": 256.0, "completions/max_terminated_length": 250.8, "completions/mean_length": 192.87083740234374, "completions/mean_terminated_length": 144.26836700439452, "completions/min_length": 62.4, "completions/min_terminated_length": 62.4, "entropy": 1.9642783045768737, "epoch": 0.7, "frac_reward_zero_std": 0.0, "grad_norm": 3.8300986289978027, "kl": 0.3175328865647316, "learning_rate": 3.255e-06, "loss": 0.033, "num_tokens": 4197028.0, "reward": 1.8747668027877809, "reward_std": 0.485896635055542, "rewards/politeness_reward_func/mean": 1.8747668027877809, "rewards/politeness_reward_func/std": 0.5172119975090027, "step": 350, "step_time": 10.599913079291582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.30416667759418486, "completions/max_length": 256.0, "completions/max_terminated_length": 245.4, "completions/mean_length": 173.94584045410156, "completions/mean_terminated_length": 138.12840881347657, "completions/min_length": 50.4, "completions/min_terminated_length": 50.4, "entropy": 1.6719454765319823, "epoch": 0.71, "frac_reward_zero_std": 0.0, "grad_norm": 3.62943172454834, "kl": 0.354923115670681, "learning_rate": 3.2300000000000004e-06, "loss": 0.04, "num_tokens": 4256759.0, "reward": 1.746033239364624, "reward_std": 0.46536014080047605, "rewards/politeness_reward_func/mean": 1.7460331916809082, "rewards/politeness_reward_func/std": 0.513632845878601, "step": 355, "step_time": 10.574684323370457 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2166666716337204, "completions/max_length": 256.0, "completions/max_terminated_length": 246.4, "completions/mean_length": 164.38750610351562, "completions/mean_terminated_length": 139.19703826904296, "completions/min_length": 58.2, "completions/min_terminated_length": 58.2, "entropy": 1.7568098187446595, "epoch": 0.72, "frac_reward_zero_std": 0.0, "grad_norm": 6.490025997161865, "kl": 0.2866540029644966, "learning_rate": 3.2050000000000002e-06, "loss": -0.0106, "num_tokens": 4313620.0, "reward": 1.7767923593521118, "reward_std": 0.4481996238231659, "rewards/politeness_reward_func/mean": 1.776792311668396, "rewards/politeness_reward_func/std": 0.4569097697734833, "step": 360, "step_time": 10.563734823465348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3833333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 249.8, "completions/mean_length": 192.51250305175782, "completions/mean_terminated_length": 152.48777465820314, "completions/min_length": 54.8, "completions/min_terminated_length": 54.8, "entropy": 1.7603564739227295, "epoch": 0.73, "frac_reward_zero_std": 0.0, "grad_norm": 3.731227159500122, "kl": 0.2559724062681198, "learning_rate": 3.1800000000000005e-06, "loss": 0.0058, "num_tokens": 4377423.0, "reward": 1.7886356353759765, "reward_std": 0.43659440279006956, "rewards/politeness_reward_func/mean": 1.7886356592178345, "rewards/politeness_reward_func/std": 0.4865987479686737, "step": 365, "step_time": 10.596465566009282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3208333432674408, "completions/max_length": 256.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 183.37500610351563, "completions/mean_terminated_length": 150.60201873779297, "completions/min_length": 56.2, "completions/min_terminated_length": 56.2, "entropy": 1.599543821811676, "epoch": 0.74, "frac_reward_zero_std": 0.0, "grad_norm": 4.495799541473389, "kl": 0.26432290226221083, "learning_rate": 3.1550000000000003e-06, "loss": 0.0365, "num_tokens": 4438841.0, "reward": 1.663888192176819, "reward_std": 0.4724281966686249, "rewards/politeness_reward_func/mean": 1.663888168334961, "rewards/politeness_reward_func/std": 0.49500845074653627, "step": 370, "step_time": 10.550004740059375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2833333402872086, "completions/max_length": 256.0, "completions/max_terminated_length": 245.8, "completions/mean_length": 176.43333740234374, "completions/mean_terminated_length": 145.9316650390625, "completions/min_length": 46.8, "completions/min_terminated_length": 46.8, "entropy": 1.649828588962555, "epoch": 0.75, "frac_reward_zero_std": 0.0, "grad_norm": 4.405642509460449, "kl": 0.3023358851671219, "learning_rate": 3.13e-06, "loss": 0.036, "num_tokens": 4498209.0, "reward": 1.7260100841522217, "reward_std": 0.4888251006603241, "rewards/politeness_reward_func/mean": 1.726010012626648, "rewards/politeness_reward_func/std": 0.5257126033306122, "step": 375, "step_time": 10.784504148364068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2750000059604645, "completions/max_length": 256.0, "completions/max_terminated_length": 251.0, "completions/mean_length": 177.4791717529297, "completions/mean_terminated_length": 149.0453887939453, "completions/min_length": 52.6, "completions/min_terminated_length": 52.6, "entropy": 1.669038712978363, "epoch": 0.76, "frac_reward_zero_std": 0.0, "grad_norm": 4.453659534454346, "kl": 0.26074831187725067, "learning_rate": 3.1050000000000003e-06, "loss": 0.0133, "num_tokens": 4558116.0, "reward": 1.7338483333587646, "reward_std": 0.40128968358039857, "rewards/politeness_reward_func/mean": 1.7338482856750488, "rewards/politeness_reward_func/std": 0.4201090157032013, "step": 380, "step_time": 10.85576168820262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3166666775941849, "completions/max_length": 256.0, "completions/max_terminated_length": 248.4, "completions/mean_length": 175.9791687011719, "completions/mean_terminated_length": 141.09287872314454, "completions/min_length": 60.8, "completions/min_terminated_length": 60.8, "entropy": 1.70971120595932, "epoch": 0.77, "frac_reward_zero_std": 0.0, "grad_norm": 4.21511173248291, "kl": 0.33568228632211683, "learning_rate": 3.08e-06, "loss": 0.0539, "num_tokens": 4617679.0, "reward": 1.8021876573562623, "reward_std": 0.39429293274879457, "rewards/politeness_reward_func/mean": 1.8021875858306884, "rewards/politeness_reward_func/std": 0.4176251709461212, "step": 385, "step_time": 10.889974600821734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.24583333879709243, "completions/max_length": 256.0, "completions/max_terminated_length": 250.4, "completions/mean_length": 162.72084045410156, "completions/mean_terminated_length": 133.57429962158204, "completions/min_length": 56.8, "completions/min_terminated_length": 56.8, "entropy": 1.5501957058906555, "epoch": 0.78, "frac_reward_zero_std": 0.0, "grad_norm": 5.835730075836182, "kl": 0.30908732712268827, "learning_rate": 3.0550000000000004e-06, "loss": 0.0817, "num_tokens": 4674108.0, "reward": 1.7301536321640014, "reward_std": 0.44032427072525027, "rewards/politeness_reward_func/mean": 1.7301535844802856, "rewards/politeness_reward_func/std": 0.4708732008934021, "step": 390, "step_time": 10.865944185107946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1791666716337204, "completions/max_length": 256.0, "completions/max_terminated_length": 247.8, "completions/mean_length": 157.25000915527343, "completions/mean_terminated_length": 135.5115005493164, "completions/min_length": 51.2, "completions/min_terminated_length": 51.2, "entropy": 1.6409943580627442, "epoch": 0.79, "frac_reward_zero_std": 0.0, "grad_norm": 6.950301647186279, "kl": 0.36195332258939744, "learning_rate": 3.0300000000000002e-06, "loss": 0.0805, "num_tokens": 4729176.0, "reward": 1.737137007713318, "reward_std": 0.4313428819179535, "rewards/politeness_reward_func/mean": 1.737137007713318, "rewards/politeness_reward_func/std": 0.4497229218482971, "step": 395, "step_time": 10.928832749277353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3291666775941849, "completions/max_length": 256.0, "completions/max_terminated_length": 250.4, "completions/mean_length": 180.15833740234376, "completions/mean_terminated_length": 143.84142608642577, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 1.7865488886833192, "epoch": 0.8, "frac_reward_zero_std": 0.0, "grad_norm": 3.5700058937072754, "kl": 0.2882836848497391, "learning_rate": 3.005e-06, "loss": 0.0432, "num_tokens": 4790094.0, "reward": 1.7929600477218628, "reward_std": 0.4272151470184326, "rewards/politeness_reward_func/mean": 1.792960000038147, "rewards/politeness_reward_func/std": 0.4529218733310699, "step": 400, "step_time": 10.825692503154277 } ], "logging_steps": 5, "max_steps": 1000, "num_input_tokens_seen": 4790094, "num_train_epochs": 2, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }