| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.6666666666666666, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1142.7, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1142.7, |
| "completions/max_terminated_length": 902.5, |
| "completions/mean_length": 314.125, |
| "completions/mean_terminated_length": 290.0325042724609, |
| "completions/min_length": 22.9, |
| "completions/min_terminated_length": 22.9, |
| "epoch": 0.013333333333333334, |
| "frac_reward_zero_std": 0.575, |
| "grad_norm": 0.03041459619998932, |
| "kl": 0.04187124692685681, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0, |
| "num_tokens": 67628.0, |
| "reward": -2.325, |
| "reward_std": 0.23799361288547516, |
| "rewards/reward_func/mean": -2.325, |
| "rewards/reward_func/std": 0.37357009053230283, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1404.2, |
| "completions/clipped_ratio": 0.05, |
| "completions/max_length": 1404.2, |
| "completions/max_terminated_length": 983.1, |
| "completions/mean_length": 350.225, |
| "completions/mean_terminated_length": 286.28453063964844, |
| "completions/min_length": 21.8, |
| "completions/min_terminated_length": 21.8, |
| "epoch": 0.02666666666666667, |
| "frac_reward_zero_std": 0.525, |
| "grad_norm": 77.34902954101562, |
| "kl": 0.17394654396921397, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0002, |
| "num_tokens": 140212.0, |
| "reward": -2.31875, |
| "reward_std": 0.2846687763929367, |
| "rewards/reward_func/mean": -2.31875, |
| "rewards/reward_func/std": 0.4204031050205231, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1156.8, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1156.8, |
| "completions/max_terminated_length": 965.6, |
| "completions/mean_length": 319.85625, |
| "completions/mean_terminated_length": 281.527326965332, |
| "completions/min_length": 21.8, |
| "completions/min_terminated_length": 21.8, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 54.36892318725586, |
| "kl": 0.198792327567935, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0002, |
| "num_tokens": 207321.0, |
| "reward": -2.26875, |
| "reward_std": 0.38884909451007843, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.5729059964418411, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1250.6, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1250.6, |
| "completions/max_terminated_length": 877.9, |
| "completions/mean_length": 273.88125, |
| "completions/mean_terminated_length": 233.0241729736328, |
| "completions/min_length": 19.5, |
| "completions/min_terminated_length": 19.5, |
| "epoch": 0.05333333333333334, |
| "frac_reward_zero_std": 0.525, |
| "grad_norm": 62.96562194824219, |
| "kl": 23.047201949357987, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.023, |
| "num_tokens": 268846.0, |
| "reward": -2.3375, |
| "reward_std": 0.2683012694120407, |
| "rewards/reward_func/mean": -2.3375, |
| "rewards/reward_func/std": 0.3865860253572464, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1285.8, |
| "completions/clipped_ratio": 0.05, |
| "completions/max_length": 1285.8, |
| "completions/max_terminated_length": 984.4, |
| "completions/mean_length": 335.86875, |
| "completions/mean_terminated_length": 274.3717956542969, |
| "completions/min_length": 36.2, |
| "completions/min_terminated_length": 36.2, |
| "epoch": 0.06666666666666667, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 50.94906997680664, |
| "kl": 43.621040439605714, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0436, |
| "num_tokens": 339865.0, |
| "reward": -2.325, |
| "reward_std": 0.29716877937316893, |
| "rewards/reward_func/mean": -2.325, |
| "rewards/reward_func/std": 0.40061245262622835, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1238.2, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1238.2, |
| "completions/max_terminated_length": 1031.5, |
| "completions/mean_length": 331.9375, |
| "completions/mean_terminated_length": 301.5768478393555, |
| "completions/min_length": 28.5, |
| "completions/min_terminated_length": 28.5, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 54.65301513671875, |
| "kl": 18.56655087471008, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0186, |
| "num_tokens": 410479.0, |
| "reward": -2.24375, |
| "reward_std": 0.43427384495735166, |
| "rewards/reward_func/mean": -2.24375, |
| "rewards/reward_func/std": 0.6787217557430267, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1068.7, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1068.7, |
| "completions/max_terminated_length": 728.7, |
| "completions/mean_length": 276.9625, |
| "completions/mean_terminated_length": 235.63208923339843, |
| "completions/min_length": 17.2, |
| "completions/min_terminated_length": 17.2, |
| "epoch": 0.09333333333333334, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 76.15401458740234, |
| "kl": 44.26115598678589, |
| "learning_rate": 4.7888888888888894e-06, |
| "loss": 0.0443, |
| "num_tokens": 472129.0, |
| "reward": -2.25625, |
| "reward_std": 0.3837158679962158, |
| "rewards/reward_func/mean": -2.25625, |
| "rewards/reward_func/std": 0.5701160997152328, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 949.2, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 949.2, |
| "completions/max_terminated_length": 865.0, |
| "completions/mean_length": 280.79375, |
| "completions/mean_terminated_length": 257.9735137939453, |
| "completions/min_length": 29.4, |
| "completions/min_terminated_length": 29.4, |
| "epoch": 0.10666666666666667, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 82.72232818603516, |
| "kl": 26.10418074131012, |
| "learning_rate": 4.677777777777778e-06, |
| "loss": 0.0261, |
| "num_tokens": 534296.0, |
| "reward": -2.225, |
| "reward_std": 0.5031255036592484, |
| "rewards/reward_func/mean": -2.225, |
| "rewards/reward_func/std": 0.6819553226232529, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 947.2, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 947.2, |
| "completions/max_terminated_length": 814.9, |
| "completions/mean_length": 277.56875, |
| "completions/mean_terminated_length": 261.88625183105466, |
| "completions/min_length": 27.9, |
| "completions/min_terminated_length": 27.9, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.325, |
| "grad_norm": 69.23222351074219, |
| "kl": 22.169475841522218, |
| "learning_rate": 4.566666666666667e-06, |
| "loss": 0.0222, |
| "num_tokens": 595867.0, |
| "reward": -2.23125, |
| "reward_std": 0.43445787727832796, |
| "rewards/reward_func/mean": -2.23125, |
| "rewards/reward_func/std": 0.5476110845804214, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1021.2, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1021.2, |
| "completions/max_terminated_length": 858.1, |
| "completions/mean_length": 291.4375, |
| "completions/mean_terminated_length": 265.9560745239258, |
| "completions/min_length": 25.5, |
| "completions/min_terminated_length": 25.5, |
| "epoch": 0.13333333333333333, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 55.19097137451172, |
| "kl": 38.881617593765256, |
| "learning_rate": 4.455555555555555e-06, |
| "loss": 0.0389, |
| "num_tokens": 659557.0, |
| "reward": -2.25, |
| "reward_std": 0.3368710219860077, |
| "rewards/reward_func/mean": -2.25, |
| "rewards/reward_func/std": 0.5741409182548523, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1103.4, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1103.4, |
| "completions/max_terminated_length": 926.0, |
| "completions/mean_length": 286.44375, |
| "completions/mean_terminated_length": 269.99083709716797, |
| "completions/min_length": 23.5, |
| "completions/min_terminated_length": 23.5, |
| "epoch": 0.14666666666666667, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 44.06207275390625, |
| "kl": 10.720393800735474, |
| "learning_rate": 4.344444444444445e-06, |
| "loss": 0.0107, |
| "num_tokens": 722664.0, |
| "reward": -2.3, |
| "reward_std": 0.3366025388240814, |
| "rewards/reward_func/mean": -2.3, |
| "rewards/reward_func/std": 0.4434426426887512, |
| "step": 110 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 762.9, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 762.9, |
| "completions/max_terminated_length": 762.9, |
| "completions/mean_length": 244.29375, |
| "completions/mean_terminated_length": 244.29375, |
| "completions/min_length": 28.0, |
| "completions/min_terminated_length": 28.0, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.525, |
| "grad_norm": 98.8595199584961, |
| "kl": 3.3859562277793884, |
| "learning_rate": 4.233333333333334e-06, |
| "loss": 0.0034, |
| "num_tokens": 778203.0, |
| "reward": -2.28125, |
| "reward_std": 0.3689758062362671, |
| "rewards/reward_func/mean": -2.28125, |
| "rewards/reward_func/std": 0.5127301692962647, |
| "step": 120 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1152.1, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1152.1, |
| "completions/max_terminated_length": 764.6, |
| "completions/mean_length": 291.2875, |
| "completions/mean_terminated_length": 257.580419921875, |
| "completions/min_length": 21.6, |
| "completions/min_terminated_length": 21.6, |
| "epoch": 0.17333333333333334, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 113.74595642089844, |
| "kl": 5.273260289430619, |
| "learning_rate": 4.122222222222222e-06, |
| "loss": 0.0053, |
| "num_tokens": 842517.0, |
| "reward": -2.3, |
| "reward_std": 0.31371368169784547, |
| "rewards/reward_func/mean": -2.3, |
| "rewards/reward_func/std": 0.42710726857185366, |
| "step": 130 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1270.1, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1270.1, |
| "completions/max_terminated_length": 928.7, |
| "completions/mean_length": 317.6875, |
| "completions/mean_terminated_length": 270.543994140625, |
| "completions/min_length": 26.9, |
| "completions/min_terminated_length": 26.9, |
| "epoch": 0.18666666666666668, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 66.77887725830078, |
| "kl": 10.910183477401734, |
| "learning_rate": 4.011111111111111e-06, |
| "loss": 0.0109, |
| "num_tokens": 911419.0, |
| "reward": -2.20625, |
| "reward_std": 0.4734624236822128, |
| "rewards/reward_func/mean": -2.20625, |
| "rewards/reward_func/std": 0.7416160374879837, |
| "step": 140 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 927.5, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 927.5, |
| "completions/max_terminated_length": 674.2, |
| "completions/mean_length": 267.95625, |
| "completions/mean_terminated_length": 243.36625213623046, |
| "completions/min_length": 22.6, |
| "completions/min_terminated_length": 22.6, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 3.4511423110961914, |
| "kl": 7.64983651638031, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0076, |
| "num_tokens": 970928.0, |
| "reward": -2.325, |
| "reward_std": 0.2786566108465195, |
| "rewards/reward_func/mean": -2.325, |
| "rewards/reward_func/std": 0.3554343104362488, |
| "step": 150 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1125.2, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1125.2, |
| "completions/max_terminated_length": 787.6, |
| "completions/mean_length": 282.80625, |
| "completions/mean_terminated_length": 234.86697082519532, |
| "completions/min_length": 17.5, |
| "completions/min_terminated_length": 17.5, |
| "epoch": 0.21333333333333335, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 78.64956665039062, |
| "kl": 31.174254989624025, |
| "learning_rate": 3.7888888888888893e-06, |
| "loss": 0.0312, |
| "num_tokens": 1032913.0, |
| "reward": -2.26875, |
| "reward_std": 0.4256745457649231, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.6542905002832413, |
| "step": 160 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1234.9, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1234.9, |
| "completions/max_terminated_length": 850.8, |
| "completions/mean_length": 293.04375, |
| "completions/mean_terminated_length": 236.12315979003907, |
| "completions/min_length": 17.5, |
| "completions/min_terminated_length": 17.5, |
| "epoch": 0.22666666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 74.24949645996094, |
| "kl": 17.39362871646881, |
| "learning_rate": 3.6777777777777778e-06, |
| "loss": 0.0174, |
| "num_tokens": 1096696.0, |
| "reward": -2.2375, |
| "reward_std": 0.3502942889928818, |
| "rewards/reward_func/mean": -2.2375, |
| "rewards/reward_func/std": 0.5108247697353363, |
| "step": 170 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1094.1, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1094.1, |
| "completions/max_terminated_length": 862.9, |
| "completions/mean_length": 306.3875, |
| "completions/mean_terminated_length": 275.8087554931641, |
| "completions/min_length": 19.2, |
| "completions/min_terminated_length": 19.2, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 114.97906494140625, |
| "kl": 3.446433424949646, |
| "learning_rate": 3.566666666666667e-06, |
| "loss": 0.0034, |
| "num_tokens": 1163210.0, |
| "reward": -2.26875, |
| "reward_std": 0.36489740014076233, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.5350149899721146, |
| "step": 180 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1129.5, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1129.5, |
| "completions/max_terminated_length": 810.2, |
| "completions/mean_length": 286.28125, |
| "completions/mean_terminated_length": 238.94435272216796, |
| "completions/min_length": 23.6, |
| "completions/min_terminated_length": 23.6, |
| "epoch": 0.25333333333333335, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 103.09403228759766, |
| "kl": 20.17943916320801, |
| "learning_rate": 3.455555555555556e-06, |
| "loss": 0.0202, |
| "num_tokens": 1226187.0, |
| "reward": -2.35625, |
| "reward_std": 0.26175799071788786, |
| "rewards/reward_func/mean": -2.35625, |
| "rewards/reward_func/std": 0.3903583735227585, |
| "step": 190 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1171.8, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1171.8, |
| "completions/max_terminated_length": 780.4, |
| "completions/mean_length": 289.95, |
| "completions/mean_terminated_length": 242.0620880126953, |
| "completions/min_length": 21.9, |
| "completions/min_terminated_length": 21.9, |
| "epoch": 0.26666666666666666, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 162.74258422851562, |
| "kl": 39.28117027282715, |
| "learning_rate": 3.3444444444444445e-06, |
| "loss": 0.0393, |
| "num_tokens": 1290015.0, |
| "reward": -2.225, |
| "reward_std": 0.42369674146175385, |
| "rewards/reward_func/mean": -2.225, |
| "rewards/reward_func/std": 0.6618488401174545, |
| "step": 200 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 953.0, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 953.0, |
| "completions/max_terminated_length": 883.8, |
| "completions/mean_length": 306.9625, |
| "completions/mean_terminated_length": 285.81047973632815, |
| "completions/min_length": 20.9, |
| "completions/min_terminated_length": 20.9, |
| "epoch": 0.28, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 56.53829574584961, |
| "kl": 4.3735075235366825, |
| "learning_rate": 3.2333333333333334e-06, |
| "loss": 0.0044, |
| "num_tokens": 1355825.0, |
| "reward": -2.30625, |
| "reward_std": 0.2741025388240814, |
| "rewards/reward_func/mean": -2.30625, |
| "rewards/reward_func/std": 0.3696858823299408, |
| "step": 210 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1269.1, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1269.1, |
| "completions/max_terminated_length": 1032.2, |
| "completions/mean_length": 292.15625, |
| "completions/mean_terminated_length": 260.2483383178711, |
| "completions/min_length": 17.8, |
| "completions/min_terminated_length": 17.8, |
| "epoch": 0.29333333333333333, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 74.96534729003906, |
| "kl": 16.07765164375305, |
| "learning_rate": 3.1222222222222228e-06, |
| "loss": 0.0161, |
| "num_tokens": 1418942.0, |
| "reward": -2.26875, |
| "reward_std": 0.35119174122810365, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.48719866275787355, |
| "step": 220 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1035.2, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1035.2, |
| "completions/max_terminated_length": 911.0, |
| "completions/mean_length": 256.95, |
| "completions/mean_terminated_length": 233.14875335693358, |
| "completions/min_length": 8.9, |
| "completions/min_terminated_length": 8.9, |
| "epoch": 0.30666666666666664, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 36.64857482910156, |
| "kl": 14.746024799346923, |
| "learning_rate": 3.0111111111111113e-06, |
| "loss": 0.0147, |
| "num_tokens": 1478462.0, |
| "reward": -2.225, |
| "reward_std": 0.4513490855693817, |
| "rewards/reward_func/mean": -2.225, |
| "rewards/reward_func/std": 0.6648978263139724, |
| "step": 230 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1116.4, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1116.4, |
| "completions/max_terminated_length": 758.3, |
| "completions/mean_length": 295.925, |
| "completions/mean_terminated_length": 255.58333892822264, |
| "completions/min_length": 21.1, |
| "completions/min_terminated_length": 21.1, |
| "epoch": 0.32, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 84.84314727783203, |
| "kl": 5.60067400932312, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0056, |
| "num_tokens": 1543490.0, |
| "reward": -2.26875, |
| "reward_std": 0.35537946224212646, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.4750110626220703, |
| "step": 240 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1023.8, |
| "completions/clipped_ratio": 0.00625, |
| "completions/max_length": 1023.8, |
| "completions/max_terminated_length": 945.7, |
| "completions/mean_length": 266.0375, |
| "completions/mean_terminated_length": 257.5279174804688, |
| "completions/min_length": 17.0, |
| "completions/min_terminated_length": 17.0, |
| "epoch": 0.3333333333333333, |
| "frac_reward_zero_std": 0.575, |
| "grad_norm": 81.47051239013672, |
| "kl": 18.14775743484497, |
| "learning_rate": 2.788888888888889e-06, |
| "loss": 0.0181, |
| "num_tokens": 1603676.0, |
| "reward": -2.3375, |
| "reward_std": 0.2645611822605133, |
| "rewards/reward_func/mean": -2.3375, |
| "rewards/reward_func/std": 0.4000595390796661, |
| "step": 250 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1079.9, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1079.9, |
| "completions/max_terminated_length": 947.9, |
| "completions/mean_length": 340.54375, |
| "completions/mean_terminated_length": 318.7557769775391, |
| "completions/min_length": 26.2, |
| "completions/min_terminated_length": 26.2, |
| "epoch": 0.3466666666666667, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 54.74143600463867, |
| "kl": 20.487924671173097, |
| "learning_rate": 2.677777777777778e-06, |
| "loss": 0.0205, |
| "num_tokens": 1675795.0, |
| "reward": -2.3125, |
| "reward_std": 0.2760362923145294, |
| "rewards/reward_func/mean": -2.3125, |
| "rewards/reward_func/std": 0.3895877480506897, |
| "step": 260 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1192.1, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1192.1, |
| "completions/max_terminated_length": 858.8, |
| "completions/mean_length": 302.7625, |
| "completions/mean_terminated_length": 246.45244750976562, |
| "completions/min_length": 12.2, |
| "completions/min_terminated_length": 12.2, |
| "epoch": 0.36, |
| "frac_reward_zero_std": 0.65, |
| "grad_norm": 0.1731022149324417, |
| "kl": 7.915937983989716, |
| "learning_rate": 2.566666666666667e-06, |
| "loss": 0.0079, |
| "num_tokens": 1740341.0, |
| "reward": -2.25, |
| "reward_std": 0.4122216373682022, |
| "rewards/reward_func/mean": -2.25, |
| "rewards/reward_func/std": 0.6856489062309266, |
| "step": 270 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1110.5, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1110.5, |
| "completions/max_terminated_length": 891.8, |
| "completions/mean_length": 309.71875, |
| "completions/mean_terminated_length": 286.5383361816406, |
| "completions/min_length": 24.0, |
| "completions/min_terminated_length": 24.0, |
| "epoch": 0.37333333333333335, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 122.04984283447266, |
| "kl": 3.3861916780471804, |
| "learning_rate": 2.455555555555556e-06, |
| "loss": 0.0034, |
| "num_tokens": 1806904.0, |
| "reward": -2.2375, |
| "reward_std": 0.4326493889093399, |
| "rewards/reward_func/mean": -2.2375, |
| "rewards/reward_func/std": 0.6847750276327134, |
| "step": 280 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1069.2, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1069.2, |
| "completions/max_terminated_length": 853.0, |
| "completions/mean_length": 301.1875, |
| "completions/mean_terminated_length": 269.63209075927733, |
| "completions/min_length": 18.7, |
| "completions/min_terminated_length": 18.7, |
| "epoch": 0.38666666666666666, |
| "frac_reward_zero_std": 0.575, |
| "grad_norm": 25.8874454498291, |
| "kl": 14.604678845405578, |
| "learning_rate": 2.3444444444444448e-06, |
| "loss": 0.0146, |
| "num_tokens": 1872494.0, |
| "reward": -2.36875, |
| "reward_std": 0.22780319154262543, |
| "rewards/reward_func/mean": -2.36875, |
| "rewards/reward_func/std": 0.3246472626924515, |
| "step": 290 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1228.1, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1228.1, |
| "completions/max_terminated_length": 817.4, |
| "completions/mean_length": 287.74375, |
| "completions/mean_terminated_length": 238.6010757446289, |
| "completions/min_length": 20.4, |
| "completions/min_terminated_length": 20.4, |
| "epoch": 0.4, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 50.89199447631836, |
| "kl": 12.896457767486572, |
| "learning_rate": 2.2333333333333333e-06, |
| "loss": 0.0129, |
| "num_tokens": 1936649.0, |
| "reward": -2.26875, |
| "reward_std": 0.3961044609546661, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.5396905303001404, |
| "step": 300 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1038.1, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1038.1, |
| "completions/max_terminated_length": 889.8, |
| "completions/mean_length": 258.28125, |
| "completions/mean_terminated_length": 241.79458618164062, |
| "completions/min_length": 17.1, |
| "completions/min_terminated_length": 17.1, |
| "epoch": 0.41333333333333333, |
| "frac_reward_zero_std": 0.65, |
| "grad_norm": 48.88945770263672, |
| "kl": 4.541426551342011, |
| "learning_rate": 2.1222222222222226e-06, |
| "loss": 0.0045, |
| "num_tokens": 1994798.0, |
| "reward": -2.4, |
| "reward_std": 0.17886751294136047, |
| "rewards/reward_func/mean": -2.4, |
| "rewards/reward_func/std": 0.2989355862140656, |
| "step": 310 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1238.9, |
| "completions/clipped_ratio": 0.05625, |
| "completions/max_length": 1238.9, |
| "completions/max_terminated_length": 795.1, |
| "completions/mean_length": 315.44375, |
| "completions/mean_terminated_length": 243.35834197998048, |
| "completions/min_length": 15.4, |
| "completions/min_terminated_length": 15.4, |
| "epoch": 0.4266666666666667, |
| "frac_reward_zero_std": 0.525, |
| "grad_norm": 54.9058837890625, |
| "kl": 4.1825002312660216, |
| "learning_rate": 2.011111111111111e-06, |
| "loss": 0.0042, |
| "num_tokens": 2062905.0, |
| "reward": -2.29375, |
| "reward_std": 0.34910253882408143, |
| "rewards/reward_func/mean": -2.29375, |
| "rewards/reward_func/std": 0.538089805841446, |
| "step": 320 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1048.0, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1048.0, |
| "completions/max_terminated_length": 917.3, |
| "completions/mean_length": 277.575, |
| "completions/mean_terminated_length": 261.64583587646484, |
| "completions/min_length": 19.7, |
| "completions/min_terminated_length": 19.7, |
| "epoch": 0.44, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 127.28370666503906, |
| "kl": 4.280037522315979, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0043, |
| "num_tokens": 2124265.0, |
| "reward": -2.2375, |
| "reward_std": 0.44838539361953733, |
| "rewards/reward_func/mean": -2.2375, |
| "rewards/reward_func/std": 0.6401138246059418, |
| "step": 330 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 927.5, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 927.5, |
| "completions/max_terminated_length": 798.2, |
| "completions/mean_length": 283.7125, |
| "completions/mean_terminated_length": 267.95792083740236, |
| "completions/min_length": 30.1, |
| "completions/min_terminated_length": 30.1, |
| "epoch": 0.4533333333333333, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 146.22303771972656, |
| "kl": 30.70813355445862, |
| "learning_rate": 1.788888888888889e-06, |
| "loss": 0.0307, |
| "num_tokens": 2187843.0, |
| "reward": -2.25625, |
| "reward_std": 0.39784331917762755, |
| "rewards/reward_func/mean": -2.25625, |
| "rewards/reward_func/std": 0.5563936829566956, |
| "step": 340 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1043.1, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1043.1, |
| "completions/max_terminated_length": 899.7, |
| "completions/mean_length": 307.55625, |
| "completions/mean_terminated_length": 284.3370895385742, |
| "completions/min_length": 24.6, |
| "completions/min_terminated_length": 24.6, |
| "epoch": 0.4666666666666667, |
| "frac_reward_zero_std": 0.55, |
| "grad_norm": 32.36188888549805, |
| "kl": 64.3428849220276, |
| "learning_rate": 1.6777777777777779e-06, |
| "loss": 0.0643, |
| "num_tokens": 2254616.0, |
| "reward": -2.26875, |
| "reward_std": 0.3944930166006088, |
| "rewards/reward_func/mean": -2.26875, |
| "rewards/reward_func/std": 0.6516756772994995, |
| "step": 350 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1245.1, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1245.1, |
| "completions/max_terminated_length": 964.9, |
| "completions/mean_length": 309.43125, |
| "completions/mean_terminated_length": 261.41810150146483, |
| "completions/min_length": 27.0, |
| "completions/min_terminated_length": 27.0, |
| "epoch": 0.48, |
| "frac_reward_zero_std": 0.5, |
| "grad_norm": 38.63202667236328, |
| "kl": 21.073752880096436, |
| "learning_rate": 1.566666666666667e-06, |
| "loss": 0.0211, |
| "num_tokens": 2320821.0, |
| "reward": -2.275, |
| "reward_std": 0.39782515466213225, |
| "rewards/reward_func/mean": -2.275, |
| "rewards/reward_func/std": 0.5283936500549317, |
| "step": 360 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1342.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1342.0, |
| "completions/max_terminated_length": 1053.9, |
| "completions/mean_length": 318.3625, |
| "completions/mean_terminated_length": 278.52917327880857, |
| "completions/min_length": 21.3, |
| "completions/min_terminated_length": 21.3, |
| "epoch": 0.49333333333333335, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 45.37665557861328, |
| "kl": 4.191655397415161, |
| "learning_rate": 1.4555555555555557e-06, |
| "loss": 0.0042, |
| "num_tokens": 2388327.0, |
| "reward": -2.24375, |
| "reward_std": 0.40966878533363343, |
| "rewards/reward_func/mean": -2.24375, |
| "rewards/reward_func/std": 0.5739072471857071, |
| "step": 370 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1146.1, |
| "completions/clipped_ratio": 0.05, |
| "completions/max_length": 1146.1, |
| "completions/max_terminated_length": 806.6, |
| "completions/mean_length": 309.30625, |
| "completions/mean_terminated_length": 246.01378784179687, |
| "completions/min_length": 14.3, |
| "completions/min_terminated_length": 14.3, |
| "epoch": 0.5066666666666667, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 51.08633804321289, |
| "kl": 10.670231366157532, |
| "learning_rate": 1.3444444444444446e-06, |
| "loss": 0.0107, |
| "num_tokens": 2454412.0, |
| "reward": -2.28125, |
| "reward_std": 0.39523502588272097, |
| "rewards/reward_func/mean": -2.28125, |
| "rewards/reward_func/std": 0.5278610855340957, |
| "step": 380 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1179.6, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1179.6, |
| "completions/max_terminated_length": 819.6, |
| "completions/mean_length": 301.68125, |
| "completions/mean_terminated_length": 261.32500610351565, |
| "completions/min_length": 18.2, |
| "completions/min_terminated_length": 18.2, |
| "epoch": 0.52, |
| "frac_reward_zero_std": 0.525, |
| "grad_norm": 106.32160949707031, |
| "kl": 21.585847473144533, |
| "learning_rate": 1.2333333333333335e-06, |
| "loss": 0.0216, |
| "num_tokens": 2519509.0, |
| "reward": -2.29375, |
| "reward_std": 0.3345831483602524, |
| "rewards/reward_func/mean": -2.29375, |
| "rewards/reward_func/std": 0.5108768463134765, |
| "step": 390 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 995.2, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 995.2, |
| "completions/max_terminated_length": 679.8, |
| "completions/mean_length": 274.5875, |
| "completions/mean_terminated_length": 232.36595611572267, |
| "completions/min_length": 17.9, |
| "completions/min_terminated_length": 17.9, |
| "epoch": 0.5333333333333333, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 75.17499542236328, |
| "kl": 9.158978390693665, |
| "learning_rate": 1.1222222222222222e-06, |
| "loss": 0.0092, |
| "num_tokens": 2579431.0, |
| "reward": -2.1875, |
| "reward_std": 0.4814758121967316, |
| "rewards/reward_func/mean": -2.1875, |
| "rewards/reward_func/std": 0.6688827067613602, |
| "step": 400 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1195.5, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1195.5, |
| "completions/max_terminated_length": 997.3, |
| "completions/mean_length": 286.84375, |
| "completions/mean_terminated_length": 254.71875457763673, |
| "completions/min_length": 15.5, |
| "completions/min_terminated_length": 15.5, |
| "epoch": 0.5466666666666666, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 95.73103332519531, |
| "kl": 8.273300886154175, |
| "learning_rate": 1.0111111111111111e-06, |
| "loss": 0.0083, |
| "num_tokens": 2641798.0, |
| "reward": -2.25, |
| "reward_std": 0.39309037029743193, |
| "rewards/reward_func/mean": -2.25, |
| "rewards/reward_func/std": 0.5219055712223053, |
| "step": 410 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 974.9, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 974.9, |
| "completions/max_terminated_length": 869.7, |
| "completions/mean_length": 261.23125, |
| "completions/mean_terminated_length": 244.99042053222655, |
| "completions/min_length": 16.0, |
| "completions/min_terminated_length": 16.0, |
| "epoch": 0.56, |
| "frac_reward_zero_std": 0.375, |
| "grad_norm": 99.72528839111328, |
| "kl": 6.985241413116455, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.007, |
| "num_tokens": 2700271.0, |
| "reward": -2.23125, |
| "reward_std": 0.4483605235815048, |
| "rewards/reward_func/mean": -2.23125, |
| "rewards/reward_func/std": 0.5773171722888947, |
| "step": 420 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 964.0, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 964.0, |
| "completions/max_terminated_length": 776.9, |
| "completions/mean_length": 280.24375, |
| "completions/mean_terminated_length": 263.74458465576174, |
| "completions/min_length": 14.2, |
| "completions/min_terminated_length": 14.2, |
| "epoch": 0.5733333333333334, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 124.6417007446289, |
| "kl": 11.635133910179139, |
| "learning_rate": 7.888888888888889e-07, |
| "loss": 0.0116, |
| "num_tokens": 2762894.0, |
| "reward": -2.1875, |
| "reward_std": 0.5182177871465683, |
| "rewards/reward_func/mean": -2.1875, |
| "rewards/reward_func/std": 0.7841762661933899, |
| "step": 430 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1059.5, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1059.5, |
| "completions/max_terminated_length": 908.1, |
| "completions/mean_length": 311.675, |
| "completions/mean_terminated_length": 289.6191696166992, |
| "completions/min_length": 21.8, |
| "completions/min_terminated_length": 21.8, |
| "epoch": 0.5866666666666667, |
| "frac_reward_zero_std": 0.475, |
| "grad_norm": 132.1418914794922, |
| "kl": 115.09008512496948, |
| "learning_rate": 6.777777777777779e-07, |
| "loss": 0.1151, |
| "num_tokens": 2830866.0, |
| "reward": -2.35, |
| "reward_std": 0.26443375647068024, |
| "rewards/reward_func/mean": -2.35, |
| "rewards/reward_func/std": 0.35543124079704286, |
| "step": 440 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1278.8, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 1278.8, |
| "completions/max_terminated_length": 777.6, |
| "completions/mean_length": 327.28125, |
| "completions/mean_terminated_length": 247.8854248046875, |
| "completions/min_length": 25.5, |
| "completions/min_terminated_length": 25.5, |
| "epoch": 0.6, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 121.0811538696289, |
| "kl": 21.60447826385498, |
| "learning_rate": 5.666666666666667e-07, |
| "loss": 0.0216, |
| "num_tokens": 2900423.0, |
| "reward": -2.24375, |
| "reward_std": 0.4444930166006088, |
| "rewards/reward_func/mean": -2.24375, |
| "rewards/reward_func/std": 0.6568617165088654, |
| "step": 450 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1040.1, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1040.1, |
| "completions/max_terminated_length": 760.2, |
| "completions/mean_length": 274.375, |
| "completions/mean_terminated_length": 249.7200035095215, |
| "completions/min_length": 19.1, |
| "completions/min_terminated_length": 19.1, |
| "epoch": 0.6133333333333333, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 38.948699951171875, |
| "kl": 14.533638858795166, |
| "learning_rate": 4.5555555555555563e-07, |
| "loss": 0.0145, |
| "num_tokens": 2961131.0, |
| "reward": -2.3125, |
| "reward_std": 0.31917070150375365, |
| "rewards/reward_func/mean": -2.3125, |
| "rewards/reward_func/std": 0.4087942600250244, |
| "step": 460 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1305.9, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1305.9, |
| "completions/max_terminated_length": 1012.3, |
| "completions/mean_length": 315.675, |
| "completions/mean_terminated_length": 259.3705429077148, |
| "completions/min_length": 18.0, |
| "completions/min_terminated_length": 18.0, |
| "epoch": 0.6266666666666667, |
| "frac_reward_zero_std": 0.35, |
| "grad_norm": 60.82999038696289, |
| "kl": 27.974109888076782, |
| "learning_rate": 3.444444444444445e-07, |
| "loss": 0.028, |
| "num_tokens": 3028239.0, |
| "reward": -2.24375, |
| "reward_std": 0.3991025358438492, |
| "rewards/reward_func/mean": -2.24375, |
| "rewards/reward_func/std": 0.5254836320877075, |
| "step": 470 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 984.0, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 984.0, |
| "completions/max_terminated_length": 822.9, |
| "completions/mean_length": 262.79375, |
| "completions/mean_terminated_length": 246.77666931152345, |
| "completions/min_length": 21.2, |
| "completions/min_terminated_length": 21.2, |
| "epoch": 0.64, |
| "frac_reward_zero_std": 0.425, |
| "grad_norm": 104.03584289550781, |
| "kl": 13.309299659729003, |
| "learning_rate": 2.3333333333333336e-07, |
| "loss": 0.0133, |
| "num_tokens": 3087690.0, |
| "reward": -2.25625, |
| "reward_std": 0.4189758121967316, |
| "rewards/reward_func/mean": -2.25625, |
| "rewards/reward_func/std": 0.5928926885128021, |
| "step": 480 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 999.4, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 999.4, |
| "completions/max_terminated_length": 889.3, |
| "completions/mean_length": 273.45625, |
| "completions/mean_terminated_length": 257.5420852661133, |
| "completions/min_length": 34.7, |
| "completions/min_terminated_length": 34.7, |
| "epoch": 0.6533333333333333, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 56.064544677734375, |
| "kl": 25.44476251602173, |
| "learning_rate": 1.2222222222222225e-07, |
| "loss": 0.0254, |
| "num_tokens": 3148447.0, |
| "reward": -2.3625, |
| "reward_std": 0.23339140117168428, |
| "rewards/reward_func/mean": -2.3625, |
| "rewards/reward_func/std": 0.37498362064361573, |
| "step": 490 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1092.4, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1092.4, |
| "completions/max_terminated_length": 675.7, |
| "completions/mean_length": 294.625, |
| "completions/mean_terminated_length": 246.04232788085938, |
| "completions/min_length": 19.8, |
| "completions/min_terminated_length": 19.8, |
| "epoch": 0.6666666666666666, |
| "frac_reward_zero_std": 0.45, |
| "grad_norm": 122.21601104736328, |
| "kl": 17.26050043106079, |
| "learning_rate": 1.1111111111111112e-08, |
| "loss": 0.0173, |
| "num_tokens": 3212059.0, |
| "reward": -2.2125, |
| "reward_std": 0.4100087761878967, |
| "rewards/reward_func/mean": -2.2125, |
| "rewards/reward_func/std": 0.5889766067266464, |
| "step": 500 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 500, |
| "num_input_tokens_seen": 3212059, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
