{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.6666666666666666, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1142.7, "completions/clipped_ratio": 0.01875, "completions/max_length": 1142.7, "completions/max_terminated_length": 902.5, "completions/mean_length": 314.125, "completions/mean_terminated_length": 290.0325042724609, "completions/min_length": 22.9, "completions/min_terminated_length": 22.9, "epoch": 0.013333333333333334, "frac_reward_zero_std": 0.575, "grad_norm": 0.03041459619998932, "kl": 0.04187124692685681, "learning_rate": 9.000000000000001e-07, "loss": 0.0, "num_tokens": 67628.0, "reward": -2.325, "reward_std": 0.23799361288547516, "rewards/reward_func/mean": -2.325, "rewards/reward_func/std": 0.37357009053230283, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1404.2, "completions/clipped_ratio": 0.05, "completions/max_length": 1404.2, "completions/max_terminated_length": 983.1, "completions/mean_length": 350.225, "completions/mean_terminated_length": 286.28453063964844, "completions/min_length": 21.8, "completions/min_terminated_length": 21.8, "epoch": 0.02666666666666667, "frac_reward_zero_std": 0.525, "grad_norm": 77.34902954101562, "kl": 0.17394654396921397, "learning_rate": 1.9000000000000002e-06, "loss": 0.0002, "num_tokens": 140212.0, "reward": -2.31875, "reward_std": 0.2846687763929367, "rewards/reward_func/mean": -2.31875, "rewards/reward_func/std": 0.4204031050205231, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1156.8, "completions/clipped_ratio": 0.03125, "completions/max_length": 1156.8, "completions/max_terminated_length": 965.6, "completions/mean_length": 319.85625, "completions/mean_terminated_length": 281.527326965332, "completions/min_length": 21.8, "completions/min_terminated_length": 21.8, "epoch": 0.04, "frac_reward_zero_std": 0.5, "grad_norm": 54.36892318725586, "kl": 0.198792327567935, "learning_rate": 2.9e-06, "loss": 0.0002, "num_tokens": 207321.0, "reward": -2.26875, "reward_std": 0.38884909451007843, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.5729059964418411, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1250.6, "completions/clipped_ratio": 0.03125, "completions/max_length": 1250.6, "completions/max_terminated_length": 877.9, "completions/mean_length": 273.88125, "completions/mean_terminated_length": 233.0241729736328, "completions/min_length": 19.5, "completions/min_terminated_length": 19.5, "epoch": 0.05333333333333334, "frac_reward_zero_std": 0.525, "grad_norm": 62.96562194824219, "kl": 23.047201949357987, "learning_rate": 3.900000000000001e-06, "loss": 0.023, "num_tokens": 268846.0, "reward": -2.3375, "reward_std": 0.2683012694120407, "rewards/reward_func/mean": -2.3375, "rewards/reward_func/std": 0.3865860253572464, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1285.8, "completions/clipped_ratio": 0.05, "completions/max_length": 1285.8, "completions/max_terminated_length": 984.4, "completions/mean_length": 335.86875, "completions/mean_terminated_length": 274.3717956542969, "completions/min_length": 36.2, "completions/min_terminated_length": 36.2, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.5, "grad_norm": 50.94906997680664, "kl": 43.621040439605714, "learning_rate": 4.9000000000000005e-06, "loss": 0.0436, "num_tokens": 339865.0, "reward": -2.325, "reward_std": 0.29716877937316893, "rewards/reward_func/mean": -2.325, "rewards/reward_func/std": 0.40061245262622835, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1238.2, "completions/clipped_ratio": 0.025, "completions/max_length": 1238.2, "completions/max_terminated_length": 1031.5, "completions/mean_length": 331.9375, "completions/mean_terminated_length": 301.5768478393555, "completions/min_length": 28.5, "completions/min_terminated_length": 28.5, "epoch": 0.08, "frac_reward_zero_std": 0.475, "grad_norm": 54.65301513671875, "kl": 18.56655087471008, "learning_rate": 4.9000000000000005e-06, "loss": 0.0186, "num_tokens": 410479.0, "reward": -2.24375, "reward_std": 0.43427384495735166, "rewards/reward_func/mean": -2.24375, "rewards/reward_func/std": 0.6787217557430267, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1068.7, "completions/clipped_ratio": 0.03125, "completions/max_length": 1068.7, "completions/max_terminated_length": 728.7, "completions/mean_length": 276.9625, "completions/mean_terminated_length": 235.63208923339843, "completions/min_length": 17.2, "completions/min_terminated_length": 17.2, "epoch": 0.09333333333333334, "frac_reward_zero_std": 0.475, "grad_norm": 76.15401458740234, "kl": 44.26115598678589, "learning_rate": 4.7888888888888894e-06, "loss": 0.0443, "num_tokens": 472129.0, "reward": -2.25625, "reward_std": 0.3837158679962158, "rewards/reward_func/mean": -2.25625, "rewards/reward_func/std": 0.5701160997152328, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 949.2, "completions/clipped_ratio": 0.01875, "completions/max_length": 949.2, "completions/max_terminated_length": 865.0, "completions/mean_length": 280.79375, "completions/mean_terminated_length": 257.9735137939453, "completions/min_length": 29.4, "completions/min_terminated_length": 29.4, "epoch": 0.10666666666666667, "frac_reward_zero_std": 0.45, "grad_norm": 82.72232818603516, "kl": 26.10418074131012, "learning_rate": 4.677777777777778e-06, "loss": 0.0261, "num_tokens": 534296.0, "reward": -2.225, "reward_std": 0.5031255036592484, "rewards/reward_func/mean": -2.225, "rewards/reward_func/std": 0.6819553226232529, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 947.2, "completions/clipped_ratio": 0.0125, "completions/max_length": 947.2, "completions/max_terminated_length": 814.9, "completions/mean_length": 277.56875, "completions/mean_terminated_length": 261.88625183105466, "completions/min_length": 27.9, "completions/min_terminated_length": 27.9, "epoch": 0.12, "frac_reward_zero_std": 0.325, "grad_norm": 69.23222351074219, "kl": 22.169475841522218, "learning_rate": 4.566666666666667e-06, "loss": 0.0222, "num_tokens": 595867.0, "reward": -2.23125, "reward_std": 0.43445787727832796, "rewards/reward_func/mean": -2.23125, "rewards/reward_func/std": 0.5476110845804214, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1021.2, "completions/clipped_ratio": 0.01875, "completions/max_length": 1021.2, "completions/max_terminated_length": 858.1, "completions/mean_length": 291.4375, "completions/mean_terminated_length": 265.9560745239258, "completions/min_length": 25.5, "completions/min_terminated_length": 25.5, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.6, "grad_norm": 55.19097137451172, "kl": 38.881617593765256, "learning_rate": 4.455555555555555e-06, "loss": 0.0389, "num_tokens": 659557.0, "reward": -2.25, "reward_std": 0.3368710219860077, "rewards/reward_func/mean": -2.25, "rewards/reward_func/std": 0.5741409182548523, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1103.4, "completions/clipped_ratio": 0.0125, "completions/max_length": 1103.4, "completions/max_terminated_length": 926.0, "completions/mean_length": 286.44375, "completions/mean_terminated_length": 269.99083709716797, "completions/min_length": 23.5, "completions/min_terminated_length": 23.5, "epoch": 0.14666666666666667, "frac_reward_zero_std": 0.425, "grad_norm": 44.06207275390625, "kl": 10.720393800735474, "learning_rate": 4.344444444444445e-06, "loss": 0.0107, "num_tokens": 722664.0, "reward": -2.3, "reward_std": 0.3366025388240814, "rewards/reward_func/mean": -2.3, "rewards/reward_func/std": 0.4434426426887512, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 762.9, "completions/clipped_ratio": 0.0, "completions/max_length": 762.9, "completions/max_terminated_length": 762.9, "completions/mean_length": 244.29375, "completions/mean_terminated_length": 244.29375, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.16, "frac_reward_zero_std": 0.525, "grad_norm": 98.8595199584961, "kl": 3.3859562277793884, "learning_rate": 4.233333333333334e-06, "loss": 0.0034, "num_tokens": 778203.0, "reward": -2.28125, "reward_std": 0.3689758062362671, "rewards/reward_func/mean": -2.28125, "rewards/reward_func/std": 0.5127301692962647, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1152.1, "completions/clipped_ratio": 0.025, "completions/max_length": 1152.1, "completions/max_terminated_length": 764.6, "completions/mean_length": 291.2875, "completions/mean_terminated_length": 257.580419921875, "completions/min_length": 21.6, "completions/min_terminated_length": 21.6, "epoch": 0.17333333333333334, "frac_reward_zero_std": 0.425, "grad_norm": 113.74595642089844, "kl": 5.273260289430619, "learning_rate": 4.122222222222222e-06, "loss": 0.0053, "num_tokens": 842517.0, "reward": -2.3, "reward_std": 0.31371368169784547, "rewards/reward_func/mean": -2.3, "rewards/reward_func/std": 0.42710726857185366, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1270.1, "completions/clipped_ratio": 0.0375, "completions/max_length": 1270.1, "completions/max_terminated_length": 928.7, "completions/mean_length": 317.6875, "completions/mean_terminated_length": 270.543994140625, "completions/min_length": 26.9, "completions/min_terminated_length": 26.9, "epoch": 0.18666666666666668, "frac_reward_zero_std": 0.475, "grad_norm": 66.77887725830078, "kl": 10.910183477401734, "learning_rate": 4.011111111111111e-06, "loss": 0.0109, "num_tokens": 911419.0, "reward": -2.20625, "reward_std": 0.4734624236822128, "rewards/reward_func/mean": -2.20625, "rewards/reward_func/std": 0.7416160374879837, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 927.5, "completions/clipped_ratio": 0.01875, "completions/max_length": 927.5, "completions/max_terminated_length": 674.2, "completions/mean_length": 267.95625, "completions/mean_terminated_length": 243.36625213623046, "completions/min_length": 22.6, "completions/min_terminated_length": 22.6, "epoch": 0.2, "frac_reward_zero_std": 0.5, "grad_norm": 3.4511423110961914, "kl": 7.64983651638031, "learning_rate": 3.900000000000001e-06, "loss": 0.0076, "num_tokens": 970928.0, "reward": -2.325, "reward_std": 0.2786566108465195, "rewards/reward_func/mean": -2.325, "rewards/reward_func/std": 0.3554343104362488, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1125.2, "completions/clipped_ratio": 0.0375, "completions/max_length": 1125.2, "completions/max_terminated_length": 787.6, "completions/mean_length": 282.80625, "completions/mean_terminated_length": 234.86697082519532, "completions/min_length": 17.5, "completions/min_terminated_length": 17.5, "epoch": 0.21333333333333335, "frac_reward_zero_std": 0.475, "grad_norm": 78.64956665039062, "kl": 31.174254989624025, "learning_rate": 3.7888888888888893e-06, "loss": 0.0312, "num_tokens": 1032913.0, "reward": -2.26875, "reward_std": 0.4256745457649231, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.6542905002832413, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1234.9, "completions/clipped_ratio": 0.04375, "completions/max_length": 1234.9, "completions/max_terminated_length": 850.8, "completions/mean_length": 293.04375, "completions/mean_terminated_length": 236.12315979003907, "completions/min_length": 17.5, "completions/min_terminated_length": 17.5, "epoch": 0.22666666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 74.24949645996094, "kl": 17.39362871646881, "learning_rate": 3.6777777777777778e-06, "loss": 0.0174, "num_tokens": 1096696.0, "reward": -2.2375, "reward_std": 0.3502942889928818, "rewards/reward_func/mean": -2.2375, "rewards/reward_func/std": 0.5108247697353363, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1094.1, "completions/clipped_ratio": 0.025, "completions/max_length": 1094.1, "completions/max_terminated_length": 862.9, "completions/mean_length": 306.3875, "completions/mean_terminated_length": 275.8087554931641, "completions/min_length": 19.2, "completions/min_terminated_length": 19.2, "epoch": 0.24, "frac_reward_zero_std": 0.45, "grad_norm": 114.97906494140625, "kl": 3.446433424949646, "learning_rate": 3.566666666666667e-06, "loss": 0.0034, "num_tokens": 1163210.0, "reward": -2.26875, "reward_std": 0.36489740014076233, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.5350149899721146, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1129.5, "completions/clipped_ratio": 0.0375, "completions/max_length": 1129.5, "completions/max_terminated_length": 810.2, "completions/mean_length": 286.28125, "completions/mean_terminated_length": 238.94435272216796, "completions/min_length": 23.6, "completions/min_terminated_length": 23.6, "epoch": 0.25333333333333335, "frac_reward_zero_std": 0.55, "grad_norm": 103.09403228759766, "kl": 20.17943916320801, "learning_rate": 3.455555555555556e-06, "loss": 0.0202, "num_tokens": 1226187.0, "reward": -2.35625, "reward_std": 0.26175799071788786, "rewards/reward_func/mean": -2.35625, "rewards/reward_func/std": 0.3903583735227585, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1171.8, "completions/clipped_ratio": 0.0375, "completions/max_length": 1171.8, "completions/max_terminated_length": 780.4, "completions/mean_length": 289.95, "completions/mean_terminated_length": 242.0620880126953, "completions/min_length": 21.9, "completions/min_terminated_length": 21.9, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.5, "grad_norm": 162.74258422851562, "kl": 39.28117027282715, "learning_rate": 3.3444444444444445e-06, "loss": 0.0393, "num_tokens": 1290015.0, "reward": -2.225, "reward_std": 0.42369674146175385, "rewards/reward_func/mean": -2.225, "rewards/reward_func/std": 0.6618488401174545, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 953.0, "completions/clipped_ratio": 0.01875, "completions/max_length": 953.0, "completions/max_terminated_length": 883.8, "completions/mean_length": 306.9625, "completions/mean_terminated_length": 285.81047973632815, "completions/min_length": 20.9, "completions/min_terminated_length": 20.9, "epoch": 0.28, "frac_reward_zero_std": 0.475, "grad_norm": 56.53829574584961, "kl": 4.3735075235366825, "learning_rate": 3.2333333333333334e-06, "loss": 0.0044, "num_tokens": 1355825.0, "reward": -2.30625, "reward_std": 0.2741025388240814, "rewards/reward_func/mean": -2.30625, "rewards/reward_func/std": 0.3696858823299408, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1269.1, "completions/clipped_ratio": 0.025, "completions/max_length": 1269.1, "completions/max_terminated_length": 1032.2, "completions/mean_length": 292.15625, "completions/mean_terminated_length": 260.2483383178711, "completions/min_length": 17.8, "completions/min_terminated_length": 17.8, "epoch": 0.29333333333333333, "frac_reward_zero_std": 0.425, "grad_norm": 74.96534729003906, "kl": 16.07765164375305, "learning_rate": 3.1222222222222228e-06, "loss": 0.0161, "num_tokens": 1418942.0, "reward": -2.26875, "reward_std": 0.35119174122810365, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.48719866275787355, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1035.2, "completions/clipped_ratio": 0.01875, "completions/max_length": 1035.2, "completions/max_terminated_length": 911.0, "completions/mean_length": 256.95, "completions/mean_terminated_length": 233.14875335693358, "completions/min_length": 8.9, "completions/min_terminated_length": 8.9, "epoch": 0.30666666666666664, "frac_reward_zero_std": 0.475, "grad_norm": 36.64857482910156, "kl": 14.746024799346923, "learning_rate": 3.0111111111111113e-06, "loss": 0.0147, "num_tokens": 1478462.0, "reward": -2.225, "reward_std": 0.4513490855693817, "rewards/reward_func/mean": -2.225, "rewards/reward_func/std": 0.6648978263139724, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1116.4, "completions/clipped_ratio": 0.03125, "completions/max_length": 1116.4, "completions/max_terminated_length": 758.3, "completions/mean_length": 295.925, "completions/mean_terminated_length": 255.58333892822264, "completions/min_length": 21.1, "completions/min_terminated_length": 21.1, "epoch": 0.32, "frac_reward_zero_std": 0.475, "grad_norm": 84.84314727783203, "kl": 5.60067400932312, "learning_rate": 2.9e-06, "loss": 0.0056, "num_tokens": 1543490.0, "reward": -2.26875, "reward_std": 0.35537946224212646, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.4750110626220703, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1023.8, "completions/clipped_ratio": 0.00625, "completions/max_length": 1023.8, "completions/max_terminated_length": 945.7, "completions/mean_length": 266.0375, "completions/mean_terminated_length": 257.5279174804688, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.575, "grad_norm": 81.47051239013672, "kl": 18.14775743484497, "learning_rate": 2.788888888888889e-06, "loss": 0.0181, "num_tokens": 1603676.0, "reward": -2.3375, "reward_std": 0.2645611822605133, "rewards/reward_func/mean": -2.3375, "rewards/reward_func/std": 0.4000595390796661, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1079.9, "completions/clipped_ratio": 0.01875, "completions/max_length": 1079.9, "completions/max_terminated_length": 947.9, "completions/mean_length": 340.54375, "completions/mean_terminated_length": 318.7557769775391, "completions/min_length": 26.2, "completions/min_terminated_length": 26.2, "epoch": 0.3466666666666667, "frac_reward_zero_std": 0.55, "grad_norm": 54.74143600463867, "kl": 20.487924671173097, "learning_rate": 2.677777777777778e-06, "loss": 0.0205, "num_tokens": 1675795.0, "reward": -2.3125, "reward_std": 0.2760362923145294, "rewards/reward_func/mean": -2.3125, "rewards/reward_func/std": 0.3895877480506897, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1192.1, "completions/clipped_ratio": 0.04375, "completions/max_length": 1192.1, "completions/max_terminated_length": 858.8, "completions/mean_length": 302.7625, "completions/mean_terminated_length": 246.45244750976562, "completions/min_length": 12.2, "completions/min_terminated_length": 12.2, "epoch": 0.36, "frac_reward_zero_std": 0.65, "grad_norm": 0.1731022149324417, "kl": 7.915937983989716, "learning_rate": 2.566666666666667e-06, "loss": 0.0079, "num_tokens": 1740341.0, "reward": -2.25, "reward_std": 0.4122216373682022, "rewards/reward_func/mean": -2.25, "rewards/reward_func/std": 0.6856489062309266, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1110.5, "completions/clipped_ratio": 0.01875, "completions/max_length": 1110.5, "completions/max_terminated_length": 891.8, "completions/mean_length": 309.71875, "completions/mean_terminated_length": 286.5383361816406, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.37333333333333335, "frac_reward_zero_std": 0.5, "grad_norm": 122.04984283447266, "kl": 3.3861916780471804, "learning_rate": 2.455555555555556e-06, "loss": 0.0034, "num_tokens": 1806904.0, "reward": -2.2375, "reward_std": 0.4326493889093399, "rewards/reward_func/mean": -2.2375, "rewards/reward_func/std": 0.6847750276327134, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1069.2, "completions/clipped_ratio": 0.025, "completions/max_length": 1069.2, "completions/max_terminated_length": 853.0, "completions/mean_length": 301.1875, "completions/mean_terminated_length": 269.63209075927733, "completions/min_length": 18.7, "completions/min_terminated_length": 18.7, "epoch": 0.38666666666666666, "frac_reward_zero_std": 0.575, "grad_norm": 25.8874454498291, "kl": 14.604678845405578, "learning_rate": 2.3444444444444448e-06, "loss": 0.0146, "num_tokens": 1872494.0, "reward": -2.36875, "reward_std": 0.22780319154262543, "rewards/reward_func/mean": -2.36875, "rewards/reward_func/std": 0.3246472626924515, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1228.1, "completions/clipped_ratio": 0.0375, "completions/max_length": 1228.1, "completions/max_terminated_length": 817.4, "completions/mean_length": 287.74375, "completions/mean_terminated_length": 238.6010757446289, "completions/min_length": 20.4, "completions/min_terminated_length": 20.4, "epoch": 0.4, "frac_reward_zero_std": 0.45, "grad_norm": 50.89199447631836, "kl": 12.896457767486572, "learning_rate": 2.2333333333333333e-06, "loss": 0.0129, "num_tokens": 1936649.0, "reward": -2.26875, "reward_std": 0.3961044609546661, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.5396905303001404, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1038.1, "completions/clipped_ratio": 0.0125, "completions/max_length": 1038.1, "completions/max_terminated_length": 889.8, "completions/mean_length": 258.28125, "completions/mean_terminated_length": 241.79458618164062, "completions/min_length": 17.1, "completions/min_terminated_length": 17.1, "epoch": 0.41333333333333333, "frac_reward_zero_std": 0.65, "grad_norm": 48.88945770263672, "kl": 4.541426551342011, "learning_rate": 2.1222222222222226e-06, "loss": 0.0045, "num_tokens": 1994798.0, "reward": -2.4, "reward_std": 0.17886751294136047, "rewards/reward_func/mean": -2.4, "rewards/reward_func/std": 0.2989355862140656, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1238.9, "completions/clipped_ratio": 0.05625, "completions/max_length": 1238.9, "completions/max_terminated_length": 795.1, "completions/mean_length": 315.44375, "completions/mean_terminated_length": 243.35834197998048, "completions/min_length": 15.4, "completions/min_terminated_length": 15.4, "epoch": 0.4266666666666667, "frac_reward_zero_std": 0.525, "grad_norm": 54.9058837890625, "kl": 4.1825002312660216, "learning_rate": 2.011111111111111e-06, "loss": 0.0042, "num_tokens": 2062905.0, "reward": -2.29375, "reward_std": 0.34910253882408143, "rewards/reward_func/mean": -2.29375, "rewards/reward_func/std": 0.538089805841446, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1048.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 1048.0, "completions/max_terminated_length": 917.3, "completions/mean_length": 277.575, "completions/mean_terminated_length": 261.64583587646484, "completions/min_length": 19.7, "completions/min_terminated_length": 19.7, "epoch": 0.44, "frac_reward_zero_std": 0.425, "grad_norm": 127.28370666503906, "kl": 4.280037522315979, "learning_rate": 1.9000000000000002e-06, "loss": 0.0043, "num_tokens": 2124265.0, "reward": -2.2375, "reward_std": 0.44838539361953733, "rewards/reward_func/mean": -2.2375, "rewards/reward_func/std": 0.6401138246059418, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 927.5, "completions/clipped_ratio": 0.0125, "completions/max_length": 927.5, "completions/max_terminated_length": 798.2, "completions/mean_length": 283.7125, "completions/mean_terminated_length": 267.95792083740236, "completions/min_length": 30.1, "completions/min_terminated_length": 30.1, "epoch": 0.4533333333333333, "frac_reward_zero_std": 0.425, "grad_norm": 146.22303771972656, "kl": 30.70813355445862, "learning_rate": 1.788888888888889e-06, "loss": 0.0307, "num_tokens": 2187843.0, "reward": -2.25625, "reward_std": 0.39784331917762755, "rewards/reward_func/mean": -2.25625, "rewards/reward_func/std": 0.5563936829566956, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1043.1, "completions/clipped_ratio": 0.01875, "completions/max_length": 1043.1, "completions/max_terminated_length": 899.7, "completions/mean_length": 307.55625, "completions/mean_terminated_length": 284.3370895385742, "completions/min_length": 24.6, "completions/min_terminated_length": 24.6, "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.55, "grad_norm": 32.36188888549805, "kl": 64.3428849220276, "learning_rate": 1.6777777777777779e-06, "loss": 0.0643, "num_tokens": 2254616.0, "reward": -2.26875, "reward_std": 0.3944930166006088, "rewards/reward_func/mean": -2.26875, "rewards/reward_func/std": 0.6516756772994995, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1245.1, "completions/clipped_ratio": 0.0375, "completions/max_length": 1245.1, "completions/max_terminated_length": 964.9, "completions/mean_length": 309.43125, "completions/mean_terminated_length": 261.41810150146483, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "epoch": 0.48, "frac_reward_zero_std": 0.5, "grad_norm": 38.63202667236328, "kl": 21.073752880096436, "learning_rate": 1.566666666666667e-06, "loss": 0.0211, "num_tokens": 2320821.0, "reward": -2.275, "reward_std": 0.39782515466213225, "rewards/reward_func/mean": -2.275, "rewards/reward_func/std": 0.5283936500549317, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1342.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1342.0, "completions/max_terminated_length": 1053.9, "completions/mean_length": 318.3625, "completions/mean_terminated_length": 278.52917327880857, "completions/min_length": 21.3, "completions/min_terminated_length": 21.3, "epoch": 0.49333333333333335, "frac_reward_zero_std": 0.4, "grad_norm": 45.37665557861328, "kl": 4.191655397415161, "learning_rate": 1.4555555555555557e-06, "loss": 0.0042, "num_tokens": 2388327.0, "reward": -2.24375, "reward_std": 0.40966878533363343, "rewards/reward_func/mean": -2.24375, "rewards/reward_func/std": 0.5739072471857071, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1146.1, "completions/clipped_ratio": 0.05, "completions/max_length": 1146.1, "completions/max_terminated_length": 806.6, "completions/mean_length": 309.30625, "completions/mean_terminated_length": 246.01378784179687, "completions/min_length": 14.3, "completions/min_terminated_length": 14.3, "epoch": 0.5066666666666667, "frac_reward_zero_std": 0.425, "grad_norm": 51.08633804321289, "kl": 10.670231366157532, "learning_rate": 1.3444444444444446e-06, "loss": 0.0107, "num_tokens": 2454412.0, "reward": -2.28125, "reward_std": 0.39523502588272097, "rewards/reward_func/mean": -2.28125, "rewards/reward_func/std": 0.5278610855340957, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1179.6, "completions/clipped_ratio": 0.03125, "completions/max_length": 1179.6, "completions/max_terminated_length": 819.6, "completions/mean_length": 301.68125, "completions/mean_terminated_length": 261.32500610351565, "completions/min_length": 18.2, "completions/min_terminated_length": 18.2, "epoch": 0.52, "frac_reward_zero_std": 0.525, "grad_norm": 106.32160949707031, "kl": 21.585847473144533, "learning_rate": 1.2333333333333335e-06, "loss": 0.0216, "num_tokens": 2519509.0, "reward": -2.29375, "reward_std": 0.3345831483602524, "rewards/reward_func/mean": -2.29375, "rewards/reward_func/std": 0.5108768463134765, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 995.2, "completions/clipped_ratio": 0.03125, "completions/max_length": 995.2, "completions/max_terminated_length": 679.8, "completions/mean_length": 274.5875, "completions/mean_terminated_length": 232.36595611572267, "completions/min_length": 17.9, "completions/min_terminated_length": 17.9, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.45, "grad_norm": 75.17499542236328, "kl": 9.158978390693665, "learning_rate": 1.1222222222222222e-06, "loss": 0.0092, "num_tokens": 2579431.0, "reward": -2.1875, "reward_std": 0.4814758121967316, "rewards/reward_func/mean": -2.1875, "rewards/reward_func/std": 0.6688827067613602, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1195.5, "completions/clipped_ratio": 0.025, "completions/max_length": 1195.5, "completions/max_terminated_length": 997.3, "completions/mean_length": 286.84375, "completions/mean_terminated_length": 254.71875457763673, "completions/min_length": 15.5, "completions/min_terminated_length": 15.5, "epoch": 0.5466666666666666, "frac_reward_zero_std": 0.475, "grad_norm": 95.73103332519531, "kl": 8.273300886154175, "learning_rate": 1.0111111111111111e-06, "loss": 0.0083, "num_tokens": 2641798.0, "reward": -2.25, "reward_std": 0.39309037029743193, "rewards/reward_func/mean": -2.25, "rewards/reward_func/std": 0.5219055712223053, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 974.9, "completions/clipped_ratio": 0.0125, "completions/max_length": 974.9, "completions/max_terminated_length": 869.7, "completions/mean_length": 261.23125, "completions/mean_terminated_length": 244.99042053222655, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "epoch": 0.56, "frac_reward_zero_std": 0.375, "grad_norm": 99.72528839111328, "kl": 6.985241413116455, "learning_rate": 9.000000000000001e-07, "loss": 0.007, "num_tokens": 2700271.0, "reward": -2.23125, "reward_std": 0.4483605235815048, "rewards/reward_func/mean": -2.23125, "rewards/reward_func/std": 0.5773171722888947, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 964.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 964.0, "completions/max_terminated_length": 776.9, "completions/mean_length": 280.24375, "completions/mean_terminated_length": 263.74458465576174, "completions/min_length": 14.2, "completions/min_terminated_length": 14.2, "epoch": 0.5733333333333334, "frac_reward_zero_std": 0.45, "grad_norm": 124.6417007446289, "kl": 11.635133910179139, "learning_rate": 7.888888888888889e-07, "loss": 0.0116, "num_tokens": 2762894.0, "reward": -2.1875, "reward_std": 0.5182177871465683, "rewards/reward_func/mean": -2.1875, "rewards/reward_func/std": 0.7841762661933899, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1059.5, "completions/clipped_ratio": 0.01875, "completions/max_length": 1059.5, "completions/max_terminated_length": 908.1, "completions/mean_length": 311.675, "completions/mean_terminated_length": 289.6191696166992, "completions/min_length": 21.8, "completions/min_terminated_length": 21.8, "epoch": 0.5866666666666667, "frac_reward_zero_std": 0.475, "grad_norm": 132.1418914794922, "kl": 115.09008512496948, "learning_rate": 6.777777777777779e-07, "loss": 0.1151, "num_tokens": 2830866.0, "reward": -2.35, "reward_std": 0.26443375647068024, "rewards/reward_func/mean": -2.35, "rewards/reward_func/std": 0.35543124079704286, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1278.8, "completions/clipped_ratio": 0.0625, "completions/max_length": 1278.8, "completions/max_terminated_length": 777.6, "completions/mean_length": 327.28125, "completions/mean_terminated_length": 247.8854248046875, "completions/min_length": 25.5, "completions/min_terminated_length": 25.5, "epoch": 0.6, "frac_reward_zero_std": 0.45, "grad_norm": 121.0811538696289, "kl": 21.60447826385498, "learning_rate": 5.666666666666667e-07, "loss": 0.0216, "num_tokens": 2900423.0, "reward": -2.24375, "reward_std": 0.4444930166006088, "rewards/reward_func/mean": -2.24375, "rewards/reward_func/std": 0.6568617165088654, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1040.1, "completions/clipped_ratio": 0.01875, "completions/max_length": 1040.1, "completions/max_terminated_length": 760.2, "completions/mean_length": 274.375, "completions/mean_terminated_length": 249.7200035095215, "completions/min_length": 19.1, "completions/min_terminated_length": 19.1, "epoch": 0.6133333333333333, "frac_reward_zero_std": 0.425, "grad_norm": 38.948699951171875, "kl": 14.533638858795166, "learning_rate": 4.5555555555555563e-07, "loss": 0.0145, "num_tokens": 2961131.0, "reward": -2.3125, "reward_std": 0.31917070150375365, "rewards/reward_func/mean": -2.3125, "rewards/reward_func/std": 0.4087942600250244, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1305.9, "completions/clipped_ratio": 0.04375, "completions/max_length": 1305.9, "completions/max_terminated_length": 1012.3, "completions/mean_length": 315.675, "completions/mean_terminated_length": 259.3705429077148, "completions/min_length": 18.0, "completions/min_terminated_length": 18.0, "epoch": 0.6266666666666667, "frac_reward_zero_std": 0.35, "grad_norm": 60.82999038696289, "kl": 27.974109888076782, "learning_rate": 3.444444444444445e-07, "loss": 0.028, "num_tokens": 3028239.0, "reward": -2.24375, "reward_std": 0.3991025358438492, "rewards/reward_func/mean": -2.24375, "rewards/reward_func/std": 0.5254836320877075, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 984.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 984.0, "completions/max_terminated_length": 822.9, "completions/mean_length": 262.79375, "completions/mean_terminated_length": 246.77666931152345, "completions/min_length": 21.2, "completions/min_terminated_length": 21.2, "epoch": 0.64, "frac_reward_zero_std": 0.425, "grad_norm": 104.03584289550781, "kl": 13.309299659729003, "learning_rate": 2.3333333333333336e-07, "loss": 0.0133, "num_tokens": 3087690.0, "reward": -2.25625, "reward_std": 0.4189758121967316, "rewards/reward_func/mean": -2.25625, "rewards/reward_func/std": 0.5928926885128021, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 999.4, "completions/clipped_ratio": 0.0125, "completions/max_length": 999.4, "completions/max_terminated_length": 889.3, "completions/mean_length": 273.45625, "completions/mean_terminated_length": 257.5420852661133, "completions/min_length": 34.7, "completions/min_terminated_length": 34.7, "epoch": 0.6533333333333333, "frac_reward_zero_std": 0.6, "grad_norm": 56.064544677734375, "kl": 25.44476251602173, "learning_rate": 1.2222222222222225e-07, "loss": 0.0254, "num_tokens": 3148447.0, "reward": -2.3625, "reward_std": 0.23339140117168428, "rewards/reward_func/mean": -2.3625, "rewards/reward_func/std": 0.37498362064361573, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1092.4, "completions/clipped_ratio": 0.0375, "completions/max_length": 1092.4, "completions/max_terminated_length": 675.7, "completions/mean_length": 294.625, "completions/mean_terminated_length": 246.04232788085938, "completions/min_length": 19.8, "completions/min_terminated_length": 19.8, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.45, "grad_norm": 122.21601104736328, "kl": 17.26050043106079, "learning_rate": 1.1111111111111112e-08, "loss": 0.0173, "num_tokens": 3212059.0, "reward": -2.2125, "reward_std": 0.4100087761878967, "rewards/reward_func/mean": -2.2125, "rewards/reward_func/std": 0.5889766067266464, "step": 500 } ], "logging_steps": 10, "max_steps": 500, "num_input_tokens_seen": 3212059, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }