| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.5333333333333333, |
| "eval_steps": 500, |
| "global_step": 400, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1369.5, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1369.5, |
| "completions/max_terminated_length": 967.4, |
| "completions/mean_length": 349.9875, |
| "completions/mean_terminated_length": 310.6795928955078, |
| "completions/min_length": 19.7, |
| "completions/min_terminated_length": 19.7, |
| "epoch": 0.013333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 175.513671875, |
| "kl": 0.406684020392967, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0004, |
| "num_tokens": 73366.0, |
| "reward": -1.9828404784202576, |
| "reward_std": 0.43738200813531875, |
| "rewards/reward_func/mean": -1.9828404903411865, |
| "rewards/reward_func/std": 0.5390992075204849, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1114.4, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1114.4, |
| "completions/max_terminated_length": 878.8, |
| "completions/mean_length": 285.65, |
| "completions/mean_terminated_length": 253.98928833007812, |
| "completions/min_length": 21.2, |
| "completions/min_terminated_length": 21.2, |
| "epoch": 0.02666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 84.8028335571289, |
| "kl": 0.06926110610365868, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 135618.0, |
| "reward": -2.0299901008605956, |
| "reward_std": 0.34289126843214035, |
| "rewards/reward_func/mean": -2.0299901247024534, |
| "rewards/reward_func/std": 0.36671974062919616, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1320.9, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1320.9, |
| "completions/max_terminated_length": 1137.8, |
| "completions/mean_length": 378.5125, |
| "completions/mean_terminated_length": 332.92625885009767, |
| "completions/min_length": 32.1, |
| "completions/min_terminated_length": 32.1, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 171.62295532226562, |
| "kl": 0.6327452264726162, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0006, |
| "num_tokens": 212112.0, |
| "reward": -1.8967698693275452, |
| "reward_std": 0.5322311833500862, |
| "rewards/reward_func/mean": -1.8967698216438293, |
| "rewards/reward_func/std": 0.6287578240036964, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1158.5, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1158.5, |
| "completions/max_terminated_length": 814.9, |
| "completions/mean_length": 302.51875, |
| "completions/mean_terminated_length": 245.66839904785155, |
| "completions/min_length": 21.4, |
| "completions/min_terminated_length": 21.4, |
| "epoch": 0.05333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 155.87843322753906, |
| "kl": 20.71284626722336, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0207, |
| "num_tokens": 278219.0, |
| "reward": -1.8949542403221131, |
| "reward_std": 0.5414254635572433, |
| "rewards/reward_func/mean": -1.8949542760848999, |
| "rewards/reward_func/std": 0.6462407022714615, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1438.0, |
| "completions/clipped_ratio": 0.05625, |
| "completions/max_length": 1438.0, |
| "completions/max_terminated_length": 957.8, |
| "completions/mean_length": 368.88125, |
| "completions/mean_terminated_length": 298.481379699707, |
| "completions/min_length": 25.8, |
| "completions/min_terminated_length": 25.8, |
| "epoch": 0.06666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 246.18365478515625, |
| "kl": 81.723886013031, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0817, |
| "num_tokens": 354520.0, |
| "reward": -1.9963937759399415, |
| "reward_std": 0.4492200046777725, |
| "rewards/reward_func/mean": -1.9963937878608704, |
| "rewards/reward_func/std": 0.5674424737691879, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1460.3, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 1460.3, |
| "completions/max_terminated_length": 968.9, |
| "completions/mean_length": 354.225, |
| "completions/mean_terminated_length": 275.43447265625, |
| "completions/min_length": 24.4, |
| "completions/min_terminated_length": 24.4, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 101.76862335205078, |
| "kl": 7.8479931354522705, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0078, |
| "num_tokens": 428700.0, |
| "reward": -2.0827497005462647, |
| "reward_std": 0.33130341917276385, |
| "rewards/reward_func/mean": -2.0827497005462647, |
| "rewards/reward_func/std": 0.4056150123476982, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1161.2, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1161.2, |
| "completions/max_terminated_length": 900.7, |
| "completions/mean_length": 321.36875, |
| "completions/mean_terminated_length": 282.4824447631836, |
| "completions/min_length": 11.5, |
| "completions/min_terminated_length": 11.5, |
| "epoch": 0.09333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 140.64456176757812, |
| "kl": 15.888973093032837, |
| "learning_rate": 4.7888888888888894e-06, |
| "loss": 0.0159, |
| "num_tokens": 497455.0, |
| "reward": -1.948601198196411, |
| "reward_std": 0.4984253913164139, |
| "rewards/reward_func/mean": -1.9486011862754822, |
| "rewards/reward_func/std": 0.587457275390625, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1319.1, |
| "completions/clipped_ratio": 0.075, |
| "completions/max_length": 1319.1, |
| "completions/max_terminated_length": 917.3, |
| "completions/mean_length": 363.85, |
| "completions/mean_terminated_length": 271.0082809448242, |
| "completions/min_length": 31.9, |
| "completions/min_terminated_length": 31.9, |
| "epoch": 0.10666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 151.1342010498047, |
| "kl": 612.0117300510407, |
| "learning_rate": 4.677777777777778e-06, |
| "loss": 0.612, |
| "num_tokens": 572911.0, |
| "reward": -2.036101925373077, |
| "reward_std": 0.3775564029812813, |
| "rewards/reward_func/mean": -2.0361019134521485, |
| "rewards/reward_func/std": 0.44103380143642423, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1144.8, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1144.8, |
| "completions/max_terminated_length": 842.0, |
| "completions/mean_length": 297.375, |
| "completions/mean_terminated_length": 264.98167114257814, |
| "completions/min_length": 13.7, |
| "completions/min_terminated_length": 13.7, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 76.42510223388672, |
| "kl": 29.60899884700775, |
| "learning_rate": 4.566666666666667e-06, |
| "loss": 0.0296, |
| "num_tokens": 637651.0, |
| "reward": -2.0117875218391417, |
| "reward_std": 0.48156112879514695, |
| "rewards/reward_func/mean": -2.011787533760071, |
| "rewards/reward_func/std": 0.6340822532773018, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1019.2, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1019.2, |
| "completions/max_terminated_length": 805.0, |
| "completions/mean_length": 292.1625, |
| "completions/mean_terminated_length": 243.4895050048828, |
| "completions/min_length": 21.2, |
| "completions/min_terminated_length": 21.2, |
| "epoch": 0.13333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 52.52836990356445, |
| "kl": 5.190238785743714, |
| "learning_rate": 4.455555555555555e-06, |
| "loss": 0.0052, |
| "num_tokens": 701457.0, |
| "reward": -2.0103227376937864, |
| "reward_std": 0.3981830716133118, |
| "rewards/reward_func/mean": -2.010322690010071, |
| "rewards/reward_func/std": 0.46229155659675597, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1195.6, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1195.6, |
| "completions/max_terminated_length": 707.5, |
| "completions/mean_length": 273.6875, |
| "completions/mean_terminated_length": 217.00977020263673, |
| "completions/min_length": 18.2, |
| "completions/min_terminated_length": 18.2, |
| "epoch": 0.14666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 111.76286315917969, |
| "kl": 11.202206754684449, |
| "learning_rate": 4.344444444444445e-06, |
| "loss": 0.0112, |
| "num_tokens": 762523.0, |
| "reward": -1.9468143582344055, |
| "reward_std": 0.541467210650444, |
| "rewards/reward_func/mean": -1.9468143582344055, |
| "rewards/reward_func/std": 0.7460459738969802, |
| "step": 110 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 964.2, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 964.2, |
| "completions/max_terminated_length": 663.5, |
| "completions/mean_length": 232.1, |
| "completions/mean_terminated_length": 206.75458526611328, |
| "completions/min_length": 26.5, |
| "completions/min_terminated_length": 26.5, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 85.74720764160156, |
| "kl": 9.81872730255127, |
| "learning_rate": 4.233333333333334e-06, |
| "loss": 0.0098, |
| "num_tokens": 816111.0, |
| "reward": -1.8584201455116272, |
| "reward_std": 0.6271616354584694, |
| "rewards/reward_func/mean": -1.8584201216697693, |
| "rewards/reward_func/std": 0.8233610987663269, |
| "step": 120 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1176.6, |
| "completions/clipped_ratio": 0.05, |
| "completions/max_length": 1176.6, |
| "completions/max_terminated_length": 865.2, |
| "completions/mean_length": 337.5375, |
| "completions/mean_terminated_length": 275.18006286621096, |
| "completions/min_length": 22.3, |
| "completions/min_terminated_length": 22.3, |
| "epoch": 0.17333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 26.790021896362305, |
| "kl": 8.61414155960083, |
| "learning_rate": 4.122222222222222e-06, |
| "loss": 0.0086, |
| "num_tokens": 887825.0, |
| "reward": -2.0355328917503357, |
| "reward_std": 0.3970488652586937, |
| "rewards/reward_func/mean": -2.0355329513549805, |
| "rewards/reward_func/std": 0.5011579066514968, |
| "step": 130 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1129.0, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1129.0, |
| "completions/max_terminated_length": 1012.8, |
| "completions/mean_length": 338.84375, |
| "completions/mean_terminated_length": 316.7058380126953, |
| "completions/min_length": 29.9, |
| "completions/min_terminated_length": 29.9, |
| "epoch": 0.18666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 42.169864654541016, |
| "kl": 3.7959092617034913, |
| "learning_rate": 4.011111111111111e-06, |
| "loss": 0.0038, |
| "num_tokens": 960112.0, |
| "reward": -1.9997008681297301, |
| "reward_std": 0.42410101294517516, |
| "rewards/reward_func/mean": -1.9997008919715882, |
| "rewards/reward_func/std": 0.5003743663430213, |
| "step": 140 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1149.0, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1149.0, |
| "completions/max_terminated_length": 891.2, |
| "completions/mean_length": 321.8125, |
| "completions/mean_terminated_length": 275.2519104003906, |
| "completions/min_length": 28.0, |
| "completions/min_terminated_length": 28.0, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 68.53196716308594, |
| "kl": 5.498324632644653, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0055, |
| "num_tokens": 1028238.0, |
| "reward": -1.9284112930297852, |
| "reward_std": 0.5184856146574021, |
| "rewards/reward_func/mean": -1.928411316871643, |
| "rewards/reward_func/std": 0.6621677219867707, |
| "step": 150 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1062.3, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1062.3, |
| "completions/max_terminated_length": 784.8, |
| "completions/mean_length": 272.725, |
| "completions/mean_terminated_length": 233.19851989746093, |
| "completions/min_length": 23.0, |
| "completions/min_terminated_length": 23.0, |
| "epoch": 0.21333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 60.397335052490234, |
| "kl": 6.759932923316955, |
| "learning_rate": 3.7888888888888893e-06, |
| "loss": 0.0068, |
| "num_tokens": 1088610.0, |
| "reward": -1.950536346435547, |
| "reward_std": 0.536798107624054, |
| "rewards/reward_func/mean": -1.9505364656448365, |
| "rewards/reward_func/std": 0.6081681102514267, |
| "step": 160 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1134.6, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1134.6, |
| "completions/max_terminated_length": 823.1, |
| "completions/mean_length": 282.64375, |
| "completions/mean_terminated_length": 250.63875427246094, |
| "completions/min_length": 20.3, |
| "completions/min_terminated_length": 20.3, |
| "epoch": 0.22666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 123.81739044189453, |
| "kl": 27.248933839797974, |
| "learning_rate": 3.6777777777777778e-06, |
| "loss": 0.0272, |
| "num_tokens": 1150729.0, |
| "reward": -1.9780633449554443, |
| "reward_std": 0.4464616924524307, |
| "rewards/reward_func/mean": -1.9780633687973022, |
| "rewards/reward_func/std": 0.5329610571265221, |
| "step": 170 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1139.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1139.0, |
| "completions/max_terminated_length": 907.5, |
| "completions/mean_length": 327.10625, |
| "completions/mean_terminated_length": 287.2654800415039, |
| "completions/min_length": 21.8, |
| "completions/min_terminated_length": 21.8, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 93.7626953125, |
| "kl": 31.076939582824707, |
| "learning_rate": 3.566666666666667e-06, |
| "loss": 0.0311, |
| "num_tokens": 1220558.0, |
| "reward": -1.978137457370758, |
| "reward_std": 0.43074723780155183, |
| "rewards/reward_func/mean": -1.978137445449829, |
| "rewards/reward_func/std": 0.5373590663075447, |
| "step": 180 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1082.6, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1082.6, |
| "completions/max_terminated_length": 809.4, |
| "completions/mean_length": 307.125, |
| "completions/mean_terminated_length": 260.3149475097656, |
| "completions/min_length": 23.9, |
| "completions/min_terminated_length": 23.9, |
| "epoch": 0.25333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 121.53104400634766, |
| "kl": 30.77431230545044, |
| "learning_rate": 3.455555555555556e-06, |
| "loss": 0.0308, |
| "num_tokens": 1286870.0, |
| "reward": -1.9268421530723572, |
| "reward_std": 0.4559009030461311, |
| "rewards/reward_func/mean": -1.926842176914215, |
| "rewards/reward_func/std": 0.5596317678689957, |
| "step": 190 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1152.0, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1152.0, |
| "completions/max_terminated_length": 1065.2, |
| "completions/mean_length": 340.56875, |
| "completions/mean_terminated_length": 326.0941680908203, |
| "completions/min_length": 21.5, |
| "completions/min_terminated_length": 21.5, |
| "epoch": 0.26666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 68.84746551513672, |
| "kl": 13.801477253437042, |
| "learning_rate": 3.3444444444444445e-06, |
| "loss": 0.0138, |
| "num_tokens": 1358797.0, |
| "reward": -1.9825191140174865, |
| "reward_std": 0.45144801288843156, |
| "rewards/reward_func/mean": -1.9825191378593445, |
| "rewards/reward_func/std": 0.5391437739133835, |
| "step": 200 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1045.2, |
| "completions/clipped_ratio": 0.00625, |
| "completions/max_length": 1045.2, |
| "completions/max_terminated_length": 940.7, |
| "completions/mean_length": 295.15625, |
| "completions/mean_terminated_length": 286.61666717529295, |
| "completions/min_length": 30.2, |
| "completions/min_terminated_length": 30.2, |
| "epoch": 0.28, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 106.53809356689453, |
| "kl": 9.078963446617127, |
| "learning_rate": 3.2333333333333334e-06, |
| "loss": 0.0091, |
| "num_tokens": 1422718.0, |
| "reward": -2.008833181858063, |
| "reward_std": 0.3771885186433792, |
| "rewards/reward_func/mean": -2.008833181858063, |
| "rewards/reward_func/std": 0.43363637626171114, |
| "step": 210 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1084.6, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1084.6, |
| "completions/max_terminated_length": 730.9, |
| "completions/mean_length": 274.09375, |
| "completions/mean_terminated_length": 241.44417266845704, |
| "completions/min_length": 22.3, |
| "completions/min_terminated_length": 22.3, |
| "epoch": 0.29333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 19.20754623413086, |
| "kl": 6.902122008800506, |
| "learning_rate": 3.1222222222222228e-06, |
| "loss": 0.0069, |
| "num_tokens": 1482945.0, |
| "reward": -1.932115340232849, |
| "reward_std": 0.5408788859844208, |
| "rewards/reward_func/mean": -1.932115364074707, |
| "rewards/reward_func/std": 0.7402323096990585, |
| "step": 220 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1171.2, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1171.2, |
| "completions/max_terminated_length": 816.4, |
| "completions/mean_length": 299.49375, |
| "completions/mean_terminated_length": 260.0383361816406, |
| "completions/min_length": 27.2, |
| "completions/min_terminated_length": 27.2, |
| "epoch": 0.30666666666666664, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 44.67654800415039, |
| "kl": 12.831368160247802, |
| "learning_rate": 3.0111111111111113e-06, |
| "loss": 0.0128, |
| "num_tokens": 1549272.0, |
| "reward": -1.9424951314926147, |
| "reward_std": 0.5370972290635109, |
| "rewards/reward_func/mean": -1.942495095729828, |
| "rewards/reward_func/std": 0.7188851982355118, |
| "step": 230 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1064.1, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1064.1, |
| "completions/max_terminated_length": 788.5, |
| "completions/mean_length": 281.80625, |
| "completions/mean_terminated_length": 249.73833923339845, |
| "completions/min_length": 29.4, |
| "completions/min_terminated_length": 29.4, |
| "epoch": 0.32, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 100.51697540283203, |
| "kl": 15.14843816757202, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0151, |
| "num_tokens": 1612041.0, |
| "reward": -1.9903531432151795, |
| "reward_std": 0.39030813723802565, |
| "rewards/reward_func/mean": -1.9903532028198243, |
| "rewards/reward_func/std": 0.4980658650398254, |
| "step": 240 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1058.6, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1058.6, |
| "completions/max_terminated_length": 832.1, |
| "completions/mean_length": 284.3625, |
| "completions/mean_terminated_length": 251.54393310546874, |
| "completions/min_length": 14.3, |
| "completions/min_terminated_length": 14.3, |
| "epoch": 0.3333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 55.237518310546875, |
| "kl": 48.27505903244018, |
| "learning_rate": 2.788888888888889e-06, |
| "loss": 0.0483, |
| "num_tokens": 1675159.0, |
| "reward": -2.030545675754547, |
| "reward_std": 0.3841791346669197, |
| "rewards/reward_func/mean": -2.030545711517334, |
| "rewards/reward_func/std": 0.4474460884928703, |
| "step": 250 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1412.5, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1412.5, |
| "completions/max_terminated_length": 1067.5, |
| "completions/mean_length": 355.99375, |
| "completions/mean_terminated_length": 301.65709533691404, |
| "completions/min_length": 25.9, |
| "completions/min_terminated_length": 25.9, |
| "epoch": 0.3466666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 222.0747833251953, |
| "kl": 44.72619161605835, |
| "learning_rate": 2.677777777777778e-06, |
| "loss": 0.0447, |
| "num_tokens": 1749750.0, |
| "reward": -1.8721598148345948, |
| "reward_std": 0.5266131952404975, |
| "rewards/reward_func/mean": -1.8721598148345948, |
| "rewards/reward_func/std": 0.6318833231925964, |
| "step": 260 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1247.9, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1247.9, |
| "completions/max_terminated_length": 1026.8, |
| "completions/mean_length": 331.88125, |
| "completions/mean_terminated_length": 292.92720794677734, |
| "completions/min_length": 22.8, |
| "completions/min_terminated_length": 22.8, |
| "epoch": 0.36, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 95.26795196533203, |
| "kl": 15.422927379608154, |
| "learning_rate": 2.566666666666667e-06, |
| "loss": 0.0154, |
| "num_tokens": 1818955.0, |
| "reward": -2.040046548843384, |
| "reward_std": 0.3661203935742378, |
| "rewards/reward_func/mean": -2.04004647731781, |
| "rewards/reward_func/std": 0.4822279900312424, |
| "step": 270 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1170.3, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1170.3, |
| "completions/max_terminated_length": 935.0, |
| "completions/mean_length": 272.13125, |
| "completions/mean_terminated_length": 231.76422882080078, |
| "completions/min_length": 18.8, |
| "completions/min_terminated_length": 18.8, |
| "epoch": 0.37333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 86.07445526123047, |
| "kl": 22.66034655570984, |
| "learning_rate": 2.455555555555556e-06, |
| "loss": 0.0227, |
| "num_tokens": 1879504.0, |
| "reward": -1.9990869402885436, |
| "reward_std": 0.4163683235645294, |
| "rewards/reward_func/mean": -1.9990868806838988, |
| "rewards/reward_func/std": 0.5129999920725823, |
| "step": 280 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1052.1, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1052.1, |
| "completions/max_terminated_length": 759.0, |
| "completions/mean_length": 286.4375, |
| "completions/mean_terminated_length": 239.75702819824218, |
| "completions/min_length": 16.8, |
| "completions/min_terminated_length": 16.8, |
| "epoch": 0.38666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 60.21363067626953, |
| "kl": 5.911079692840576, |
| "learning_rate": 2.3444444444444448e-06, |
| "loss": 0.0059, |
| "num_tokens": 1942734.0, |
| "reward": -2.023274374008179, |
| "reward_std": 0.38210188746452334, |
| "rewards/reward_func/mean": -2.0232744336128237, |
| "rewards/reward_func/std": 0.42558410167694094, |
| "step": 290 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1312.5, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1312.5, |
| "completions/max_terminated_length": 902.8, |
| "completions/mean_length": 323.3375, |
| "completions/mean_terminated_length": 269.14376068115234, |
| "completions/min_length": 22.4, |
| "completions/min_terminated_length": 22.4, |
| "epoch": 0.4, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 56.95078659057617, |
| "kl": 13.540953445434571, |
| "learning_rate": 2.2333333333333333e-06, |
| "loss": 0.0135, |
| "num_tokens": 2012584.0, |
| "reward": -2.0278689742088316, |
| "reward_std": 0.37870625257492063, |
| "rewards/reward_func/mean": -2.0278690218925477, |
| "rewards/reward_func/std": 0.44234977960586547, |
| "step": 300 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1077.3, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1077.3, |
| "completions/max_terminated_length": 862.6, |
| "completions/mean_length": 335.9, |
| "completions/mean_terminated_length": 281.6435180664063, |
| "completions/min_length": 21.6, |
| "completions/min_terminated_length": 21.6, |
| "epoch": 0.41333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 37.47978591918945, |
| "kl": 23.301832151412963, |
| "learning_rate": 2.1222222222222226e-06, |
| "loss": 0.0233, |
| "num_tokens": 2083152.0, |
| "reward": -1.9517748951911926, |
| "reward_std": 0.4163149356842041, |
| "rewards/reward_func/mean": -1.9517749428749085, |
| "rewards/reward_func/std": 0.480777782201767, |
| "step": 310 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1048.3, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1048.3, |
| "completions/max_terminated_length": 862.8, |
| "completions/mean_length": 282.93125, |
| "completions/mean_terminated_length": 266.6545867919922, |
| "completions/min_length": 24.0, |
| "completions/min_terminated_length": 24.0, |
| "epoch": 0.4266666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 83.24932861328125, |
| "kl": 17.49512481689453, |
| "learning_rate": 2.011111111111111e-06, |
| "loss": 0.0175, |
| "num_tokens": 2146057.0, |
| "reward": -1.887987744808197, |
| "reward_std": 0.5960402369499207, |
| "rewards/reward_func/mean": -1.887987732887268, |
| "rewards/reward_func/std": 0.7254746288061142, |
| "step": 320 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1162.3, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1162.3, |
| "completions/max_terminated_length": 897.5, |
| "completions/mean_length": 292.51875, |
| "completions/mean_terminated_length": 252.55708770751954, |
| "completions/min_length": 30.4, |
| "completions/min_terminated_length": 30.4, |
| "epoch": 0.44, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 54.589046478271484, |
| "kl": 21.82258458137512, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0218, |
| "num_tokens": 2209808.0, |
| "reward": -1.9544290542602538, |
| "reward_std": 0.5244767040014267, |
| "rewards/reward_func/mean": -1.954429066181183, |
| "rewards/reward_func/std": 0.6934701532125473, |
| "step": 330 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1149.8, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1149.8, |
| "completions/max_terminated_length": 781.8, |
| "completions/mean_length": 322.54375, |
| "completions/mean_terminated_length": 275.83464965820315, |
| "completions/min_length": 19.6, |
| "completions/min_terminated_length": 19.6, |
| "epoch": 0.4533333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 79.33943939208984, |
| "kl": 9.442901134490967, |
| "learning_rate": 1.788888888888889e-06, |
| "loss": 0.0094, |
| "num_tokens": 2279599.0, |
| "reward": -2.0039703845977783, |
| "reward_std": 0.3999079495668411, |
| "rewards/reward_func/mean": -2.0039703965187075, |
| "rewards/reward_func/std": 0.4593930631875992, |
| "step": 340 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1226.6, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1226.6, |
| "completions/max_terminated_length": 945.3, |
| "completions/mean_length": 315.2, |
| "completions/mean_terminated_length": 276.0145919799805, |
| "completions/min_length": 21.1, |
| "completions/min_terminated_length": 21.1, |
| "epoch": 0.4666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 131.43572998046875, |
| "kl": 4.923084235191345, |
| "learning_rate": 1.6777777777777779e-06, |
| "loss": 0.0049, |
| "num_tokens": 2347595.0, |
| "reward": -2.047995126247406, |
| "reward_std": 0.3629095882177353, |
| "rewards/reward_func/mean": -2.047995185852051, |
| "rewards/reward_func/std": 0.3990507185459137, |
| "step": 350 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1274.8, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1274.8, |
| "completions/max_terminated_length": 943.8, |
| "completions/mean_length": 321.025, |
| "completions/mean_terminated_length": 264.1110778808594, |
| "completions/min_length": 26.2, |
| "completions/min_terminated_length": 26.2, |
| "epoch": 0.48, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 221.5507354736328, |
| "kl": 6.493558549880982, |
| "learning_rate": 1.566666666666667e-06, |
| "loss": 0.0065, |
| "num_tokens": 2415655.0, |
| "reward": -1.9452160477638245, |
| "reward_std": 0.4852162510156631, |
| "rewards/reward_func/mean": -1.9452160835266112, |
| "rewards/reward_func/std": 0.5771495312452316, |
| "step": 360 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1008.8, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1008.8, |
| "completions/max_terminated_length": 715.8, |
| "completions/mean_length": 252.5125, |
| "completions/mean_terminated_length": 227.52708740234374, |
| "completions/min_length": 18.5, |
| "completions/min_terminated_length": 18.5, |
| "epoch": 0.49333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 29.632518768310547, |
| "kl": 4.376008677482605, |
| "learning_rate": 1.4555555555555557e-06, |
| "loss": 0.0044, |
| "num_tokens": 2472625.0, |
| "reward": -1.9830026388168336, |
| "reward_std": 0.4942955255508423, |
| "rewards/reward_func/mean": -1.9830026626586914, |
| "rewards/reward_func/std": 0.5954068779945374, |
| "step": 370 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1075.1, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1075.1, |
| "completions/max_terminated_length": 656.2, |
| "completions/mean_length": 275.98125, |
| "completions/mean_terminated_length": 235.95750732421874, |
| "completions/min_length": 27.2, |
| "completions/min_terminated_length": 27.2, |
| "epoch": 0.5066666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 28.514373779296875, |
| "kl": 5.024366116523742, |
| "learning_rate": 1.3444444444444446e-06, |
| "loss": 0.005, |
| "num_tokens": 2533378.0, |
| "reward": -1.9329654932022096, |
| "reward_std": 0.5160692006349563, |
| "rewards/reward_func/mean": -1.9329654693603515, |
| "rewards/reward_func/std": 0.5836088061332703, |
| "step": 380 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1022.1, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1022.1, |
| "completions/max_terminated_length": 855.7, |
| "completions/mean_length": 266.0625, |
| "completions/mean_terminated_length": 242.06958770751953, |
| "completions/min_length": 22.1, |
| "completions/min_terminated_length": 22.1, |
| "epoch": 0.52, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 71.61882781982422, |
| "kl": 4.43538601398468, |
| "learning_rate": 1.2333333333333335e-06, |
| "loss": 0.0044, |
| "num_tokens": 2592776.0, |
| "reward": -1.9920026421546937, |
| "reward_std": 0.3986089646816254, |
| "rewards/reward_func/mean": -1.9920025587081909, |
| "rewards/reward_func/std": 0.48581980764865873, |
| "step": 390 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1261.4, |
| "completions/clipped_ratio": 0.05625, |
| "completions/max_length": 1261.4, |
| "completions/max_terminated_length": 805.4, |
| "completions/mean_length": 308.36875, |
| "completions/mean_terminated_length": 234.26142807006835, |
| "completions/min_length": 22.4, |
| "completions/min_terminated_length": 22.4, |
| "epoch": 0.5333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 114.13717651367188, |
| "kl": 14.524896955490112, |
| "learning_rate": 1.1222222222222222e-06, |
| "loss": 0.0145, |
| "num_tokens": 2658103.0, |
| "reward": -1.9985215425491334, |
| "reward_std": 0.4493471711874008, |
| "rewards/reward_func/mean": -1.9985215425491334, |
| "rewards/reward_func/std": 0.5260480105876922, |
| "step": 400 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 500, |
| "num_input_tokens_seen": 2658103, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
