| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.26666666666666666, |
| "eval_steps": 500, |
| "global_step": 200, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1369.5, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1369.5, |
| "completions/max_terminated_length": 967.4, |
| "completions/mean_length": 349.9875, |
| "completions/mean_terminated_length": 310.6795928955078, |
| "completions/min_length": 19.7, |
| "completions/min_terminated_length": 19.7, |
| "epoch": 0.013333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 175.513671875, |
| "kl": 0.406684020392967, |
| "learning_rate": 9.000000000000001e-07, |
| "loss": 0.0004, |
| "num_tokens": 73366.0, |
| "reward": -1.9828404784202576, |
| "reward_std": 0.43738200813531875, |
| "rewards/reward_func/mean": -1.9828404903411865, |
| "rewards/reward_func/std": 0.5390992075204849, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1114.4, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1114.4, |
| "completions/max_terminated_length": 878.8, |
| "completions/mean_length": 285.65, |
| "completions/mean_terminated_length": 253.98928833007812, |
| "completions/min_length": 21.2, |
| "completions/min_terminated_length": 21.2, |
| "epoch": 0.02666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 84.8028335571289, |
| "kl": 0.06926110610365868, |
| "learning_rate": 1.9000000000000002e-06, |
| "loss": 0.0001, |
| "num_tokens": 135618.0, |
| "reward": -2.0299901008605956, |
| "reward_std": 0.34289126843214035, |
| "rewards/reward_func/mean": -2.0299901247024534, |
| "rewards/reward_func/std": 0.36671974062919616, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1320.9, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1320.9, |
| "completions/max_terminated_length": 1137.8, |
| "completions/mean_length": 378.5125, |
| "completions/mean_terminated_length": 332.92625885009767, |
| "completions/min_length": 32.1, |
| "completions/min_terminated_length": 32.1, |
| "epoch": 0.04, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 171.62295532226562, |
| "kl": 0.6327452264726162, |
| "learning_rate": 2.9e-06, |
| "loss": 0.0006, |
| "num_tokens": 212112.0, |
| "reward": -1.8967698693275452, |
| "reward_std": 0.5322311833500862, |
| "rewards/reward_func/mean": -1.8967698216438293, |
| "rewards/reward_func/std": 0.6287578240036964, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1158.5, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1158.5, |
| "completions/max_terminated_length": 814.9, |
| "completions/mean_length": 302.51875, |
| "completions/mean_terminated_length": 245.66839904785155, |
| "completions/min_length": 21.4, |
| "completions/min_terminated_length": 21.4, |
| "epoch": 0.05333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 155.87843322753906, |
| "kl": 20.71284626722336, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0207, |
| "num_tokens": 278219.0, |
| "reward": -1.8949542403221131, |
| "reward_std": 0.5414254635572433, |
| "rewards/reward_func/mean": -1.8949542760848999, |
| "rewards/reward_func/std": 0.6462407022714615, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1438.0, |
| "completions/clipped_ratio": 0.05625, |
| "completions/max_length": 1438.0, |
| "completions/max_terminated_length": 957.8, |
| "completions/mean_length": 368.88125, |
| "completions/mean_terminated_length": 298.481379699707, |
| "completions/min_length": 25.8, |
| "completions/min_terminated_length": 25.8, |
| "epoch": 0.06666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 246.18365478515625, |
| "kl": 81.723886013031, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0817, |
| "num_tokens": 354520.0, |
| "reward": -1.9963937759399415, |
| "reward_std": 0.4492200046777725, |
| "rewards/reward_func/mean": -1.9963937878608704, |
| "rewards/reward_func/std": 0.5674424737691879, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1460.3, |
| "completions/clipped_ratio": 0.0625, |
| "completions/max_length": 1460.3, |
| "completions/max_terminated_length": 968.9, |
| "completions/mean_length": 354.225, |
| "completions/mean_terminated_length": 275.43447265625, |
| "completions/min_length": 24.4, |
| "completions/min_terminated_length": 24.4, |
| "epoch": 0.08, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 101.76862335205078, |
| "kl": 7.8479931354522705, |
| "learning_rate": 4.9000000000000005e-06, |
| "loss": 0.0078, |
| "num_tokens": 428700.0, |
| "reward": -2.0827497005462647, |
| "reward_std": 0.33130341917276385, |
| "rewards/reward_func/mean": -2.0827497005462647, |
| "rewards/reward_func/std": 0.4056150123476982, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1161.2, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1161.2, |
| "completions/max_terminated_length": 900.7, |
| "completions/mean_length": 321.36875, |
| "completions/mean_terminated_length": 282.4824447631836, |
| "completions/min_length": 11.5, |
| "completions/min_terminated_length": 11.5, |
| "epoch": 0.09333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 140.64456176757812, |
| "kl": 15.888973093032837, |
| "learning_rate": 4.7888888888888894e-06, |
| "loss": 0.0159, |
| "num_tokens": 497455.0, |
| "reward": -1.948601198196411, |
| "reward_std": 0.4984253913164139, |
| "rewards/reward_func/mean": -1.9486011862754822, |
| "rewards/reward_func/std": 0.587457275390625, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1319.1, |
| "completions/clipped_ratio": 0.075, |
| "completions/max_length": 1319.1, |
| "completions/max_terminated_length": 917.3, |
| "completions/mean_length": 363.85, |
| "completions/mean_terminated_length": 271.0082809448242, |
| "completions/min_length": 31.9, |
| "completions/min_terminated_length": 31.9, |
| "epoch": 0.10666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 151.1342010498047, |
| "kl": 612.0117300510407, |
| "learning_rate": 4.677777777777778e-06, |
| "loss": 0.612, |
| "num_tokens": 572911.0, |
| "reward": -2.036101925373077, |
| "reward_std": 0.3775564029812813, |
| "rewards/reward_func/mean": -2.0361019134521485, |
| "rewards/reward_func/std": 0.44103380143642423, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1144.8, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1144.8, |
| "completions/max_terminated_length": 842.0, |
| "completions/mean_length": 297.375, |
| "completions/mean_terminated_length": 264.98167114257814, |
| "completions/min_length": 13.7, |
| "completions/min_terminated_length": 13.7, |
| "epoch": 0.12, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 76.42510223388672, |
| "kl": 29.60899884700775, |
| "learning_rate": 4.566666666666667e-06, |
| "loss": 0.0296, |
| "num_tokens": 637651.0, |
| "reward": -2.0117875218391417, |
| "reward_std": 0.48156112879514695, |
| "rewards/reward_func/mean": -2.011787533760071, |
| "rewards/reward_func/std": 0.6340822532773018, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1019.2, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1019.2, |
| "completions/max_terminated_length": 805.0, |
| "completions/mean_length": 292.1625, |
| "completions/mean_terminated_length": 243.4895050048828, |
| "completions/min_length": 21.2, |
| "completions/min_terminated_length": 21.2, |
| "epoch": 0.13333333333333333, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 52.52836990356445, |
| "kl": 5.190238785743714, |
| "learning_rate": 4.455555555555555e-06, |
| "loss": 0.0052, |
| "num_tokens": 701457.0, |
| "reward": -2.0103227376937864, |
| "reward_std": 0.3981830716133118, |
| "rewards/reward_func/mean": -2.010322690010071, |
| "rewards/reward_func/std": 0.46229155659675597, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1195.6, |
| "completions/clipped_ratio": 0.04375, |
| "completions/max_length": 1195.6, |
| "completions/max_terminated_length": 707.5, |
| "completions/mean_length": 273.6875, |
| "completions/mean_terminated_length": 217.00977020263673, |
| "completions/min_length": 18.2, |
| "completions/min_terminated_length": 18.2, |
| "epoch": 0.14666666666666667, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 111.76286315917969, |
| "kl": 11.202206754684449, |
| "learning_rate": 4.344444444444445e-06, |
| "loss": 0.0112, |
| "num_tokens": 762523.0, |
| "reward": -1.9468143582344055, |
| "reward_std": 0.541467210650444, |
| "rewards/reward_func/mean": -1.9468143582344055, |
| "rewards/reward_func/std": 0.7460459738969802, |
| "step": 110 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 964.2, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 964.2, |
| "completions/max_terminated_length": 663.5, |
| "completions/mean_length": 232.1, |
| "completions/mean_terminated_length": 206.75458526611328, |
| "completions/min_length": 26.5, |
| "completions/min_terminated_length": 26.5, |
| "epoch": 0.16, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 85.74720764160156, |
| "kl": 9.81872730255127, |
| "learning_rate": 4.233333333333334e-06, |
| "loss": 0.0098, |
| "num_tokens": 816111.0, |
| "reward": -1.8584201455116272, |
| "reward_std": 0.6271616354584694, |
| "rewards/reward_func/mean": -1.8584201216697693, |
| "rewards/reward_func/std": 0.8233610987663269, |
| "step": 120 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1176.6, |
| "completions/clipped_ratio": 0.05, |
| "completions/max_length": 1176.6, |
| "completions/max_terminated_length": 865.2, |
| "completions/mean_length": 337.5375, |
| "completions/mean_terminated_length": 275.18006286621096, |
| "completions/min_length": 22.3, |
| "completions/min_terminated_length": 22.3, |
| "epoch": 0.17333333333333334, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 26.790021896362305, |
| "kl": 8.61414155960083, |
| "learning_rate": 4.122222222222222e-06, |
| "loss": 0.0086, |
| "num_tokens": 887825.0, |
| "reward": -2.0355328917503357, |
| "reward_std": 0.3970488652586937, |
| "rewards/reward_func/mean": -2.0355329513549805, |
| "rewards/reward_func/std": 0.5011579066514968, |
| "step": 130 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1129.0, |
| "completions/clipped_ratio": 0.01875, |
| "completions/max_length": 1129.0, |
| "completions/max_terminated_length": 1012.8, |
| "completions/mean_length": 338.84375, |
| "completions/mean_terminated_length": 316.7058380126953, |
| "completions/min_length": 29.9, |
| "completions/min_terminated_length": 29.9, |
| "epoch": 0.18666666666666668, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 42.169864654541016, |
| "kl": 3.7959092617034913, |
| "learning_rate": 4.011111111111111e-06, |
| "loss": 0.0038, |
| "num_tokens": 960112.0, |
| "reward": -1.9997008681297301, |
| "reward_std": 0.42410101294517516, |
| "rewards/reward_func/mean": -1.9997008919715882, |
| "rewards/reward_func/std": 0.5003743663430213, |
| "step": 140 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1149.0, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1149.0, |
| "completions/max_terminated_length": 891.2, |
| "completions/mean_length": 321.8125, |
| "completions/mean_terminated_length": 275.2519104003906, |
| "completions/min_length": 28.0, |
| "completions/min_terminated_length": 28.0, |
| "epoch": 0.2, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 68.53196716308594, |
| "kl": 5.498324632644653, |
| "learning_rate": 3.900000000000001e-06, |
| "loss": 0.0055, |
| "num_tokens": 1028238.0, |
| "reward": -1.9284112930297852, |
| "reward_std": 0.5184856146574021, |
| "rewards/reward_func/mean": -1.928411316871643, |
| "rewards/reward_func/std": 0.6621677219867707, |
| "step": 150 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1062.3, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1062.3, |
| "completions/max_terminated_length": 784.8, |
| "completions/mean_length": 272.725, |
| "completions/mean_terminated_length": 233.19851989746093, |
| "completions/min_length": 23.0, |
| "completions/min_terminated_length": 23.0, |
| "epoch": 0.21333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 60.397335052490234, |
| "kl": 6.759932923316955, |
| "learning_rate": 3.7888888888888893e-06, |
| "loss": 0.0068, |
| "num_tokens": 1088610.0, |
| "reward": -1.950536346435547, |
| "reward_std": 0.536798107624054, |
| "rewards/reward_func/mean": -1.9505364656448365, |
| "rewards/reward_func/std": 0.6081681102514267, |
| "step": 160 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1134.6, |
| "completions/clipped_ratio": 0.025, |
| "completions/max_length": 1134.6, |
| "completions/max_terminated_length": 823.1, |
| "completions/mean_length": 282.64375, |
| "completions/mean_terminated_length": 250.63875427246094, |
| "completions/min_length": 20.3, |
| "completions/min_terminated_length": 20.3, |
| "epoch": 0.22666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 123.81739044189453, |
| "kl": 27.248933839797974, |
| "learning_rate": 3.6777777777777778e-06, |
| "loss": 0.0272, |
| "num_tokens": 1150729.0, |
| "reward": -1.9780633449554443, |
| "reward_std": 0.4464616924524307, |
| "rewards/reward_func/mean": -1.9780633687973022, |
| "rewards/reward_func/std": 0.5329610571265221, |
| "step": 170 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1139.0, |
| "completions/clipped_ratio": 0.03125, |
| "completions/max_length": 1139.0, |
| "completions/max_terminated_length": 907.5, |
| "completions/mean_length": 327.10625, |
| "completions/mean_terminated_length": 287.2654800415039, |
| "completions/min_length": 21.8, |
| "completions/min_terminated_length": 21.8, |
| "epoch": 0.24, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 93.7626953125, |
| "kl": 31.076939582824707, |
| "learning_rate": 3.566666666666667e-06, |
| "loss": 0.0311, |
| "num_tokens": 1220558.0, |
| "reward": -1.978137457370758, |
| "reward_std": 0.43074723780155183, |
| "rewards/reward_func/mean": -1.978137445449829, |
| "rewards/reward_func/std": 0.5373590663075447, |
| "step": 180 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1082.6, |
| "completions/clipped_ratio": 0.0375, |
| "completions/max_length": 1082.6, |
| "completions/max_terminated_length": 809.4, |
| "completions/mean_length": 307.125, |
| "completions/mean_terminated_length": 260.3149475097656, |
| "completions/min_length": 23.9, |
| "completions/min_terminated_length": 23.9, |
| "epoch": 0.25333333333333335, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 121.53104400634766, |
| "kl": 30.77431230545044, |
| "learning_rate": 3.455555555555556e-06, |
| "loss": 0.0308, |
| "num_tokens": 1286870.0, |
| "reward": -1.9268421530723572, |
| "reward_std": 0.4559009030461311, |
| "rewards/reward_func/mean": -1.926842176914215, |
| "rewards/reward_func/std": 0.5596317678689957, |
| "step": 190 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completion_length": 1152.0, |
| "completions/clipped_ratio": 0.0125, |
| "completions/max_length": 1152.0, |
| "completions/max_terminated_length": 1065.2, |
| "completions/mean_length": 340.56875, |
| "completions/mean_terminated_length": 326.0941680908203, |
| "completions/min_length": 21.5, |
| "completions/min_terminated_length": 21.5, |
| "epoch": 0.26666666666666666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 68.84746551513672, |
| "kl": 13.801477253437042, |
| "learning_rate": 3.3444444444444445e-06, |
| "loss": 0.0138, |
| "num_tokens": 1358797.0, |
| "reward": -1.9825191140174865, |
| "reward_std": 0.45144801288843156, |
| "rewards/reward_func/mean": -1.9825191378593445, |
| "rewards/reward_func/std": 0.5391437739133835, |
| "step": 200 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 500, |
| "num_input_tokens_seen": 1358797, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
