{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5333333333333333, "eval_steps": 500, "global_step": 400, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1369.5, "completions/clipped_ratio": 0.03125, "completions/max_length": 1369.5, "completions/max_terminated_length": 967.4, "completions/mean_length": 349.9875, "completions/mean_terminated_length": 310.6795928955078, "completions/min_length": 19.7, "completions/min_terminated_length": 19.7, "epoch": 0.013333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 175.513671875, "kl": 0.406684020392967, "learning_rate": 9.000000000000001e-07, "loss": 0.0004, "num_tokens": 73366.0, "reward": -1.9828404784202576, "reward_std": 0.43738200813531875, "rewards/reward_func/mean": -1.9828404903411865, "rewards/reward_func/std": 0.5390992075204849, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1114.4, "completions/clipped_ratio": 0.025, "completions/max_length": 1114.4, "completions/max_terminated_length": 878.8, "completions/mean_length": 285.65, "completions/mean_terminated_length": 253.98928833007812, "completions/min_length": 21.2, "completions/min_terminated_length": 21.2, "epoch": 0.02666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 84.8028335571289, "kl": 0.06926110610365868, "learning_rate": 1.9000000000000002e-06, "loss": 0.0001, "num_tokens": 135618.0, "reward": -2.0299901008605956, "reward_std": 0.34289126843214035, "rewards/reward_func/mean": -2.0299901247024534, "rewards/reward_func/std": 0.36671974062919616, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1320.9, "completions/clipped_ratio": 0.0375, "completions/max_length": 1320.9, "completions/max_terminated_length": 1137.8, "completions/mean_length": 378.5125, "completions/mean_terminated_length": 332.92625885009767, "completions/min_length": 32.1, "completions/min_terminated_length": 32.1, "epoch": 0.04, "frac_reward_zero_std": 0.0, "grad_norm": 171.62295532226562, "kl": 0.6327452264726162, "learning_rate": 2.9e-06, "loss": 0.0006, "num_tokens": 212112.0, "reward": -1.8967698693275452, "reward_std": 0.5322311833500862, "rewards/reward_func/mean": -1.8967698216438293, "rewards/reward_func/std": 0.6287578240036964, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1158.5, "completions/clipped_ratio": 0.04375, "completions/max_length": 1158.5, "completions/max_terminated_length": 814.9, "completions/mean_length": 302.51875, "completions/mean_terminated_length": 245.66839904785155, "completions/min_length": 21.4, "completions/min_terminated_length": 21.4, "epoch": 0.05333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 155.87843322753906, "kl": 20.71284626722336, "learning_rate": 3.900000000000001e-06, "loss": 0.0207, "num_tokens": 278219.0, "reward": -1.8949542403221131, "reward_std": 0.5414254635572433, "rewards/reward_func/mean": -1.8949542760848999, "rewards/reward_func/std": 0.6462407022714615, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1438.0, "completions/clipped_ratio": 0.05625, "completions/max_length": 1438.0, "completions/max_terminated_length": 957.8, "completions/mean_length": 368.88125, "completions/mean_terminated_length": 298.481379699707, "completions/min_length": 25.8, "completions/min_terminated_length": 25.8, "epoch": 0.06666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 246.18365478515625, "kl": 81.723886013031, "learning_rate": 4.9000000000000005e-06, "loss": 0.0817, "num_tokens": 354520.0, "reward": -1.9963937759399415, "reward_std": 0.4492200046777725, "rewards/reward_func/mean": -1.9963937878608704, "rewards/reward_func/std": 0.5674424737691879, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1460.3, "completions/clipped_ratio": 0.0625, "completions/max_length": 1460.3, "completions/max_terminated_length": 968.9, "completions/mean_length": 354.225, "completions/mean_terminated_length": 275.43447265625, "completions/min_length": 24.4, "completions/min_terminated_length": 24.4, "epoch": 0.08, "frac_reward_zero_std": 0.0, "grad_norm": 101.76862335205078, "kl": 7.8479931354522705, "learning_rate": 4.9000000000000005e-06, "loss": 0.0078, "num_tokens": 428700.0, "reward": -2.0827497005462647, "reward_std": 0.33130341917276385, "rewards/reward_func/mean": -2.0827497005462647, "rewards/reward_func/std": 0.4056150123476982, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1161.2, "completions/clipped_ratio": 0.03125, "completions/max_length": 1161.2, "completions/max_terminated_length": 900.7, "completions/mean_length": 321.36875, "completions/mean_terminated_length": 282.4824447631836, "completions/min_length": 11.5, "completions/min_terminated_length": 11.5, "epoch": 0.09333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 140.64456176757812, "kl": 15.888973093032837, "learning_rate": 4.7888888888888894e-06, "loss": 0.0159, "num_tokens": 497455.0, "reward": -1.948601198196411, "reward_std": 0.4984253913164139, "rewards/reward_func/mean": -1.9486011862754822, "rewards/reward_func/std": 0.587457275390625, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1319.1, "completions/clipped_ratio": 0.075, "completions/max_length": 1319.1, "completions/max_terminated_length": 917.3, "completions/mean_length": 363.85, "completions/mean_terminated_length": 271.0082809448242, "completions/min_length": 31.9, "completions/min_terminated_length": 31.9, "epoch": 0.10666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 151.1342010498047, "kl": 612.0117300510407, "learning_rate": 4.677777777777778e-06, "loss": 0.612, "num_tokens": 572911.0, "reward": -2.036101925373077, "reward_std": 0.3775564029812813, "rewards/reward_func/mean": -2.0361019134521485, "rewards/reward_func/std": 0.44103380143642423, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1144.8, "completions/clipped_ratio": 0.025, "completions/max_length": 1144.8, "completions/max_terminated_length": 842.0, "completions/mean_length": 297.375, "completions/mean_terminated_length": 264.98167114257814, "completions/min_length": 13.7, "completions/min_terminated_length": 13.7, "epoch": 0.12, "frac_reward_zero_std": 0.0, "grad_norm": 76.42510223388672, "kl": 29.60899884700775, "learning_rate": 4.566666666666667e-06, "loss": 0.0296, "num_tokens": 637651.0, "reward": -2.0117875218391417, "reward_std": 0.48156112879514695, "rewards/reward_func/mean": -2.011787533760071, "rewards/reward_func/std": 0.6340822532773018, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1019.2, "completions/clipped_ratio": 0.0375, "completions/max_length": 1019.2, "completions/max_terminated_length": 805.0, "completions/mean_length": 292.1625, "completions/mean_terminated_length": 243.4895050048828, "completions/min_length": 21.2, "completions/min_terminated_length": 21.2, "epoch": 0.13333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 52.52836990356445, "kl": 5.190238785743714, "learning_rate": 4.455555555555555e-06, "loss": 0.0052, "num_tokens": 701457.0, "reward": -2.0103227376937864, "reward_std": 0.3981830716133118, "rewards/reward_func/mean": -2.010322690010071, "rewards/reward_func/std": 0.46229155659675597, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1195.6, "completions/clipped_ratio": 0.04375, "completions/max_length": 1195.6, "completions/max_terminated_length": 707.5, "completions/mean_length": 273.6875, "completions/mean_terminated_length": 217.00977020263673, "completions/min_length": 18.2, "completions/min_terminated_length": 18.2, "epoch": 0.14666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 111.76286315917969, "kl": 11.202206754684449, "learning_rate": 4.344444444444445e-06, "loss": 0.0112, "num_tokens": 762523.0, "reward": -1.9468143582344055, "reward_std": 0.541467210650444, "rewards/reward_func/mean": -1.9468143582344055, "rewards/reward_func/std": 0.7460459738969802, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 964.2, "completions/clipped_ratio": 0.01875, "completions/max_length": 964.2, "completions/max_terminated_length": 663.5, "completions/mean_length": 232.1, "completions/mean_terminated_length": 206.75458526611328, "completions/min_length": 26.5, "completions/min_terminated_length": 26.5, "epoch": 0.16, "frac_reward_zero_std": 0.0, "grad_norm": 85.74720764160156, "kl": 9.81872730255127, "learning_rate": 4.233333333333334e-06, "loss": 0.0098, "num_tokens": 816111.0, "reward": -1.8584201455116272, "reward_std": 0.6271616354584694, "rewards/reward_func/mean": -1.8584201216697693, "rewards/reward_func/std": 0.8233610987663269, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1176.6, "completions/clipped_ratio": 0.05, "completions/max_length": 1176.6, "completions/max_terminated_length": 865.2, "completions/mean_length": 337.5375, "completions/mean_terminated_length": 275.18006286621096, "completions/min_length": 22.3, "completions/min_terminated_length": 22.3, "epoch": 0.17333333333333334, "frac_reward_zero_std": 0.0, "grad_norm": 26.790021896362305, "kl": 8.61414155960083, "learning_rate": 4.122222222222222e-06, "loss": 0.0086, "num_tokens": 887825.0, "reward": -2.0355328917503357, "reward_std": 0.3970488652586937, "rewards/reward_func/mean": -2.0355329513549805, "rewards/reward_func/std": 0.5011579066514968, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1129.0, "completions/clipped_ratio": 0.01875, "completions/max_length": 1129.0, "completions/max_terminated_length": 1012.8, "completions/mean_length": 338.84375, "completions/mean_terminated_length": 316.7058380126953, "completions/min_length": 29.9, "completions/min_terminated_length": 29.9, "epoch": 0.18666666666666668, "frac_reward_zero_std": 0.0, "grad_norm": 42.169864654541016, "kl": 3.7959092617034913, "learning_rate": 4.011111111111111e-06, "loss": 0.0038, "num_tokens": 960112.0, "reward": -1.9997008681297301, "reward_std": 0.42410101294517516, "rewards/reward_func/mean": -1.9997008919715882, "rewards/reward_func/std": 0.5003743663430213, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1149.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 1149.0, "completions/max_terminated_length": 891.2, "completions/mean_length": 321.8125, "completions/mean_terminated_length": 275.2519104003906, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "epoch": 0.2, "frac_reward_zero_std": 0.0, "grad_norm": 68.53196716308594, "kl": 5.498324632644653, "learning_rate": 3.900000000000001e-06, "loss": 0.0055, "num_tokens": 1028238.0, "reward": -1.9284112930297852, "reward_std": 0.5184856146574021, "rewards/reward_func/mean": -1.928411316871643, "rewards/reward_func/std": 0.6621677219867707, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1062.3, "completions/clipped_ratio": 0.03125, "completions/max_length": 1062.3, "completions/max_terminated_length": 784.8, "completions/mean_length": 272.725, "completions/mean_terminated_length": 233.19851989746093, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "epoch": 0.21333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 60.397335052490234, "kl": 6.759932923316955, "learning_rate": 3.7888888888888893e-06, "loss": 0.0068, "num_tokens": 1088610.0, "reward": -1.950536346435547, "reward_std": 0.536798107624054, "rewards/reward_func/mean": -1.9505364656448365, "rewards/reward_func/std": 0.6081681102514267, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1134.6, "completions/clipped_ratio": 0.025, "completions/max_length": 1134.6, "completions/max_terminated_length": 823.1, "completions/mean_length": 282.64375, "completions/mean_terminated_length": 250.63875427246094, "completions/min_length": 20.3, "completions/min_terminated_length": 20.3, "epoch": 0.22666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 123.81739044189453, "kl": 27.248933839797974, "learning_rate": 3.6777777777777778e-06, "loss": 0.0272, "num_tokens": 1150729.0, "reward": -1.9780633449554443, "reward_std": 0.4464616924524307, "rewards/reward_func/mean": -1.9780633687973022, "rewards/reward_func/std": 0.5329610571265221, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1139.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 1139.0, "completions/max_terminated_length": 907.5, "completions/mean_length": 327.10625, "completions/mean_terminated_length": 287.2654800415039, "completions/min_length": 21.8, "completions/min_terminated_length": 21.8, "epoch": 0.24, "frac_reward_zero_std": 0.0, "grad_norm": 93.7626953125, "kl": 31.076939582824707, "learning_rate": 3.566666666666667e-06, "loss": 0.0311, "num_tokens": 1220558.0, "reward": -1.978137457370758, "reward_std": 0.43074723780155183, "rewards/reward_func/mean": -1.978137445449829, "rewards/reward_func/std": 0.5373590663075447, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1082.6, "completions/clipped_ratio": 0.0375, "completions/max_length": 1082.6, "completions/max_terminated_length": 809.4, "completions/mean_length": 307.125, "completions/mean_terminated_length": 260.3149475097656, "completions/min_length": 23.9, "completions/min_terminated_length": 23.9, "epoch": 0.25333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 121.53104400634766, "kl": 30.77431230545044, "learning_rate": 3.455555555555556e-06, "loss": 0.0308, "num_tokens": 1286870.0, "reward": -1.9268421530723572, "reward_std": 0.4559009030461311, "rewards/reward_func/mean": -1.926842176914215, "rewards/reward_func/std": 0.5596317678689957, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1152.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 1152.0, "completions/max_terminated_length": 1065.2, "completions/mean_length": 340.56875, "completions/mean_terminated_length": 326.0941680908203, "completions/min_length": 21.5, "completions/min_terminated_length": 21.5, "epoch": 0.26666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 68.84746551513672, "kl": 13.801477253437042, "learning_rate": 3.3444444444444445e-06, "loss": 0.0138, "num_tokens": 1358797.0, "reward": -1.9825191140174865, "reward_std": 0.45144801288843156, "rewards/reward_func/mean": -1.9825191378593445, "rewards/reward_func/std": 0.5391437739133835, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1045.2, "completions/clipped_ratio": 0.00625, "completions/max_length": 1045.2, "completions/max_terminated_length": 940.7, "completions/mean_length": 295.15625, "completions/mean_terminated_length": 286.61666717529295, "completions/min_length": 30.2, "completions/min_terminated_length": 30.2, "epoch": 0.28, "frac_reward_zero_std": 0.0, "grad_norm": 106.53809356689453, "kl": 9.078963446617127, "learning_rate": 3.2333333333333334e-06, "loss": 0.0091, "num_tokens": 1422718.0, "reward": -2.008833181858063, "reward_std": 0.3771885186433792, "rewards/reward_func/mean": -2.008833181858063, "rewards/reward_func/std": 0.43363637626171114, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1084.6, "completions/clipped_ratio": 0.025, "completions/max_length": 1084.6, "completions/max_terminated_length": 730.9, "completions/mean_length": 274.09375, "completions/mean_terminated_length": 241.44417266845704, "completions/min_length": 22.3, "completions/min_terminated_length": 22.3, "epoch": 0.29333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 19.20754623413086, "kl": 6.902122008800506, "learning_rate": 3.1222222222222228e-06, "loss": 0.0069, "num_tokens": 1482945.0, "reward": -1.932115340232849, "reward_std": 0.5408788859844208, "rewards/reward_func/mean": -1.932115364074707, "rewards/reward_func/std": 0.7402323096990585, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1171.2, "completions/clipped_ratio": 0.03125, "completions/max_length": 1171.2, "completions/max_terminated_length": 816.4, "completions/mean_length": 299.49375, "completions/mean_terminated_length": 260.0383361816406, "completions/min_length": 27.2, "completions/min_terminated_length": 27.2, "epoch": 0.30666666666666664, "frac_reward_zero_std": 0.0, "grad_norm": 44.67654800415039, "kl": 12.831368160247802, "learning_rate": 3.0111111111111113e-06, "loss": 0.0128, "num_tokens": 1549272.0, "reward": -1.9424951314926147, "reward_std": 0.5370972290635109, "rewards/reward_func/mean": -1.942495095729828, "rewards/reward_func/std": 0.7188851982355118, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1064.1, "completions/clipped_ratio": 0.025, "completions/max_length": 1064.1, "completions/max_terminated_length": 788.5, "completions/mean_length": 281.80625, "completions/mean_terminated_length": 249.73833923339845, "completions/min_length": 29.4, "completions/min_terminated_length": 29.4, "epoch": 0.32, "frac_reward_zero_std": 0.0, "grad_norm": 100.51697540283203, "kl": 15.14843816757202, "learning_rate": 2.9e-06, "loss": 0.0151, "num_tokens": 1612041.0, "reward": -1.9903531432151795, "reward_std": 0.39030813723802565, "rewards/reward_func/mean": -1.9903532028198243, "rewards/reward_func/std": 0.4980658650398254, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1058.6, "completions/clipped_ratio": 0.025, "completions/max_length": 1058.6, "completions/max_terminated_length": 832.1, "completions/mean_length": 284.3625, "completions/mean_terminated_length": 251.54393310546874, "completions/min_length": 14.3, "completions/min_terminated_length": 14.3, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 55.237518310546875, "kl": 48.27505903244018, "learning_rate": 2.788888888888889e-06, "loss": 0.0483, "num_tokens": 1675159.0, "reward": -2.030545675754547, "reward_std": 0.3841791346669197, "rewards/reward_func/mean": -2.030545711517334, "rewards/reward_func/std": 0.4474460884928703, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1412.5, "completions/clipped_ratio": 0.04375, "completions/max_length": 1412.5, "completions/max_terminated_length": 1067.5, "completions/mean_length": 355.99375, "completions/mean_terminated_length": 301.65709533691404, "completions/min_length": 25.9, "completions/min_terminated_length": 25.9, "epoch": 0.3466666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 222.0747833251953, "kl": 44.72619161605835, "learning_rate": 2.677777777777778e-06, "loss": 0.0447, "num_tokens": 1749750.0, "reward": -1.8721598148345948, "reward_std": 0.5266131952404975, "rewards/reward_func/mean": -1.8721598148345948, "rewards/reward_func/std": 0.6318833231925964, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1247.9, "completions/clipped_ratio": 0.03125, "completions/max_length": 1247.9, "completions/max_terminated_length": 1026.8, "completions/mean_length": 331.88125, "completions/mean_terminated_length": 292.92720794677734, "completions/min_length": 22.8, "completions/min_terminated_length": 22.8, "epoch": 0.36, "frac_reward_zero_std": 0.0, "grad_norm": 95.26795196533203, "kl": 15.422927379608154, "learning_rate": 2.566666666666667e-06, "loss": 0.0154, "num_tokens": 1818955.0, "reward": -2.040046548843384, "reward_std": 0.3661203935742378, "rewards/reward_func/mean": -2.04004647731781, "rewards/reward_func/std": 0.4822279900312424, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1170.3, "completions/clipped_ratio": 0.03125, "completions/max_length": 1170.3, "completions/max_terminated_length": 935.0, "completions/mean_length": 272.13125, "completions/mean_terminated_length": 231.76422882080078, "completions/min_length": 18.8, "completions/min_terminated_length": 18.8, "epoch": 0.37333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 86.07445526123047, "kl": 22.66034655570984, "learning_rate": 2.455555555555556e-06, "loss": 0.0227, "num_tokens": 1879504.0, "reward": -1.9990869402885436, "reward_std": 0.4163683235645294, "rewards/reward_func/mean": -1.9990868806838988, "rewards/reward_func/std": 0.5129999920725823, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1052.1, "completions/clipped_ratio": 0.0375, "completions/max_length": 1052.1, "completions/max_terminated_length": 759.0, "completions/mean_length": 286.4375, "completions/mean_terminated_length": 239.75702819824218, "completions/min_length": 16.8, "completions/min_terminated_length": 16.8, "epoch": 0.38666666666666666, "frac_reward_zero_std": 0.0, "grad_norm": 60.21363067626953, "kl": 5.911079692840576, "learning_rate": 2.3444444444444448e-06, "loss": 0.0059, "num_tokens": 1942734.0, "reward": -2.023274374008179, "reward_std": 0.38210188746452334, "rewards/reward_func/mean": -2.0232744336128237, "rewards/reward_func/std": 0.42558410167694094, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1312.5, "completions/clipped_ratio": 0.04375, "completions/max_length": 1312.5, "completions/max_terminated_length": 902.8, "completions/mean_length": 323.3375, "completions/mean_terminated_length": 269.14376068115234, "completions/min_length": 22.4, "completions/min_terminated_length": 22.4, "epoch": 0.4, "frac_reward_zero_std": 0.0, "grad_norm": 56.95078659057617, "kl": 13.540953445434571, "learning_rate": 2.2333333333333333e-06, "loss": 0.0135, "num_tokens": 2012584.0, "reward": -2.0278689742088316, "reward_std": 0.37870625257492063, "rewards/reward_func/mean": -2.0278690218925477, "rewards/reward_func/std": 0.44234977960586547, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1077.3, "completions/clipped_ratio": 0.04375, "completions/max_length": 1077.3, "completions/max_terminated_length": 862.6, "completions/mean_length": 335.9, "completions/mean_terminated_length": 281.6435180664063, "completions/min_length": 21.6, "completions/min_terminated_length": 21.6, "epoch": 0.41333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 37.47978591918945, "kl": 23.301832151412963, "learning_rate": 2.1222222222222226e-06, "loss": 0.0233, "num_tokens": 2083152.0, "reward": -1.9517748951911926, "reward_std": 0.4163149356842041, "rewards/reward_func/mean": -1.9517749428749085, "rewards/reward_func/std": 0.480777782201767, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1048.3, "completions/clipped_ratio": 0.0125, "completions/max_length": 1048.3, "completions/max_terminated_length": 862.8, "completions/mean_length": 282.93125, "completions/mean_terminated_length": 266.6545867919922, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "epoch": 0.4266666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 83.24932861328125, "kl": 17.49512481689453, "learning_rate": 2.011111111111111e-06, "loss": 0.0175, "num_tokens": 2146057.0, "reward": -1.887987744808197, "reward_std": 0.5960402369499207, "rewards/reward_func/mean": -1.887987732887268, "rewards/reward_func/std": 0.7254746288061142, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1162.3, "completions/clipped_ratio": 0.03125, "completions/max_length": 1162.3, "completions/max_terminated_length": 897.5, "completions/mean_length": 292.51875, "completions/mean_terminated_length": 252.55708770751954, "completions/min_length": 30.4, "completions/min_terminated_length": 30.4, "epoch": 0.44, "frac_reward_zero_std": 0.0, "grad_norm": 54.589046478271484, "kl": 21.82258458137512, "learning_rate": 1.9000000000000002e-06, "loss": 0.0218, "num_tokens": 2209808.0, "reward": -1.9544290542602538, "reward_std": 0.5244767040014267, "rewards/reward_func/mean": -1.954429066181183, "rewards/reward_func/std": 0.6934701532125473, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1149.8, "completions/clipped_ratio": 0.0375, "completions/max_length": 1149.8, "completions/max_terminated_length": 781.8, "completions/mean_length": 322.54375, "completions/mean_terminated_length": 275.83464965820315, "completions/min_length": 19.6, "completions/min_terminated_length": 19.6, "epoch": 0.4533333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 79.33943939208984, "kl": 9.442901134490967, "learning_rate": 1.788888888888889e-06, "loss": 0.0094, "num_tokens": 2279599.0, "reward": -2.0039703845977783, "reward_std": 0.3999079495668411, "rewards/reward_func/mean": -2.0039703965187075, "rewards/reward_func/std": 0.4593930631875992, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1226.6, "completions/clipped_ratio": 0.03125, "completions/max_length": 1226.6, "completions/max_terminated_length": 945.3, "completions/mean_length": 315.2, "completions/mean_terminated_length": 276.0145919799805, "completions/min_length": 21.1, "completions/min_terminated_length": 21.1, "epoch": 0.4666666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 131.43572998046875, "kl": 4.923084235191345, "learning_rate": 1.6777777777777779e-06, "loss": 0.0049, "num_tokens": 2347595.0, "reward": -2.047995126247406, "reward_std": 0.3629095882177353, "rewards/reward_func/mean": -2.047995185852051, "rewards/reward_func/std": 0.3990507185459137, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1274.8, "completions/clipped_ratio": 0.04375, "completions/max_length": 1274.8, "completions/max_terminated_length": 943.8, "completions/mean_length": 321.025, "completions/mean_terminated_length": 264.1110778808594, "completions/min_length": 26.2, "completions/min_terminated_length": 26.2, "epoch": 0.48, "frac_reward_zero_std": 0.0, "grad_norm": 221.5507354736328, "kl": 6.493558549880982, "learning_rate": 1.566666666666667e-06, "loss": 0.0065, "num_tokens": 2415655.0, "reward": -1.9452160477638245, "reward_std": 0.4852162510156631, "rewards/reward_func/mean": -1.9452160835266112, "rewards/reward_func/std": 0.5771495312452316, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1008.8, "completions/clipped_ratio": 0.01875, "completions/max_length": 1008.8, "completions/max_terminated_length": 715.8, "completions/mean_length": 252.5125, "completions/mean_terminated_length": 227.52708740234374, "completions/min_length": 18.5, "completions/min_terminated_length": 18.5, "epoch": 0.49333333333333335, "frac_reward_zero_std": 0.0, "grad_norm": 29.632518768310547, "kl": 4.376008677482605, "learning_rate": 1.4555555555555557e-06, "loss": 0.0044, "num_tokens": 2472625.0, "reward": -1.9830026388168336, "reward_std": 0.4942955255508423, "rewards/reward_func/mean": -1.9830026626586914, "rewards/reward_func/std": 0.5954068779945374, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1075.1, "completions/clipped_ratio": 0.03125, "completions/max_length": 1075.1, "completions/max_terminated_length": 656.2, "completions/mean_length": 275.98125, "completions/mean_terminated_length": 235.95750732421874, "completions/min_length": 27.2, "completions/min_terminated_length": 27.2, "epoch": 0.5066666666666667, "frac_reward_zero_std": 0.0, "grad_norm": 28.514373779296875, "kl": 5.024366116523742, "learning_rate": 1.3444444444444446e-06, "loss": 0.005, "num_tokens": 2533378.0, "reward": -1.9329654932022096, "reward_std": 0.5160692006349563, "rewards/reward_func/mean": -1.9329654693603515, "rewards/reward_func/std": 0.5836088061332703, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1022.1, "completions/clipped_ratio": 0.01875, "completions/max_length": 1022.1, "completions/max_terminated_length": 855.7, "completions/mean_length": 266.0625, "completions/mean_terminated_length": 242.06958770751953, "completions/min_length": 22.1, "completions/min_terminated_length": 22.1, "epoch": 0.52, "frac_reward_zero_std": 0.0, "grad_norm": 71.61882781982422, "kl": 4.43538601398468, "learning_rate": 1.2333333333333335e-06, "loss": 0.0044, "num_tokens": 2592776.0, "reward": -1.9920026421546937, "reward_std": 0.3986089646816254, "rewards/reward_func/mean": -1.9920025587081909, "rewards/reward_func/std": 0.48581980764865873, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completion_length": 1261.4, "completions/clipped_ratio": 0.05625, "completions/max_length": 1261.4, "completions/max_terminated_length": 805.4, "completions/mean_length": 308.36875, "completions/mean_terminated_length": 234.26142807006835, "completions/min_length": 22.4, "completions/min_terminated_length": 22.4, "epoch": 0.5333333333333333, "frac_reward_zero_std": 0.0, "grad_norm": 114.13717651367188, "kl": 14.524896955490112, "learning_rate": 1.1222222222222222e-06, "loss": 0.0145, "num_tokens": 2658103.0, "reward": -1.9985215425491334, "reward_std": 0.4493471711874008, "rewards/reward_func/mean": -1.9985215425491334, "rewards/reward_func/std": 0.5260480105876922, "step": 400 } ], "logging_steps": 10, "max_steps": 500, "num_input_tokens_seen": 2658103, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }