| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0005, |
| "eval_steps": 500, |
| "global_step": 50, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11287.0, |
| "completions/max_terminated_length": 11287.0, |
| "completions/mean_length": 8394.21875, |
| "completions/mean_terminated_length": 8394.21875, |
| "completions/min_length": 2887.0, |
| "completions/min_terminated_length": 2887.0, |
| "entropy": 0.11788358655758202, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6996961832046509, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": 0.1667, |
| "num_tokens": 297100.0, |
| "reward": -0.03610033914446831, |
| "reward_std": 0.7386711835861206, |
| "rewards/rollout_eval_reward_func/mean": 0.2578760087490082, |
| "rewards/rollout_eval_reward_func/std": 0.30121663212776184, |
| "rewards/rollout_reward_func/mean": -0.03610033914446831, |
| "rewards/rollout_reward_func/std": 0.728388786315918, |
| "sampling/importance_sampling_ratio/max": 1.6487237215042114, |
| "sampling/importance_sampling_ratio/mean": 0.9993462562561035, |
| "sampling/importance_sampling_ratio/min": 0.44016358256340027, |
| "sampling/sampling_logp_difference/max": 0.8206088542938232, |
| "sampling/sampling_logp_difference/mean": 0.011885393410921097, |
| "step": 1, |
| "step_time": 73.96733120499948 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.11788358655758202, |
| "epoch": 2e-05, |
| "grad_norm": 1.699277400970459, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": 0.1667, |
| "step": 2, |
| "step_time": 33.627345983999476 |
| }, |
| { |
| "clip_ratio/high_max": 0.005571161163970828, |
| "clip_ratio/high_mean": 0.0031328028126154095, |
| "clip_ratio/low_mean": 0.007165575778344646, |
| "clip_ratio/low_min": 0.0007102272938936949, |
| "clip_ratio/region_mean": 0.010298378649167717, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11163.0, |
| "completions/max_terminated_length": 11163.0, |
| "completions/mean_length": 8984.84375, |
| "completions/mean_terminated_length": 8984.84375, |
| "completions/min_length": 346.0, |
| "completions/min_terminated_length": 346.0, |
| "entropy": 0.14975268673151731, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3866604566574097, |
| "kl": 0.002339825068702339, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.2053, |
| "num_tokens": 613530.0, |
| "reward": -0.20636191964149475, |
| "reward_std": 0.5412060022354126, |
| "rewards/rollout_eval_reward_func/mean": 0.13351118564605713, |
| "rewards/rollout_eval_reward_func/std": 0.24266032874584198, |
| "rewards/rollout_reward_func/mean": -0.20636191964149475, |
| "rewards/rollout_reward_func/std": 0.542148232460022, |
| "sampling/importance_sampling_ratio/max": 1.7320148944854736, |
| "sampling/importance_sampling_ratio/mean": 1.0001366138458252, |
| "sampling/importance_sampling_ratio/min": 0.5123909115791321, |
| "sampling/sampling_logp_difference/max": 0.6686674356460571, |
| "sampling/sampling_logp_difference/mean": 0.014301672577857971, |
| "step": 3, |
| "step_time": 76.45311136299915 |
| }, |
| { |
| "clip_ratio/high_max": 0.0063213135581463575, |
| "clip_ratio/high_mean": 0.003855101269437, |
| "clip_ratio/low_mean": 0.0049675037153065205, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.00882260489743203, |
| "entropy": 0.14919065916910768, |
| "epoch": 4e-05, |
| "grad_norm": 1.35505211353302, |
| "kl": 0.0014064895312912995, |
| "learning_rate": 8.571428571428573e-06, |
| "loss": -0.2073, |
| "step": 4, |
| "step_time": 34.60768302199995 |
| }, |
| { |
| "clip_ratio/high_max": 0.007777874590829015, |
| "clip_ratio/high_mean": 0.0042565843323245645, |
| "clip_ratio/low_mean": 0.003121181536698714, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0073777658108156174, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11443.0, |
| "completions/max_terminated_length": 11443.0, |
| "completions/mean_length": 8618.625, |
| "completions/mean_terminated_length": 8618.625, |
| "completions/min_length": 336.0, |
| "completions/min_terminated_length": 336.0, |
| "entropy": 0.11723149428144097, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2567578554153442, |
| "kl": 0.0012220569333294407, |
| "learning_rate": 1.1428571428571429e-05, |
| "loss": 0.14, |
| "num_tokens": 917591.0, |
| "reward": -0.0526789128780365, |
| "reward_std": 0.7260563373565674, |
| "rewards/rollout_eval_reward_func/mean": 0.24098068475723267, |
| "rewards/rollout_eval_reward_func/std": 0.30969831347465515, |
| "rewards/rollout_reward_func/mean": -0.0526789128780365, |
| "rewards/rollout_reward_func/std": 0.7283722758293152, |
| "sampling/importance_sampling_ratio/max": 1.8090978860855103, |
| "sampling/importance_sampling_ratio/mean": 0.9998831748962402, |
| "sampling/importance_sampling_ratio/min": 0.46312808990478516, |
| "sampling/sampling_logp_difference/max": 0.7697516679763794, |
| "sampling/sampling_logp_difference/mean": 0.012965342029929161, |
| "step": 5, |
| "step_time": 76.55231227500053 |
| }, |
| { |
| "clip_ratio/high_max": 0.006731559580657631, |
| "clip_ratio/high_mean": 0.0033657797903288156, |
| "clip_ratio/low_mean": 0.0034470450482331216, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.006812824809458107, |
| "entropy": 0.1164796226657927, |
| "epoch": 6e-05, |
| "grad_norm": 1.2304322719573975, |
| "kl": 0.001537940081107081, |
| "learning_rate": 1.4285714285714285e-05, |
| "loss": 0.1411, |
| "step": 6, |
| "step_time": 34.486522669000806 |
| }, |
| { |
| "clip_ratio/high_max": 0.004329371266067028, |
| "clip_ratio/high_mean": 0.002164685633033514, |
| "clip_ratio/low_mean": 0.0018710909644141793, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.004035776626551524, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11026.0, |
| "completions/max_terminated_length": 11026.0, |
| "completions/mean_length": 8904.9375, |
| "completions/mean_terminated_length": 8904.9375, |
| "completions/min_length": 1870.0, |
| "completions/min_terminated_length": 1870.0, |
| "entropy": 0.11141919903457165, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.0392634868621826, |
| "kl": 0.0028663520533882547, |
| "learning_rate": 1.7142857142857145e-05, |
| "loss": 0.2956, |
| "num_tokens": 1231697.0, |
| "reward": 0.06170085445046425, |
| "reward_std": 0.6284129619598389, |
| "rewards/rollout_eval_reward_func/mean": 0.1812753975391388, |
| "rewards/rollout_eval_reward_func/std": 0.29651859402656555, |
| "rewards/rollout_reward_func/mean": 0.06170085445046425, |
| "rewards/rollout_reward_func/std": 0.6143175363540649, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0020291805267334, |
| "sampling/importance_sampling_ratio/min": 0.4785745441913605, |
| "sampling/sampling_logp_difference/max": 1.289139747619629, |
| "sampling/sampling_logp_difference/mean": 0.012369468808174133, |
| "step": 7, |
| "step_time": 79.23396538400038 |
| }, |
| { |
| "clip_ratio/high_max": 0.004530098114628345, |
| "clip_ratio/high_mean": 0.0022650490573141724, |
| "clip_ratio/low_mean": 0.0062050665146671236, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.008470115601085126, |
| "entropy": 0.10932181030511856, |
| "epoch": 8e-05, |
| "grad_norm": 1.4861699342727661, |
| "kl": 0.004455661201063776, |
| "learning_rate": 2e-05, |
| "loss": 0.291, |
| "step": 8, |
| "step_time": 33.553139639999245 |
| }, |
| { |
| "clip_ratio/high_max": 0.006702587008476257, |
| "clip_ratio/high_mean": 0.00395225512329489, |
| "clip_ratio/low_mean": 0.001561393088195473, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.005513648211490363, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11738.0, |
| "completions/max_terminated_length": 11738.0, |
| "completions/mean_length": 7849.21875, |
| "completions/mean_terminated_length": 7849.21875, |
| "completions/min_length": 323.0, |
| "completions/min_terminated_length": 323.0, |
| "entropy": 0.10142783727496862, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1208703517913818, |
| "kl": 0.005338215534720803, |
| "learning_rate": 2.2857142857142858e-05, |
| "loss": -0.0444, |
| "num_tokens": 1511511.0, |
| "reward": -0.1137353852391243, |
| "reward_std": 0.6742830872535706, |
| "rewards/rollout_eval_reward_func/mean": 0.21138212084770203, |
| "rewards/rollout_eval_reward_func/std": 0.2764902412891388, |
| "rewards/rollout_reward_func/mean": -0.1137353852391243, |
| "rewards/rollout_reward_func/std": 0.6757553219795227, |
| "sampling/importance_sampling_ratio/max": 1.9115883111953735, |
| "sampling/importance_sampling_ratio/mean": 0.9999130964279175, |
| "sampling/importance_sampling_ratio/min": 0.6430048942565918, |
| "sampling/sampling_logp_difference/max": 0.6479344367980957, |
| "sampling/sampling_logp_difference/mean": 0.009747568517923355, |
| "step": 9, |
| "step_time": 78.92596304499875 |
| }, |
| { |
| "clip_ratio/high_max": 0.014946904906537384, |
| "clip_ratio/high_mean": 0.008911944547435269, |
| "clip_ratio/low_mean": 0.008717332733795047, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.017629277397645637, |
| "entropy": 0.09894524328410625, |
| "epoch": 0.0001, |
| "grad_norm": 0.8779318332672119, |
| "kl": 0.012525228099548258, |
| "learning_rate": 2.5714285714285714e-05, |
| "loss": -0.0474, |
| "step": 10, |
| "step_time": 33.86218772000029 |
| }, |
| { |
| "clip_ratio/high_max": 0.005196896672714502, |
| "clip_ratio/high_mean": 0.002598448336357251, |
| "clip_ratio/low_mean": 0.00565887795528397, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00825732629164122, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11663.0, |
| "completions/max_terminated_length": 11663.0, |
| "completions/mean_length": 9176.8125, |
| "completions/mean_terminated_length": 9176.8125, |
| "completions/min_length": 2189.0, |
| "completions/min_terminated_length": 2189.0, |
| "entropy": 0.1143840029835701, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.9076919555664062, |
| "kl": 0.02556461405765731, |
| "learning_rate": 2.857142857142857e-05, |
| "loss": 0.1457, |
| "num_tokens": 1833833.0, |
| "reward": -0.07531464844942093, |
| "reward_std": 0.6547598838806152, |
| "rewards/rollout_eval_reward_func/mean": 0.1659044623374939, |
| "rewards/rollout_eval_reward_func/std": 0.27956825494766235, |
| "rewards/rollout_reward_func/mean": -0.07531464844942093, |
| "rewards/rollout_reward_func/std": 0.6382541060447693, |
| "sampling/importance_sampling_ratio/max": 2.9357423782348633, |
| "sampling/importance_sampling_ratio/mean": 0.9987922310829163, |
| "sampling/importance_sampling_ratio/min": 0.4146113395690918, |
| "sampling/sampling_logp_difference/max": 1.0769603252410889, |
| "sampling/sampling_logp_difference/mean": 0.01623542606830597, |
| "step": 11, |
| "step_time": 85.64898062899738 |
| }, |
| { |
| "clip_ratio/high_max": 0.007925633108243346, |
| "clip_ratio/high_mean": 0.003962816554121673, |
| "clip_ratio/low_mean": 0.013630853296490386, |
| "clip_ratio/low_min": 0.004900663043372333, |
| "clip_ratio/region_mean": 0.01759366990881972, |
| "entropy": 0.11427765572443604, |
| "epoch": 0.00012, |
| "grad_norm": 1.6511975526809692, |
| "kl": 0.048679544823244214, |
| "learning_rate": 3.142857142857143e-05, |
| "loss": 0.142, |
| "step": 12, |
| "step_time": 34.99860805099979 |
| }, |
| { |
| "clip_ratio/high_max": 0.003243650193326175, |
| "clip_ratio/high_mean": 0.0016218250966630876, |
| "clip_ratio/low_mean": 0.0024501511070411652, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.004071976203704253, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11007.0, |
| "completions/max_terminated_length": 11007.0, |
| "completions/mean_length": 8355.625, |
| "completions/mean_terminated_length": 8355.625, |
| "completions/min_length": 874.0, |
| "completions/min_terminated_length": 874.0, |
| "entropy": 0.11024783831089735, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.4035089015960693, |
| "kl": 0.08716274873586372, |
| "learning_rate": 3.428571428571429e-05, |
| "loss": 0.1218, |
| "num_tokens": 2130373.0, |
| "reward": 0.020019814372062683, |
| "reward_std": 0.6314284801483154, |
| "rewards/rollout_eval_reward_func/mean": 0.21633638441562653, |
| "rewards/rollout_eval_reward_func/std": 0.31151828169822693, |
| "rewards/rollout_reward_func/mean": 0.020019814372062683, |
| "rewards/rollout_reward_func/std": 0.689924418926239, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0034040212631226, |
| "sampling/importance_sampling_ratio/min": 0.20111392438411713, |
| "sampling/sampling_logp_difference/max": 1.6038837432861328, |
| "sampling/sampling_logp_difference/mean": 0.02121809497475624, |
| "step": 13, |
| "step_time": 86.70206161200258 |
| }, |
| { |
| "clip_ratio/high_max": 0.009814315650146455, |
| "clip_ratio/high_mean": 0.005678762652678415, |
| "clip_ratio/low_mean": 0.012552198284538463, |
| "clip_ratio/low_min": 0.004237600835040212, |
| "clip_ratio/region_mean": 0.01823096099542454, |
| "entropy": 0.10833629406988621, |
| "epoch": 0.00014, |
| "grad_norm": 2.254364490509033, |
| "kl": 0.07821251518907957, |
| "learning_rate": 3.7142857142857143e-05, |
| "loss": 0.1127, |
| "step": 14, |
| "step_time": 33.7453661569989 |
| }, |
| { |
| "clip_ratio/high_max": 0.004190173640381545, |
| "clip_ratio/high_mean": 0.0020950868201907724, |
| "clip_ratio/low_mean": 0.0028092548600398004, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.004904341680230573, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10761.0, |
| "completions/max_terminated_length": 10761.0, |
| "completions/mean_length": 8177.6875, |
| "completions/mean_terminated_length": 8177.6875, |
| "completions/min_length": 811.0, |
| "completions/min_terminated_length": 811.0, |
| "entropy": 0.10465786419808865, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6561901569366455, |
| "kl": 0.1072866928880103, |
| "learning_rate": 4e-05, |
| "loss": 0.356, |
| "num_tokens": 2420931.0, |
| "reward": 0.11278900504112244, |
| "reward_std": 0.6452403664588928, |
| "rewards/rollout_eval_reward_func/mean": 0.23056402802467346, |
| "rewards/rollout_eval_reward_func/std": 0.3142653703689575, |
| "rewards/rollout_reward_func/mean": 0.11278900504112244, |
| "rewards/rollout_reward_func/std": 0.6579005718231201, |
| "sampling/importance_sampling_ratio/max": 2.278707504272461, |
| "sampling/importance_sampling_ratio/mean": 0.9995414018630981, |
| "sampling/importance_sampling_ratio/min": 0.18554040789604187, |
| "sampling/sampling_logp_difference/max": 1.6844825744628906, |
| "sampling/sampling_logp_difference/mean": 0.023679137229919434, |
| "step": 15, |
| "step_time": 84.81073227599518 |
| }, |
| { |
| "clip_ratio/high_max": 0.00427410431439057, |
| "clip_ratio/high_mean": 0.002137052157195285, |
| "clip_ratio/low_mean": 0.013172850711271167, |
| "clip_ratio/low_min": 0.00423778424737975, |
| "clip_ratio/region_mean": 0.015309902926674113, |
| "entropy": 0.1049683466553688, |
| "epoch": 0.00016, |
| "grad_norm": 1.5131983757019043, |
| "kl": 0.12803897575940937, |
| "learning_rate": 4.2857142857142856e-05, |
| "loss": 0.3499, |
| "step": 16, |
| "step_time": 32.4860796890016 |
| }, |
| { |
| "clip_ratio/high_max": 0.006543069262988865, |
| "clip_ratio/high_mean": 0.003775566816329956, |
| "clip_ratio/low_mean": 0.0031609196157660335, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.00693648643209599, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11153.0, |
| "completions/max_terminated_length": 11153.0, |
| "completions/mean_length": 8529.6875, |
| "completions/mean_terminated_length": 8529.6875, |
| "completions/min_length": 330.0, |
| "completions/min_terminated_length": 330.0, |
| "entropy": 0.10206164792180061, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3023760318756104, |
| "kl": 0.09016790508758277, |
| "learning_rate": 4.5714285714285716e-05, |
| "loss": -0.055, |
| "num_tokens": 2722874.0, |
| "reward": 0.0452897846698761, |
| "reward_std": 0.5471868515014648, |
| "rewards/rollout_eval_reward_func/mean": 0.16552338004112244, |
| "rewards/rollout_eval_reward_func/std": 0.2769395411014557, |
| "rewards/rollout_reward_func/mean": 0.0452897846698761, |
| "rewards/rollout_reward_func/std": 0.5707083344459534, |
| "sampling/importance_sampling_ratio/max": 1.9813306331634521, |
| "sampling/importance_sampling_ratio/mean": 0.9965455532073975, |
| "sampling/importance_sampling_ratio/min": 0.1743103712797165, |
| "sampling/sampling_logp_difference/max": 1.746917724609375, |
| "sampling/sampling_logp_difference/mean": 0.021249916404485703, |
| "step": 17, |
| "step_time": 93.31380597800035 |
| }, |
| { |
| "clip_ratio/high_max": 0.013804474321659654, |
| "clip_ratio/high_mean": 0.008459640172077343, |
| "clip_ratio/low_mean": 0.009875382616883144, |
| "clip_ratio/low_min": 0.002192793588619679, |
| "clip_ratio/region_mean": 0.018335022643441334, |
| "entropy": 0.10283505916595459, |
| "epoch": 0.00018, |
| "grad_norm": 1.0806056261062622, |
| "kl": 0.11839380749734119, |
| "learning_rate": 4.8571428571428576e-05, |
| "loss": -0.0586, |
| "step": 18, |
| "step_time": 32.5428760199984 |
| }, |
| { |
| "clip_ratio/high_max": 0.004807963909115642, |
| "clip_ratio/high_mean": 0.002403981954557821, |
| "clip_ratio/low_mean": 0.004428166459547356, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.006832148414105177, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10399.0, |
| "completions/max_terminated_length": 10399.0, |
| "completions/mean_length": 7525.5625, |
| "completions/mean_terminated_length": 7525.5625, |
| "completions/min_length": 1309.0, |
| "completions/min_terminated_length": 1309.0, |
| "entropy": 0.10726649826392531, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3821837902069092, |
| "kl": 0.09694959409534931, |
| "learning_rate": 5.142857142857143e-05, |
| "loss": 0.2508, |
| "num_tokens": 2992614.0, |
| "reward": 0.22085225582122803, |
| "reward_std": 0.7439246773719788, |
| "rewards/rollout_eval_reward_func/mean": 0.32672765851020813, |
| "rewards/rollout_eval_reward_func/std": 0.3242030739784241, |
| "rewards/rollout_reward_func/mean": 0.22085225582122803, |
| "rewards/rollout_reward_func/std": 0.7110569477081299, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0050089359283447, |
| "sampling/importance_sampling_ratio/min": 0.2929533123970032, |
| "sampling/sampling_logp_difference/max": 1.916597604751587, |
| "sampling/sampling_logp_difference/mean": 0.022352777421474457, |
| "step": 19, |
| "step_time": 89.20460286800153 |
| }, |
| { |
| "clip_ratio/high_max": 0.011035054223611951, |
| "clip_ratio/high_mean": 0.0055175271118059754, |
| "clip_ratio/low_mean": 0.018963277572765946, |
| "clip_ratio/low_min": 0.0055555556900799274, |
| "clip_ratio/region_mean": 0.024480804742779583, |
| "entropy": 0.1047364235855639, |
| "epoch": 0.0002, |
| "grad_norm": 0.9138187170028687, |
| "kl": 0.10579214652534574, |
| "learning_rate": 5.428571428571428e-05, |
| "loss": 0.2408, |
| "step": 20, |
| "step_time": 30.517773796002075 |
| }, |
| { |
| "clip_ratio/high_max": 0.004395612399093807, |
| "clip_ratio/high_mean": 0.0028961520874872804, |
| "clip_ratio/low_mean": 0.0034880968159995973, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.006384248874383047, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11335.0, |
| "completions/max_terminated_length": 11335.0, |
| "completions/mean_length": 8358.6875, |
| "completions/mean_terminated_length": 8358.6875, |
| "completions/min_length": 749.0, |
| "completions/min_terminated_length": 749.0, |
| "entropy": 0.12294450332410634, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3461740016937256, |
| "kl": 0.19072263734415174, |
| "learning_rate": 5.714285714285714e-05, |
| "loss": -0.0037, |
| "num_tokens": 3288905.0, |
| "reward": -0.1559383124113083, |
| "reward_std": 0.5600723028182983, |
| "rewards/rollout_eval_reward_func/mean": 0.14430895447731018, |
| "rewards/rollout_eval_reward_func/std": 0.2627376317977905, |
| "rewards/rollout_reward_func/mean": -0.1559383124113083, |
| "rewards/rollout_reward_func/std": 0.5887359380722046, |
| "sampling/importance_sampling_ratio/max": 2.2654411792755127, |
| "sampling/importance_sampling_ratio/mean": 0.9976513385772705, |
| "sampling/importance_sampling_ratio/min": 0.20272883772850037, |
| "sampling/sampling_logp_difference/max": 1.595885992050171, |
| "sampling/sampling_logp_difference/mean": 0.02302836999297142, |
| "step": 21, |
| "step_time": 97.79121274199952 |
| }, |
| { |
| "clip_ratio/high_max": 0.015567402355372906, |
| "clip_ratio/high_mean": 0.009180392837151885, |
| "clip_ratio/low_mean": 0.017762881761882454, |
| "clip_ratio/low_min": 0.006345431203953922, |
| "clip_ratio/region_mean": 0.026943274919176474, |
| "entropy": 0.12321826652623713, |
| "epoch": 0.00022, |
| "grad_norm": 1.64430570602417, |
| "kl": 0.269085428211838, |
| "learning_rate": 6e-05, |
| "loss": -0.0084, |
| "step": 22, |
| "step_time": 32.807887424998626 |
| }, |
| { |
| "clip_ratio/high_max": 0.003910088969860226, |
| "clip_ratio/high_mean": 0.001955044484930113, |
| "clip_ratio/low_mean": 0.002580859523732215, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004535904008662328, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11344.0, |
| "completions/max_terminated_length": 11344.0, |
| "completions/mean_length": 8706.71875, |
| "completions/mean_terminated_length": 8706.71875, |
| "completions/min_length": 1846.0, |
| "completions/min_terminated_length": 1846.0, |
| "entropy": 0.1066858924459666, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.9362024068832397, |
| "kl": 0.12137257889844477, |
| "learning_rate": 6.285714285714286e-05, |
| "loss": 0.2385, |
| "num_tokens": 3596056.0, |
| "reward": 0.12735633552074432, |
| "reward_std": 0.6423711776733398, |
| "rewards/rollout_eval_reward_func/mean": 0.2489837408065796, |
| "rewards/rollout_eval_reward_func/std": 0.3152793347835541, |
| "rewards/rollout_reward_func/mean": 0.12735633552074432, |
| "rewards/rollout_reward_func/std": 0.7003796696662903, |
| "sampling/importance_sampling_ratio/max": 2.351552963256836, |
| "sampling/importance_sampling_ratio/mean": 1.0003013610839844, |
| "sampling/importance_sampling_ratio/min": 0.3779045045375824, |
| "sampling/sampling_logp_difference/max": 0.9731137752532959, |
| "sampling/sampling_logp_difference/mean": 0.016432534903287888, |
| "step": 23, |
| "step_time": 99.05041999199784 |
| }, |
| { |
| "clip_ratio/high_max": 0.011161994189023972, |
| "clip_ratio/high_mean": 0.0066947695158887655, |
| "clip_ratio/low_mean": 0.009474401478655636, |
| "clip_ratio/low_min": 0.0021156331058591604, |
| "clip_ratio/region_mean": 0.01616917096544057, |
| "entropy": 0.11043372261337936, |
| "epoch": 0.00024, |
| "grad_norm": 1.812207818031311, |
| "kl": 0.09031509491614997, |
| "learning_rate": 6.571428571428571e-05, |
| "loss": 0.2393, |
| "step": 24, |
| "step_time": 32.823300022000694 |
| }, |
| { |
| "clip_ratio/high_max": 0.006489152496214956, |
| "clip_ratio/high_mean": 0.003244576248107478, |
| "clip_ratio/low_mean": 0.0018077631539199501, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.005052339431131259, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10988.0, |
| "completions/max_terminated_length": 10988.0, |
| "completions/mean_length": 8457.65625, |
| "completions/mean_terminated_length": 8457.65625, |
| "completions/min_length": 1525.0, |
| "completions/min_terminated_length": 1525.0, |
| "entropy": 0.11516976961866021, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3577765226364136, |
| "kl": 0.1303024498047307, |
| "learning_rate": 6.857142857142858e-05, |
| "loss": 0.1265, |
| "num_tokens": 3894768.0, |
| "reward": 0.20315799117088318, |
| "reward_std": 0.6316713094711304, |
| "rewards/rollout_eval_reward_func/mean": 0.240091472864151, |
| "rewards/rollout_eval_reward_func/std": 0.31059667468070984, |
| "rewards/rollout_reward_func/mean": 0.20315799117088318, |
| "rewards/rollout_reward_func/std": 0.6295517683029175, |
| "sampling/importance_sampling_ratio/max": 2.256138801574707, |
| "sampling/importance_sampling_ratio/mean": 0.9993441104888916, |
| "sampling/importance_sampling_ratio/min": 0.2282668501138687, |
| "sampling/sampling_logp_difference/max": 1.477239966392517, |
| "sampling/sampling_logp_difference/mean": 0.01688190922141075, |
| "step": 25, |
| "step_time": 96.7960174280006 |
| }, |
| { |
| "clip_ratio/high_max": 0.016165015986189246, |
| "clip_ratio/high_mean": 0.010140184662304819, |
| "clip_ratio/low_mean": 0.027886587660759687, |
| "clip_ratio/low_min": 0.013233458623290062, |
| "clip_ratio/region_mean": 0.038026772264856845, |
| "entropy": 0.11189563758671284, |
| "epoch": 0.00026, |
| "grad_norm": 2.0246684551239014, |
| "kl": 0.2296465744730085, |
| "learning_rate": 7.142857142857143e-05, |
| "loss": 0.1164, |
| "step": 26, |
| "step_time": 32.81865678699978 |
| }, |
| { |
| "clip_ratio/high_max": 0.0038667930057272315, |
| "clip_ratio/high_mean": 0.0019333965028636158, |
| "clip_ratio/low_mean": 0.003156299644615501, |
| "clip_ratio/low_min": 0.0014046717551536858, |
| "clip_ratio/region_mean": 0.005089696147479117, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11043.0, |
| "completions/max_terminated_length": 11043.0, |
| "completions/mean_length": 8288.09375, |
| "completions/mean_terminated_length": 8288.09375, |
| "completions/min_length": 873.0, |
| "completions/min_terminated_length": 873.0, |
| "entropy": 0.1172712086699903, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.0188708305358887, |
| "kl": 0.14066953863948584, |
| "learning_rate": 7.428571428571429e-05, |
| "loss": 0.019, |
| "num_tokens": 4187980.0, |
| "reward": 0.3152431547641754, |
| "reward_std": 0.6300436854362488, |
| "rewards/rollout_eval_reward_func/mean": 0.31758129596710205, |
| "rewards/rollout_eval_reward_func/std": 0.32922983169555664, |
| "rewards/rollout_reward_func/mean": 0.3152431547641754, |
| "rewards/rollout_reward_func/std": 0.6763298511505127, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 0.9995622634887695, |
| "sampling/importance_sampling_ratio/min": 0.4497600793838501, |
| "sampling/sampling_logp_difference/max": 1.9223380088806152, |
| "sampling/sampling_logp_difference/mean": 0.017684465274214745, |
| "step": 27, |
| "step_time": 96.53867048800112 |
| }, |
| { |
| "clip_ratio/high_max": 0.01634173752972856, |
| "clip_ratio/high_mean": 0.00923296029213816, |
| "clip_ratio/low_mean": 0.011995688633760437, |
| "clip_ratio/low_min": 0.0042134831310249865, |
| "clip_ratio/region_mean": 0.021228648955002427, |
| "entropy": 0.11727899359539151, |
| "epoch": 0.00028, |
| "grad_norm": 1.0265206098556519, |
| "kl": 0.1646709272172302, |
| "learning_rate": 7.714285714285715e-05, |
| "loss": 0.0122, |
| "step": 28, |
| "step_time": 32.412335121999604 |
| }, |
| { |
| "clip_ratio/high_max": 0.0030616914154961705, |
| "clip_ratio/high_mean": 0.0015308457077480853, |
| "clip_ratio/low_mean": 0.0017557066748850048, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003286552324425429, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11171.0, |
| "completions/max_terminated_length": 11171.0, |
| "completions/mean_length": 7933.9375, |
| "completions/mean_terminated_length": 7933.9375, |
| "completions/min_length": 1855.0, |
| "completions/min_terminated_length": 1855.0, |
| "entropy": 0.10820061061531305, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.078338623046875, |
| "kl": 0.12938267178833485, |
| "learning_rate": 8e-05, |
| "loss": 0.1072, |
| "num_tokens": 4469763.0, |
| "reward": 0.19558671116828918, |
| "reward_std": 0.6997864842414856, |
| "rewards/rollout_eval_reward_func/mean": 0.2963668704032898, |
| "rewards/rollout_eval_reward_func/std": 0.3109610080718994, |
| "rewards/rollout_reward_func/mean": 0.19558671116828918, |
| "rewards/rollout_reward_func/std": 0.6721439361572266, |
| "sampling/importance_sampling_ratio/max": 1.8601394891738892, |
| "sampling/importance_sampling_ratio/mean": 1.0000590085983276, |
| "sampling/importance_sampling_ratio/min": 0.5378966927528381, |
| "sampling/sampling_logp_difference/max": 0.6206514835357666, |
| "sampling/sampling_logp_difference/mean": 0.01655469462275505, |
| "step": 29, |
| "step_time": 93.66265234299863 |
| }, |
| { |
| "clip_ratio/high_max": 0.017240845831111073, |
| "clip_ratio/high_mean": 0.009188604657538235, |
| "clip_ratio/low_mean": 0.018311504449229687, |
| "clip_ratio/low_min": 0.004907927650492638, |
| "clip_ratio/region_mean": 0.027500109281390905, |
| "entropy": 0.10658967308700085, |
| "epoch": 0.0003, |
| "grad_norm": 0.5991150140762329, |
| "kl": 0.14486833242699504, |
| "learning_rate": 8.285714285714287e-05, |
| "loss": 0.0961, |
| "step": 30, |
| "step_time": 32.02409046399953 |
| }, |
| { |
| "clip_ratio/high_max": 0.007639755902346224, |
| "clip_ratio/high_mean": 0.003819877951173112, |
| "clip_ratio/low_mean": 0.0032558340462855995, |
| "clip_ratio/low_min": 0.000811688310932368, |
| "clip_ratio/region_mean": 0.007075712055666372, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11207.0, |
| "completions/max_terminated_length": 11207.0, |
| "completions/mean_length": 7446.4375, |
| "completions/mean_terminated_length": 7446.4375, |
| "completions/min_length": 1941.0, |
| "completions/min_terminated_length": 1941.0, |
| "entropy": 0.1075986991636455, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3741053342819214, |
| "kl": 0.1601699753664434, |
| "learning_rate": 8.571428571428571e-05, |
| "loss": 0.3119, |
| "num_tokens": 4736188.0, |
| "reward": 0.3060583770275116, |
| "reward_std": 0.7248314619064331, |
| "rewards/rollout_eval_reward_func/mean": 0.3489583134651184, |
| "rewards/rollout_eval_reward_func/std": 0.3138476312160492, |
| "rewards/rollout_reward_func/mean": 0.3060583770275116, |
| "rewards/rollout_reward_func/std": 0.7231479287147522, |
| "sampling/importance_sampling_ratio/max": 2.0959339141845703, |
| "sampling/importance_sampling_ratio/mean": 1.0005576610565186, |
| "sampling/importance_sampling_ratio/min": 0.3962559998035431, |
| "sampling/sampling_logp_difference/max": 0.9256948232650757, |
| "sampling/sampling_logp_difference/mean": 0.015314233489334583, |
| "step": 31, |
| "step_time": 89.359079396997 |
| }, |
| { |
| "clip_ratio/high_max": 0.018800459045451134, |
| "clip_ratio/high_mean": 0.010931086115306243, |
| "clip_ratio/low_mean": 0.014964523201342672, |
| "clip_ratio/low_min": 0.002763190073892474, |
| "clip_ratio/region_mean": 0.025895609316648915, |
| "entropy": 0.11107803648337722, |
| "epoch": 0.00032, |
| "grad_norm": 0.8436726927757263, |
| "kl": 0.17236854787915945, |
| "learning_rate": 8.857142857142857e-05, |
| "loss": 0.2984, |
| "step": 32, |
| "step_time": 32.26129659800063 |
| }, |
| { |
| "clip_ratio/high_max": 0.005184833600651473, |
| "clip_ratio/high_mean": 0.0025924168003257364, |
| "clip_ratio/low_mean": 0.0021227904653642327, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004715207265689969, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11234.0, |
| "completions/max_terminated_length": 11234.0, |
| "completions/mean_length": 8056.90625, |
| "completions/mean_terminated_length": 8056.90625, |
| "completions/min_length": 834.0, |
| "completions/min_terminated_length": 834.0, |
| "entropy": 0.10890151979401708, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.012204647064209, |
| "kl": 0.13646527798846364, |
| "learning_rate": 9.142857142857143e-05, |
| "loss": 0.2208, |
| "num_tokens": 5021855.0, |
| "reward": 0.28810450434684753, |
| "reward_std": 0.6511638164520264, |
| "rewards/rollout_eval_reward_func/mean": 0.27172255516052246, |
| "rewards/rollout_eval_reward_func/std": 0.3220342993736267, |
| "rewards/rollout_reward_func/mean": 0.28810450434684753, |
| "rewards/rollout_reward_func/std": 0.6260398030281067, |
| "sampling/importance_sampling_ratio/max": 1.8270360231399536, |
| "sampling/importance_sampling_ratio/mean": 1.0004476308822632, |
| "sampling/importance_sampling_ratio/min": 0.2768998444080353, |
| "sampling/sampling_logp_difference/max": 1.2840993404388428, |
| "sampling/sampling_logp_difference/mean": 0.012767134234309196, |
| "step": 33, |
| "step_time": 93.59931743799825 |
| }, |
| { |
| "clip_ratio/high_max": 0.013980166870169342, |
| "clip_ratio/high_mean": 0.008351352356839925, |
| "clip_ratio/low_mean": 0.016900451679248363, |
| "clip_ratio/low_min": 0.0068813529796898365, |
| "clip_ratio/region_mean": 0.025251803919672966, |
| "entropy": 0.1073108701966703, |
| "epoch": 0.00034, |
| "grad_norm": 0.6274716258049011, |
| "kl": 0.14035740494728088, |
| "learning_rate": 9.428571428571429e-05, |
| "loss": 0.2118, |
| "step": 34, |
| "step_time": 32.919508103002954 |
| }, |
| { |
| "clip_ratio/high_max": 0.0018894725944846869, |
| "clip_ratio/high_mean": 0.0009447362972423434, |
| "clip_ratio/low_mean": 0.0021555489802267402, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0031002852774690837, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11099.0, |
| "completions/max_terminated_length": 11099.0, |
| "completions/mean_length": 8609.5, |
| "completions/mean_terminated_length": 8609.5, |
| "completions/min_length": 1070.0, |
| "completions/min_terminated_length": 1070.0, |
| "entropy": 0.10947995679453015, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.9109163284301758, |
| "kl": 0.24753205897286534, |
| "learning_rate": 9.714285714285715e-05, |
| "loss": 0.1542, |
| "num_tokens": 5325755.0, |
| "reward": 0.4508249759674072, |
| "reward_std": 0.5780479907989502, |
| "rewards/rollout_eval_reward_func/mean": 0.32621949911117554, |
| "rewards/rollout_eval_reward_func/std": 0.3382527828216553, |
| "rewards/rollout_reward_func/mean": 0.4508249759674072, |
| "rewards/rollout_reward_func/std": 0.5698684453964233, |
| "sampling/importance_sampling_ratio/max": 1.7834506034851074, |
| "sampling/importance_sampling_ratio/mean": 1.0010955333709717, |
| "sampling/importance_sampling_ratio/min": 0.366268515586853, |
| "sampling/sampling_logp_difference/max": 1.0043885707855225, |
| "sampling/sampling_logp_difference/mean": 0.013199402019381523, |
| "step": 35, |
| "step_time": 93.34705402800137 |
| }, |
| { |
| "clip_ratio/high_max": 0.0200398035813123, |
| "clip_ratio/high_mean": 0.01071434619370848, |
| "clip_ratio/low_mean": 0.010386000329162925, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.021100346348248422, |
| "entropy": 0.11280779354274273, |
| "epoch": 0.00036, |
| "grad_norm": 0.8509212732315063, |
| "kl": 0.24738536309450865, |
| "learning_rate": 0.0001, |
| "loss": 0.1443, |
| "step": 36, |
| "step_time": 33.26461400700191 |
| }, |
| { |
| "clip_ratio/high_max": 0.0045615925919264555, |
| "clip_ratio/high_mean": 0.002661893842741847, |
| "clip_ratio/low_mean": 0.001787908433470875, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004449802276212722, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10712.0, |
| "completions/max_terminated_length": 10712.0, |
| "completions/mean_length": 7822.9375, |
| "completions/mean_terminated_length": 7822.9375, |
| "completions/min_length": 1061.0, |
| "completions/min_terminated_length": 1061.0, |
| "entropy": 0.09347328078001738, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.7398293018341064, |
| "kl": 0.14611079543828964, |
| "learning_rate": 9.918053502751771e-05, |
| "loss": 0.0539, |
| "num_tokens": 5604207.0, |
| "reward": 0.2784110903739929, |
| "reward_std": 0.6512280702590942, |
| "rewards/rollout_eval_reward_func/mean": 0.2896341383457184, |
| "rewards/rollout_eval_reward_func/std": 0.32505732774734497, |
| "rewards/rollout_reward_func/mean": 0.2784110903739929, |
| "rewards/rollout_reward_func/std": 0.6340307593345642, |
| "sampling/importance_sampling_ratio/max": 1.785902500152588, |
| "sampling/importance_sampling_ratio/mean": 0.9995973110198975, |
| "sampling/importance_sampling_ratio/min": 0.3934570252895355, |
| "sampling/sampling_logp_difference/max": 0.9327834844589233, |
| "sampling/sampling_logp_difference/mean": 0.01090280432254076, |
| "step": 37, |
| "step_time": 89.90687796200018 |
| }, |
| { |
| "clip_ratio/high_max": 0.014242319972254336, |
| "clip_ratio/high_mean": 0.007633455039467663, |
| "clip_ratio/low_mean": 0.009654591587604955, |
| "clip_ratio/low_min": 0.001453684933949262, |
| "clip_ratio/region_mean": 0.017288046510657296, |
| "entropy": 0.09478882700204849, |
| "epoch": 0.00038, |
| "grad_norm": 0.5018179416656494, |
| "kl": 0.15739013627171516, |
| "learning_rate": 9.675795466159755e-05, |
| "loss": 0.0474, |
| "step": 38, |
| "step_time": 32.40730591600277 |
| }, |
| { |
| "clip_ratio/high_max": 0.0033184904023073614, |
| "clip_ratio/high_mean": 0.0016592452011536807, |
| "clip_ratio/low_mean": 0.002118892065482214, |
| "clip_ratio/low_min": 0.0007102272938936949, |
| "clip_ratio/region_mean": 0.0037781372957397252, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10878.0, |
| "completions/max_terminated_length": 10878.0, |
| "completions/mean_length": 8825.25, |
| "completions/mean_terminated_length": 8825.25, |
| "completions/min_length": 2113.0, |
| "completions/min_terminated_length": 2113.0, |
| "entropy": 0.13806713884696364, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.113547444343567, |
| "kl": 0.20099136885255575, |
| "learning_rate": 9.283813728906054e-05, |
| "loss": 0.1486, |
| "num_tokens": 5915217.0, |
| "reward": 0.18237435817718506, |
| "reward_std": 0.5858109593391418, |
| "rewards/rollout_eval_reward_func/mean": 0.23246951401233673, |
| "rewards/rollout_eval_reward_func/std": 0.3161698281764984, |
| "rewards/rollout_reward_func/mean": 0.18237435817718506, |
| "rewards/rollout_reward_func/std": 0.6235412955284119, |
| "sampling/importance_sampling_ratio/max": 1.8600788116455078, |
| "sampling/importance_sampling_ratio/mean": 1.0009781122207642, |
| "sampling/importance_sampling_ratio/min": 0.6195499897003174, |
| "sampling/sampling_logp_difference/max": 0.6206188201904297, |
| "sampling/sampling_logp_difference/mean": 0.01361357793211937, |
| "step": 39, |
| "step_time": 95.42290947900165 |
| }, |
| { |
| "clip_ratio/high_max": 0.01989291445352137, |
| "clip_ratio/high_mean": 0.010865762596949935, |
| "clip_ratio/low_mean": 0.01921083332854323, |
| "clip_ratio/low_min": 0.004245770338457078, |
| "clip_ratio/region_mean": 0.030076596303842962, |
| "entropy": 0.13848060183227062, |
| "epoch": 0.0004, |
| "grad_norm": 0.8446061611175537, |
| "kl": 0.22338007856160402, |
| "learning_rate": 8.759239773845718e-05, |
| "loss": 0.1364, |
| "step": 40, |
| "step_time": 33.852559433999886 |
| }, |
| { |
| "clip_ratio/high_max": 0.0030503973248414695, |
| "clip_ratio/high_mean": 0.0015251986624207348, |
| "clip_ratio/low_mean": 0.0010654326470103115, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0025906313094310462, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11107.0, |
| "completions/max_terminated_length": 11107.0, |
| "completions/mean_length": 8257.21875, |
| "completions/mean_terminated_length": 8257.21875, |
| "completions/min_length": 2666.0, |
| "completions/min_terminated_length": 2666.0, |
| "entropy": 0.11340034916065633, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0817996263504028, |
| "kl": 0.20612840354442596, |
| "learning_rate": 8.125000000000001e-05, |
| "loss": 0.3665, |
| "num_tokens": 6207607.0, |
| "reward": 0.42999449372291565, |
| "reward_std": 0.5903699994087219, |
| "rewards/rollout_eval_reward_func/mean": 0.33269816637039185, |
| "rewards/rollout_eval_reward_func/std": 0.343418687582016, |
| "rewards/rollout_reward_func/mean": 0.42999449372291565, |
| "rewards/rollout_reward_func/std": 0.5738593935966492, |
| "sampling/importance_sampling_ratio/max": 1.4319037199020386, |
| "sampling/importance_sampling_ratio/mean": 0.9990136027336121, |
| "sampling/importance_sampling_ratio/min": 0.4935825765132904, |
| "sampling/sampling_logp_difference/max": 0.7060651779174805, |
| "sampling/sampling_logp_difference/mean": 0.011390198953449726, |
| "step": 41, |
| "step_time": 91.59171992200027 |
| }, |
| { |
| "clip_ratio/high_max": 0.01730476424563676, |
| "clip_ratio/high_mean": 0.009731545054819435, |
| "clip_ratio/low_mean": 0.01579960942035541, |
| "clip_ratio/low_min": 0.006750052853021771, |
| "clip_ratio/region_mean": 0.025531154591590166, |
| "entropy": 0.11443871166557074, |
| "epoch": 0.00042, |
| "grad_norm": 0.7992480397224426, |
| "kl": 0.27106572315096855, |
| "learning_rate": 7.408813728906053e-05, |
| "loss": 0.3537, |
| "step": 42, |
| "step_time": 33.60049366600106 |
| }, |
| { |
| "clip_ratio/high_max": 0.0035054103354923427, |
| "clip_ratio/high_mean": 0.0017527051677461714, |
| "clip_ratio/low_mean": 0.0010694226948544383, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0028221279208082706, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10896.0, |
| "completions/max_terminated_length": 10896.0, |
| "completions/mean_length": 7894.15625, |
| "completions/mean_terminated_length": 7894.15625, |
| "completions/min_length": 2890.0, |
| "completions/min_terminated_length": 2890.0, |
| "entropy": 0.13558923918753862, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.9851862192153931, |
| "kl": 0.24445611471310258, |
| "learning_rate": 6.6419817372537e-05, |
| "loss": 0.1226, |
| "num_tokens": 6488287.0, |
| "reward": 0.18586361408233643, |
| "reward_std": 0.6816079616546631, |
| "rewards/rollout_eval_reward_func/mean": 0.30538618564605713, |
| "rewards/rollout_eval_reward_func/std": 0.32574331760406494, |
| "rewards/rollout_reward_func/mean": 0.18586361408233643, |
| "rewards/rollout_reward_func/std": 0.676216721534729, |
| "sampling/importance_sampling_ratio/max": 1.8440579175949097, |
| "sampling/importance_sampling_ratio/mean": 1.0000970363616943, |
| "sampling/importance_sampling_ratio/min": 0.6517351269721985, |
| "sampling/sampling_logp_difference/max": 0.6119685173034668, |
| "sampling/sampling_logp_difference/mean": 0.012094310484826565, |
| "step": 43, |
| "step_time": 92.82438364300197 |
| }, |
| { |
| "clip_ratio/high_max": 0.010269680351484567, |
| "clip_ratio/high_mean": 0.005821653438033536, |
| "clip_ratio/low_mean": 0.015070577035658062, |
| "clip_ratio/low_min": 0.0048273742431774735, |
| "clip_ratio/region_mean": 0.020892230415483937, |
| "entropy": 0.13336073094978929, |
| "epoch": 0.00044, |
| "grad_norm": 0.6729846000671387, |
| "kl": 0.2622186541557312, |
| "learning_rate": 5.858018262746301e-05, |
| "loss": 0.1142, |
| "step": 44, |
| "step_time": 32.630656069 |
| }, |
| { |
| "clip_ratio/high_max": 0.006278776447288692, |
| "clip_ratio/high_mean": 0.003139388223644346, |
| "clip_ratio/low_mean": 0.0014205463230609894, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.004559934546705335, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11255.0, |
| "completions/max_terminated_length": 11255.0, |
| "completions/mean_length": 7849.59375, |
| "completions/mean_terminated_length": 7849.59375, |
| "completions/min_length": 841.0, |
| "completions/min_terminated_length": 841.0, |
| "entropy": 0.13749901531264186, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1264039278030396, |
| "kl": 0.22676664125174284, |
| "learning_rate": 5.0911862710939485e-05, |
| "loss": -0.0333, |
| "num_tokens": 6767906.0, |
| "reward": 0.15443384647369385, |
| "reward_std": 0.693654477596283, |
| "rewards/rollout_eval_reward_func/mean": 0.2919207215309143, |
| "rewards/rollout_eval_reward_func/std": 0.29990869760513306, |
| "rewards/rollout_reward_func/mean": 0.15443384647369385, |
| "rewards/rollout_reward_func/std": 0.7213361263275146, |
| "sampling/importance_sampling_ratio/max": 1.6071842908859253, |
| "sampling/importance_sampling_ratio/mean": 0.9978848099708557, |
| "sampling/importance_sampling_ratio/min": 0.48193788528442383, |
| "sampling/sampling_logp_difference/max": 0.7299400568008423, |
| "sampling/sampling_logp_difference/mean": 0.01217048242688179, |
| "step": 45, |
| "step_time": 91.51718380699913 |
| }, |
| { |
| "clip_ratio/high_max": 0.012777922616805881, |
| "clip_ratio/high_mean": 0.006918622384546325, |
| "clip_ratio/low_mean": 0.014222246245481074, |
| "clip_ratio/low_min": 0.005445442744530737, |
| "clip_ratio/region_mean": 0.0211408686300274, |
| "entropy": 0.1343677118420601, |
| "epoch": 0.00046, |
| "grad_norm": 0.7688785791397095, |
| "kl": 0.24824338778853416, |
| "learning_rate": 4.375000000000001e-05, |
| "loss": -0.0418, |
| "step": 46, |
| "step_time": 33.231114288000754 |
| }, |
| { |
| "clip_ratio/high_max": 0.002997979463543743, |
| "clip_ratio/high_mean": 0.0018581851327326149, |
| "clip_ratio/low_mean": 0.0012389520416036248, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0030971371743362397, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 10980.0, |
| "completions/max_terminated_length": 10980.0, |
| "completions/mean_length": 7450.6875, |
| "completions/mean_terminated_length": 7450.6875, |
| "completions/min_length": 320.0, |
| "completions/min_terminated_length": 320.0, |
| "entropy": 0.11971927247941494, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0886417627334595, |
| "kl": 0.27644640300422907, |
| "learning_rate": 3.740760226154283e-05, |
| "loss": 0.0684, |
| "num_tokens": 7034802.0, |
| "reward": 0.32516512274742126, |
| "reward_std": 0.6862237453460693, |
| "rewards/rollout_eval_reward_func/mean": 0.3408282399177551, |
| "rewards/rollout_eval_reward_func/std": 0.31514763832092285, |
| "rewards/rollout_reward_func/mean": 0.32516512274742126, |
| "rewards/rollout_reward_func/std": 0.6982549428939819, |
| "sampling/importance_sampling_ratio/max": 1.4899945259094238, |
| "sampling/importance_sampling_ratio/mean": 0.9988203048706055, |
| "sampling/importance_sampling_ratio/min": 0.5982000827789307, |
| "sampling/sampling_logp_difference/max": 0.5138299465179443, |
| "sampling/sampling_logp_difference/mean": 0.010367423295974731, |
| "step": 47, |
| "step_time": 92.68640525500086 |
| }, |
| { |
| "clip_ratio/high_max": 0.005615144735202193, |
| "clip_ratio/high_mean": 0.0031981973443180323, |
| "clip_ratio/low_mean": 0.008041755587328225, |
| "clip_ratio/low_min": 0.0007102272938936949, |
| "clip_ratio/region_mean": 0.011239952960750088, |
| "entropy": 0.11511751124635339, |
| "epoch": 0.00048, |
| "grad_norm": 0.677844762802124, |
| "kl": 0.2775820675306022, |
| "learning_rate": 3.2161862710939476e-05, |
| "loss": 0.0609, |
| "step": 48, |
| "step_time": 32.67402728900106 |
| }, |
| { |
| "clip_ratio/high_max": 0.00474693434080109, |
| "clip_ratio/high_mean": 0.002373467170400545, |
| "clip_ratio/low_mean": 0.0003511235991027206, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0027245907695032656, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 11070.0, |
| "completions/max_terminated_length": 11070.0, |
| "completions/mean_length": 7308.03125, |
| "completions/mean_terminated_length": 7308.03125, |
| "completions/min_length": 854.0, |
| "completions/min_terminated_length": 854.0, |
| "entropy": 0.10222014435566962, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8171713948249817, |
| "kl": 0.22370027378201485, |
| "learning_rate": 2.8242045338402468e-05, |
| "loss": -0.068, |
| "num_tokens": 7296677.0, |
| "reward": 0.28688520193099976, |
| "reward_std": 0.7849241495132446, |
| "rewards/rollout_eval_reward_func/mean": 0.37195122241973877, |
| "rewards/rollout_eval_reward_func/std": 0.3127909302711487, |
| "rewards/rollout_reward_func/mean": 0.28688520193099976, |
| "rewards/rollout_reward_func/std": 0.7561083436012268, |
| "sampling/importance_sampling_ratio/max": 1.7672066688537598, |
| "sampling/importance_sampling_ratio/mean": 1.0014787912368774, |
| "sampling/importance_sampling_ratio/min": 0.5275546908378601, |
| "sampling/sampling_logp_difference/max": 0.639502763748169, |
| "sampling/sampling_logp_difference/mean": 0.011029250919818878, |
| "step": 49, |
| "step_time": 86.99529032999908 |
| }, |
| { |
| "clip_ratio/high_max": 0.007674970023799688, |
| "clip_ratio/high_mean": 0.004594452795572579, |
| "clip_ratio/low_mean": 0.008248247409937903, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.012842700205510482, |
| "entropy": 0.09808745188638568, |
| "epoch": 0.0005, |
| "grad_norm": 0.6178692579269409, |
| "kl": 0.23346312018111348, |
| "learning_rate": 2.5819464972482288e-05, |
| "loss": -0.0742, |
| "step": 50, |
| "step_time": 32.30095837099998 |
| }, |
| { |
| "epoch": 0.0005, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completions/clipped_ratio": 0.0, |
| "eval_completions/max_length": 10309.35, |
| "eval_completions/max_terminated_length": 10309.35, |
| "eval_completions/mean_length": 7614.075, |
| "eval_completions/mean_terminated_length": 7614.075, |
| "eval_completions/min_length": 4457.25, |
| "eval_completions/min_terminated_length": 4457.25, |
| "eval_entropy": 0.1146618027240038, |
| "eval_frac_reward_zero_std": 1.0, |
| "eval_kl": 0.2420900955796242, |
| "eval_loss": 0.00033982208697125316, |
| "eval_num_tokens": 7296677.0, |
| "eval_reward": 0.439066107571125, |
| "eval_reward_std": 0.0, |
| "eval_rewards/rollout_eval_reward_func/mean": 0.3934959270060062, |
| "eval_rewards/rollout_eval_reward_func/std": 0.30925278924405575, |
| "eval_rewards/rollout_reward_func/mean": 0.439066107571125, |
| "eval_rewards/rollout_reward_func/std": 0.643099270761013, |
| "eval_runtime": 164.9571, |
| "eval_samples_per_second": 0.061, |
| "eval_sampling/importance_sampling_ratio/max": 1.303001207113266, |
| "eval_sampling/importance_sampling_ratio/mean": 1.0004466891288757, |
| "eval_sampling/importance_sampling_ratio/min": 0.7182389438152313, |
| "eval_sampling/sampling_logp_difference/max": 0.3734502553939819, |
| "eval_sampling/sampling_logp_difference/mean": 0.009935753676109016, |
| "eval_steps_per_second": 0.018, |
| "step": 50 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 50, |
| "num_input_tokens_seen": 7296677, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
