| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.00059, |
| "eval_steps": 500, |
| "global_step": 59, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7186.0, |
| "completions/max_terminated_length": 7186.0, |
| "completions/mean_length": 6321.78125, |
| "completions/mean_terminated_length": 6321.78125, |
| "completions/min_length": 1536.0, |
| "completions/min_terminated_length": 1536.0, |
| "entropy": 0.11082597565837204, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 3.0425922870635986, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0232, |
| "num_tokens": 229639.0, |
| "reward": -0.27844929695129395, |
| "reward_std": 0.8033838272094727, |
| "rewards/rollout_reward_func/mean": -0.27844929695129395, |
| "rewards/rollout_reward_func/std": 0.9743078351020813, |
| "sampling/importance_sampling_ratio/max": 1.6097382307052612, |
| "sampling/importance_sampling_ratio/mean": 0.999904990196228, |
| "sampling/importance_sampling_ratio/min": 0.6331427693367004, |
| "sampling/sampling_logp_difference/max": 0.4760715961456299, |
| "sampling/sampling_logp_difference/mean": 0.01278754137456417, |
| "step": 1, |
| "step_time": 72.43192732900013 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.11082597565837204, |
| "epoch": 2e-05, |
| "grad_norm": 3.043464422225952, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": -0.0232, |
| "step": 2, |
| "step_time": 21.072451215 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7133.0, |
| "completions/max_terminated_length": 7133.0, |
| "completions/mean_length": 6761.40625, |
| "completions/mean_terminated_length": 6761.40625, |
| "completions/min_length": 2474.0, |
| "completions/min_terminated_length": 2474.0, |
| "entropy": 0.10073394409846514, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.356334686279297, |
| "kl": 0.0010660493712748575, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0396, |
| "num_tokens": 473937.0, |
| "reward": -0.334066778421402, |
| "reward_std": 0.47207579016685486, |
| "rewards/rollout_reward_func/mean": -0.334066778421402, |
| "rewards/rollout_reward_func/std": 0.5433834791183472, |
| "sampling/importance_sampling_ratio/max": 1.6687252521514893, |
| "sampling/importance_sampling_ratio/mean": 0.9993299245834351, |
| "sampling/importance_sampling_ratio/min": 0.4878324866294861, |
| "sampling/sampling_logp_difference/max": 0.7177832126617432, |
| "sampling/sampling_logp_difference/mean": 0.011480587534606457, |
| "step": 3, |
| "step_time": 71.67093075799994 |
| }, |
| { |
| "clip_ratio/high_max": 0.010028428107034415, |
| "clip_ratio/high_mean": 0.0054949832847341895, |
| "clip_ratio/low_mean": 0.0004960317746736109, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0059910150594078004, |
| "entropy": 0.10106482543051243, |
| "epoch": 4e-05, |
| "grad_norm": 2.0703060626983643, |
| "kl": 0.0011230208425558885, |
| "learning_rate": 8.571428571428573e-06, |
| "loss": -0.0386, |
| "step": 4, |
| "step_time": 20.236717101000067 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7137.0, |
| "completions/max_terminated_length": 7137.0, |
| "completions/mean_length": 6591.0, |
| "completions/mean_terminated_length": 6591.0, |
| "completions/min_length": 2458.0, |
| "completions/min_terminated_length": 2458.0, |
| "entropy": 0.10171629791148007, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.9237060546875, |
| "kl": 0.003934878988729906, |
| "learning_rate": 1.1428571428571429e-05, |
| "loss": -0.0051, |
| "num_tokens": 711875.0, |
| "reward": -0.33581840991973877, |
| "reward_std": 0.6337194442749023, |
| "rewards/rollout_reward_func/mean": -0.33581840991973877, |
| "rewards/rollout_reward_func/std": 0.671731173992157, |
| "sampling/importance_sampling_ratio/max": 1.6314605474472046, |
| "sampling/importance_sampling_ratio/mean": 0.9984297752380371, |
| "sampling/importance_sampling_ratio/min": 0.14402654767036438, |
| "sampling/sampling_logp_difference/max": 1.9377576112747192, |
| "sampling/sampling_logp_difference/mean": 0.012444065883755684, |
| "step": 5, |
| "step_time": 73.43241403400066 |
| }, |
| { |
| "clip_ratio/high_max": 0.008482495148200542, |
| "clip_ratio/high_mean": 0.004241247574100271, |
| "clip_ratio/low_mean": 0.0020028685103170574, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.006244116055313498, |
| "entropy": 0.10172272333875299, |
| "epoch": 6e-05, |
| "grad_norm": 2.604616403579712, |
| "kl": 0.005101401355204871, |
| "learning_rate": 1.4285714285714285e-05, |
| "loss": -0.007, |
| "step": 6, |
| "step_time": 20.171661792999885 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7193.0, |
| "completions/max_terminated_length": 7193.0, |
| "completions/mean_length": 6867.46875, |
| "completions/mean_terminated_length": 6867.46875, |
| "completions/min_length": 3615.0, |
| "completions/min_terminated_length": 3615.0, |
| "entropy": 0.11073760967701674, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 4.576971530914307, |
| "kl": 0.004255090296283015, |
| "learning_rate": 1.7142857142857145e-05, |
| "loss": -0.0014, |
| "num_tokens": 959949.0, |
| "reward": -0.5907129049301147, |
| "reward_std": 0.5521823167800903, |
| "rewards/rollout_reward_func/mean": -0.5907129049301147, |
| "rewards/rollout_reward_func/std": 0.5898951292037964, |
| "sampling/importance_sampling_ratio/max": 2.881687641143799, |
| "sampling/importance_sampling_ratio/mean": 1.0014710426330566, |
| "sampling/importance_sampling_ratio/min": 0.5214266777038574, |
| "sampling/sampling_logp_difference/max": 1.0583760738372803, |
| "sampling/sampling_logp_difference/mean": 0.014161895960569382, |
| "step": 7, |
| "step_time": 74.81416012100021 |
| }, |
| { |
| "clip_ratio/high_max": 0.012535313493572176, |
| "clip_ratio/high_mean": 0.007229195209220052, |
| "clip_ratio/low_mean": 0.005726156698074192, |
| "clip_ratio/low_min": 0.0018418201361782849, |
| "clip_ratio/region_mean": 0.012955351907294244, |
| "entropy": 0.11031321715563536, |
| "epoch": 8e-05, |
| "grad_norm": 2.7158591747283936, |
| "kl": 0.0501496486394899, |
| "learning_rate": 2e-05, |
| "loss": -0.0054, |
| "step": 8, |
| "step_time": 20.444352195000192 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7173.0, |
| "completions/max_terminated_length": 7173.0, |
| "completions/mean_length": 6770.3125, |
| "completions/mean_terminated_length": 6770.3125, |
| "completions/min_length": 4303.0, |
| "completions/min_terminated_length": 4303.0, |
| "entropy": 0.11259338050149381, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 3.6611337661743164, |
| "kl": 0.006731060013407841, |
| "learning_rate": 2.2857142857142858e-05, |
| "loss": 0.0709, |
| "num_tokens": 1204211.0, |
| "reward": -0.43597501516342163, |
| "reward_std": 0.508872389793396, |
| "rewards/rollout_reward_func/mean": -0.43597501516342163, |
| "rewards/rollout_reward_func/std": 0.5624052286148071, |
| "sampling/importance_sampling_ratio/max": 2.5312182903289795, |
| "sampling/importance_sampling_ratio/mean": 1.0003730058670044, |
| "sampling/importance_sampling_ratio/min": 0.5103842616081238, |
| "sampling/sampling_logp_difference/max": 0.9287006855010986, |
| "sampling/sampling_logp_difference/mean": 0.013469953089952469, |
| "step": 9, |
| "step_time": 75.07683681499975 |
| }, |
| { |
| "clip_ratio/high_max": 0.009207025868818164, |
| "clip_ratio/high_mean": 0.004603512934409082, |
| "clip_ratio/low_mean": 0.00531235386733897, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.009915866743540391, |
| "entropy": 0.11247325246222317, |
| "epoch": 0.0001, |
| "grad_norm": 2.5203232765197754, |
| "kl": 0.009506899892585352, |
| "learning_rate": 2.5714285714285714e-05, |
| "loss": 0.0626, |
| "step": 10, |
| "step_time": 20.971426127000313 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7140.0, |
| "completions/max_terminated_length": 7140.0, |
| "completions/mean_length": 6344.5625, |
| "completions/mean_terminated_length": 6344.5625, |
| "completions/min_length": 1757.0, |
| "completions/min_terminated_length": 1757.0, |
| "entropy": 0.08536498458124697, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.28136944770813, |
| "kl": 0.010524504003114998, |
| "learning_rate": 2.857142857142857e-05, |
| "loss": 0.24, |
| "num_tokens": 1434809.0, |
| "reward": -0.12437915802001953, |
| "reward_std": 0.7249743938446045, |
| "rewards/rollout_reward_func/mean": -0.12437915802001953, |
| "rewards/rollout_reward_func/std": 0.7129427790641785, |
| "sampling/importance_sampling_ratio/max": 1.7484616041183472, |
| "sampling/importance_sampling_ratio/mean": 1.000082015991211, |
| "sampling/importance_sampling_ratio/min": 0.2885062098503113, |
| "sampling/sampling_logp_difference/max": 1.2430386543273926, |
| "sampling/sampling_logp_difference/mean": 0.010702264495193958, |
| "step": 11, |
| "step_time": 71.37631205699972 |
| }, |
| { |
| "clip_ratio/high_max": 0.008433553797658533, |
| "clip_ratio/high_mean": 0.004216776898829266, |
| "clip_ratio/low_mean": 0.005822986277053133, |
| "clip_ratio/low_min": 0.0009920635493472219, |
| "clip_ratio/region_mean": 0.010039763175882399, |
| "entropy": 0.08729252964258194, |
| "epoch": 0.00012, |
| "grad_norm": 1.8221752643585205, |
| "kl": 0.013876965815143194, |
| "learning_rate": 3.142857142857143e-05, |
| "loss": 0.2335, |
| "step": 12, |
| "step_time": 20.700381073000244 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7187.0, |
| "completions/max_terminated_length": 7187.0, |
| "completions/mean_length": 6312.21875, |
| "completions/mean_terminated_length": 6312.21875, |
| "completions/min_length": 1046.0, |
| "completions/min_terminated_length": 1046.0, |
| "entropy": 0.11384610936511308, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.000776767730713, |
| "kl": 0.018129478237824515, |
| "learning_rate": 3.428571428571429e-05, |
| "loss": 0.0521, |
| "num_tokens": 1665083.0, |
| "reward": -0.37061113119125366, |
| "reward_std": 0.5958520770072937, |
| "rewards/rollout_reward_func/mean": -0.37061113119125366, |
| "rewards/rollout_reward_func/std": 0.6383178234100342, |
| "sampling/importance_sampling_ratio/max": 1.6943004131317139, |
| "sampling/importance_sampling_ratio/mean": 0.9994621872901917, |
| "sampling/importance_sampling_ratio/min": 0.4000489115715027, |
| "sampling/sampling_logp_difference/max": 0.9161684513092041, |
| "sampling/sampling_logp_difference/mean": 0.013028917834162712, |
| "step": 13, |
| "step_time": 71.84389305899981 |
| }, |
| { |
| "clip_ratio/high_max": 0.005226312903687358, |
| "clip_ratio/high_mean": 0.0030866413144394755, |
| "clip_ratio/low_mean": 0.012578878930071369, |
| "clip_ratio/low_min": 0.00592114357277751, |
| "clip_ratio/region_mean": 0.015665520186303183, |
| "entropy": 0.10930443368852139, |
| "epoch": 0.00014, |
| "grad_norm": 1.6448551416397095, |
| "kl": 0.03214951721020043, |
| "learning_rate": 3.7142857142857143e-05, |
| "loss": 0.0459, |
| "step": 14, |
| "step_time": 21.154496373000256 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7195.0, |
| "completions/max_terminated_length": 7195.0, |
| "completions/mean_length": 6422.4375, |
| "completions/mean_terminated_length": 6422.4375, |
| "completions/min_length": 1669.0, |
| "completions/min_terminated_length": 1669.0, |
| "entropy": 0.09530685748904943, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.60772442817688, |
| "kl": 0.058312947396188974, |
| "learning_rate": 4e-05, |
| "loss": 0.2536, |
| "num_tokens": 1898458.0, |
| "reward": -0.19722016155719757, |
| "reward_std": 0.6420445442199707, |
| "rewards/rollout_reward_func/mean": -0.19722016155719757, |
| "rewards/rollout_reward_func/std": 0.7446529865264893, |
| "sampling/importance_sampling_ratio/max": 2.3931148052215576, |
| "sampling/importance_sampling_ratio/mean": 0.9992523193359375, |
| "sampling/importance_sampling_ratio/min": 0.3422590494155884, |
| "sampling/sampling_logp_difference/max": 1.0721874237060547, |
| "sampling/sampling_logp_difference/mean": 0.015092113055288792, |
| "step": 15, |
| "step_time": 73.59496456700026 |
| }, |
| { |
| "clip_ratio/high_max": 0.015396840986795723, |
| "clip_ratio/high_mean": 0.008179189695511013, |
| "clip_ratio/low_mean": 0.012581566552398726, |
| "clip_ratio/low_min": 0.0029352466226555407, |
| "clip_ratio/region_mean": 0.02076075618970208, |
| "entropy": 0.09704524488188326, |
| "epoch": 0.00016, |
| "grad_norm": 1.5820164680480957, |
| "kl": 0.10026089521124959, |
| "learning_rate": 4.2857142857142856e-05, |
| "loss": 0.2444, |
| "step": 16, |
| "step_time": 20.932767735000198 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7189.0, |
| "completions/max_terminated_length": 7189.0, |
| "completions/mean_length": 6465.28125, |
| "completions/mean_terminated_length": 6465.28125, |
| "completions/min_length": 469.0, |
| "completions/min_terminated_length": 469.0, |
| "entropy": 0.11429568706080317, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 3.674102306365967, |
| "kl": 0.1223891562258359, |
| "learning_rate": 4.5714285714285716e-05, |
| "loss": 0.0103, |
| "num_tokens": 2133460.0, |
| "reward": -0.16014590859413147, |
| "reward_std": 0.9211455583572388, |
| "rewards/rollout_reward_func/mean": -0.16014590859413147, |
| "rewards/rollout_reward_func/std": 0.9030523896217346, |
| "sampling/importance_sampling_ratio/max": 2.774460792541504, |
| "sampling/importance_sampling_ratio/mean": 0.9994904398918152, |
| "sampling/importance_sampling_ratio/min": 0.1321282833814621, |
| "sampling/sampling_logp_difference/max": 2.023982048034668, |
| "sampling/sampling_logp_difference/mean": 0.02061440795660019, |
| "step": 17, |
| "step_time": 72.87764749799976 |
| }, |
| { |
| "clip_ratio/high_max": 0.007797610305715352, |
| "clip_ratio/high_mean": 0.003898805152857676, |
| "clip_ratio/low_mean": 0.013190117082558572, |
| "clip_ratio/low_min": 0.0028285159496590495, |
| "clip_ratio/region_mean": 0.017088922177208588, |
| "entropy": 0.11811477039009333, |
| "epoch": 0.00018, |
| "grad_norm": 2.2869365215301514, |
| "kl": 0.1371490377932787, |
| "learning_rate": 4.8571428571428576e-05, |
| "loss": 0.0051, |
| "step": 18, |
| "step_time": 20.23855368000045 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7184.0, |
| "completions/max_terminated_length": 7184.0, |
| "completions/mean_length": 6048.65625, |
| "completions/mean_terminated_length": 6048.65625, |
| "completions/min_length": 729.0, |
| "completions/min_terminated_length": 729.0, |
| "entropy": 0.12088540918193758, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.9295918941497803, |
| "kl": 0.10166991280857474, |
| "learning_rate": 5.142857142857143e-05, |
| "loss": 0.2669, |
| "num_tokens": 2354968.0, |
| "reward": -0.27598708868026733, |
| "reward_std": 0.6756124496459961, |
| "rewards/rollout_reward_func/mean": -0.27598708868026733, |
| "rewards/rollout_reward_func/std": 0.7757007479667664, |
| "sampling/importance_sampling_ratio/max": 2.1056318283081055, |
| "sampling/importance_sampling_ratio/mean": 0.9998591542243958, |
| "sampling/importance_sampling_ratio/min": 0.12148962169885635, |
| "sampling/sampling_logp_difference/max": 2.107926368713379, |
| "sampling/sampling_logp_difference/mean": 0.02129209041595459, |
| "step": 19, |
| "step_time": 72.15802609599996 |
| }, |
| { |
| "clip_ratio/high_max": 0.016534669615793973, |
| "clip_ratio/high_mean": 0.009214304416673258, |
| "clip_ratio/low_mean": 0.010016241867560893, |
| "clip_ratio/low_min": 0.0009191176504828036, |
| "clip_ratio/region_mean": 0.019230546342441812, |
| "entropy": 0.12869162112474442, |
| "epoch": 0.0002, |
| "grad_norm": 1.8469160795211792, |
| "kl": 0.10747615271247923, |
| "learning_rate": 5.428571428571428e-05, |
| "loss": 0.2522, |
| "step": 20, |
| "step_time": 20.24046250399965 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7178.0, |
| "completions/max_terminated_length": 7178.0, |
| "completions/mean_length": 6320.125, |
| "completions/mean_terminated_length": 6320.125, |
| "completions/min_length": 1352.0, |
| "completions/min_terminated_length": 1352.0, |
| "entropy": 0.12233590893447399, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.423140048980713, |
| "kl": 0.08687845710664988, |
| "learning_rate": 5.714285714285714e-05, |
| "loss": 0.03, |
| "num_tokens": 2585024.0, |
| "reward": -0.4235602617263794, |
| "reward_std": 0.35052525997161865, |
| "rewards/rollout_reward_func/mean": -0.4235602617263794, |
| "rewards/rollout_reward_func/std": 0.36363738775253296, |
| "sampling/importance_sampling_ratio/max": 1.8418291807174683, |
| "sampling/importance_sampling_ratio/mean": 1.0002148151397705, |
| "sampling/importance_sampling_ratio/min": 0.37045231461524963, |
| "sampling/sampling_logp_difference/max": 0.9930305480957031, |
| "sampling/sampling_logp_difference/mean": 0.01798679307103157, |
| "step": 21, |
| "step_time": 75.22016325100026 |
| }, |
| { |
| "clip_ratio/high_max": 0.014240824559237808, |
| "clip_ratio/high_mean": 0.00900533315143548, |
| "clip_ratio/low_mean": 0.004223970463499427, |
| "clip_ratio/low_min": 0.002718516974709928, |
| "clip_ratio/region_mean": 0.013229303614934906, |
| "entropy": 0.1283457549288869, |
| "epoch": 0.00022, |
| "grad_norm": 1.4343130588531494, |
| "kl": 0.08600935456342995, |
| "learning_rate": 6e-05, |
| "loss": 0.0166, |
| "step": 22, |
| "step_time": 20.456357018999142 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7186.0, |
| "completions/max_terminated_length": 7186.0, |
| "completions/mean_length": 6497.09375, |
| "completions/mean_terminated_length": 6497.09375, |
| "completions/min_length": 1703.0, |
| "completions/min_terminated_length": 1703.0, |
| "entropy": 0.14600737858563662, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.981393814086914, |
| "kl": 0.1366494270041585, |
| "learning_rate": 6.285714285714286e-05, |
| "loss": 0.1462, |
| "num_tokens": 2820351.0, |
| "reward": -0.16027802228927612, |
| "reward_std": 0.7403393983840942, |
| "rewards/rollout_reward_func/mean": -0.16027802228927612, |
| "rewards/rollout_reward_func/std": 0.8203603029251099, |
| "sampling/importance_sampling_ratio/max": 1.553918719291687, |
| "sampling/importance_sampling_ratio/mean": 0.9995216131210327, |
| "sampling/importance_sampling_ratio/min": 0.4670841097831726, |
| "sampling/sampling_logp_difference/max": 0.7612459063529968, |
| "sampling/sampling_logp_difference/mean": 0.02016393467783928, |
| "step": 23, |
| "step_time": 77.5452739830007 |
| }, |
| { |
| "clip_ratio/high_max": 0.01864305092021823, |
| "clip_ratio/high_mean": 0.010268495068885386, |
| "clip_ratio/low_mean": 0.005839857796672732, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.016108352894661948, |
| "entropy": 0.1439510751515627, |
| "epoch": 0.00024, |
| "grad_norm": 1.6414893865585327, |
| "kl": 0.1360741090029478, |
| "learning_rate": 6.571428571428571e-05, |
| "loss": 0.1268, |
| "step": 24, |
| "step_time": 21.017881847999433 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7192.0, |
| "completions/max_terminated_length": 7192.0, |
| "completions/mean_length": 6739.3125, |
| "completions/mean_terminated_length": 6739.3125, |
| "completions/min_length": 1137.0, |
| "completions/min_terminated_length": 1137.0, |
| "entropy": 0.13592258002609015, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7049397230148315, |
| "kl": 0.09707080130465329, |
| "learning_rate": 6.857142857142858e-05, |
| "loss": 0.0225, |
| "num_tokens": 3062812.0, |
| "reward": -0.2555980086326599, |
| "reward_std": 0.46265262365341187, |
| "rewards/rollout_reward_func/mean": -0.2555980086326599, |
| "rewards/rollout_reward_func/std": 0.5201680660247803, |
| "sampling/importance_sampling_ratio/max": 1.5951086282730103, |
| "sampling/importance_sampling_ratio/mean": 1.0003236532211304, |
| "sampling/importance_sampling_ratio/min": 0.4008389711380005, |
| "sampling/sampling_logp_difference/max": 0.9141955375671387, |
| "sampling/sampling_logp_difference/mean": 0.015921704471111298, |
| "step": 25, |
| "step_time": 74.81312160099947 |
| }, |
| { |
| "clip_ratio/high_max": 0.010763560771010816, |
| "clip_ratio/high_mean": 0.0063089781906455755, |
| "clip_ratio/low_mean": 0.00724734173854813, |
| "clip_ratio/low_min": 0.000961538462433964, |
| "clip_ratio/region_mean": 0.013556319929193705, |
| "entropy": 0.13424453372135758, |
| "epoch": 0.00026, |
| "grad_norm": 1.216122031211853, |
| "kl": 0.10550768894609064, |
| "learning_rate": 7.142857142857143e-05, |
| "loss": 0.0038, |
| "step": 26, |
| "step_time": 21.94968546100108 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7189.0, |
| "completions/max_terminated_length": 7189.0, |
| "completions/mean_length": 6695.4375, |
| "completions/mean_terminated_length": 6695.4375, |
| "completions/min_length": 348.0, |
| "completions/min_terminated_length": 348.0, |
| "entropy": 0.15573087381199002, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.8572553396224976, |
| "kl": 0.14214458037167788, |
| "learning_rate": 7.428571428571429e-05, |
| "loss": -0.0045, |
| "num_tokens": 3303798.0, |
| "reward": -0.17485392093658447, |
| "reward_std": 0.7181835174560547, |
| "rewards/rollout_reward_func/mean": -0.17485392093658447, |
| "rewards/rollout_reward_func/std": 0.7982522249221802, |
| "sampling/importance_sampling_ratio/max": 2.1118197441101074, |
| "sampling/importance_sampling_ratio/mean": 0.9986248016357422, |
| "sampling/importance_sampling_ratio/min": 0.4841018319129944, |
| "sampling/sampling_logp_difference/max": 0.7475500106811523, |
| "sampling/sampling_logp_difference/mean": 0.016640285030007362, |
| "step": 27, |
| "step_time": 76.9669843910001 |
| }, |
| { |
| "clip_ratio/high_max": 0.020623119547963142, |
| "clip_ratio/high_mean": 0.010771118570119143, |
| "clip_ratio/low_mean": 0.008652619959320873, |
| "clip_ratio/low_min": 0.0010416667209938169, |
| "clip_ratio/region_mean": 0.019423738471232355, |
| "entropy": 0.1612294539809227, |
| "epoch": 0.00028, |
| "grad_norm": 1.346195936203003, |
| "kl": 0.16462914063595235, |
| "learning_rate": 7.714285714285715e-05, |
| "loss": -0.0238, |
| "step": 28, |
| "step_time": 21.173690784000428 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7199.0, |
| "completions/max_terminated_length": 7199.0, |
| "completions/mean_length": 6783.84375, |
| "completions/mean_terminated_length": 6783.84375, |
| "completions/min_length": 1525.0, |
| "completions/min_terminated_length": 1525.0, |
| "entropy": 0.1497128251940012, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.307813882827759, |
| "kl": 0.15235059848055243, |
| "learning_rate": 8e-05, |
| "loss": 0.2215, |
| "num_tokens": 3547486.0, |
| "reward": -0.2910870611667633, |
| "reward_std": 0.45635536313056946, |
| "rewards/rollout_reward_func/mean": -0.2910870611667633, |
| "rewards/rollout_reward_func/std": 0.4955015480518341, |
| "sampling/importance_sampling_ratio/max": 2.331606864929199, |
| "sampling/importance_sampling_ratio/mean": 0.9988598227500916, |
| "sampling/importance_sampling_ratio/min": 0.3406290113925934, |
| "sampling/sampling_logp_difference/max": 1.0769612789154053, |
| "sampling/sampling_logp_difference/mean": 0.018017400056123734, |
| "step": 29, |
| "step_time": 76.03286529600018 |
| }, |
| { |
| "clip_ratio/high_max": 0.020109005854465067, |
| "clip_ratio/high_mean": 0.011001472536008805, |
| "clip_ratio/low_mean": 0.011510160518810153, |
| "clip_ratio/low_min": 0.004589171556290239, |
| "clip_ratio/region_mean": 0.02251163314213045, |
| "entropy": 0.15249854000285268, |
| "epoch": 0.0003, |
| "grad_norm": 1.2684451341629028, |
| "kl": 0.14053832553327084, |
| "learning_rate": 8.285714285714287e-05, |
| "loss": 0.2104, |
| "step": 30, |
| "step_time": 20.506643704000908 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7174.0, |
| "completions/max_terminated_length": 7174.0, |
| "completions/mean_length": 6390.03125, |
| "completions/mean_terminated_length": 6390.03125, |
| "completions/min_length": 505.0, |
| "completions/min_terminated_length": 505.0, |
| "entropy": 0.16762764751911163, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.429037094116211, |
| "kl": 0.17197725968435407, |
| "learning_rate": 8.571428571428571e-05, |
| "loss": 0.1598, |
| "num_tokens": 3778917.0, |
| "reward": -0.5515357255935669, |
| "reward_std": 0.5762243270874023, |
| "rewards/rollout_reward_func/mean": -0.5515357255935669, |
| "rewards/rollout_reward_func/std": 0.5761792063713074, |
| "sampling/importance_sampling_ratio/max": 1.4742530584335327, |
| "sampling/importance_sampling_ratio/mean": 0.9995973110198975, |
| "sampling/importance_sampling_ratio/min": 0.5588556528091431, |
| "sampling/sampling_logp_difference/max": 0.5818639993667603, |
| "sampling/sampling_logp_difference/mean": 0.01619536057114601, |
| "step": 31, |
| "step_time": 73.20797732400115 |
| }, |
| { |
| "clip_ratio/high_max": 0.029806217935401946, |
| "clip_ratio/high_mean": 0.016797483171103522, |
| "clip_ratio/low_mean": 0.011243915010709316, |
| "clip_ratio/low_min": 0.0018382353009656072, |
| "clip_ratio/region_mean": 0.028041398210916668, |
| "entropy": 0.1692466689273715, |
| "epoch": 0.00032, |
| "grad_norm": 1.6384068727493286, |
| "kl": 0.17134067474398762, |
| "learning_rate": 8.857142857142857e-05, |
| "loss": 0.1416, |
| "step": 32, |
| "step_time": 20.21315849200164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7197.0, |
| "completions/max_terminated_length": 7197.0, |
| "completions/mean_length": 6609.71875, |
| "completions/mean_terminated_length": 6609.71875, |
| "completions/min_length": 2501.0, |
| "completions/min_terminated_length": 2501.0, |
| "entropy": 0.15834833681583405, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.319058656692505, |
| "kl": 0.1458935912232846, |
| "learning_rate": 9.142857142857143e-05, |
| "loss": 0.1679, |
| "num_tokens": 4016883.0, |
| "reward": -0.26157355308532715, |
| "reward_std": 1.0493861436843872, |
| "rewards/rollout_reward_func/mean": -0.26157355308532715, |
| "rewards/rollout_reward_func/std": 1.119583249092102, |
| "sampling/importance_sampling_ratio/max": 1.5931501388549805, |
| "sampling/importance_sampling_ratio/mean": 0.9999998807907104, |
| "sampling/importance_sampling_ratio/min": 0.4589182734489441, |
| "sampling/sampling_logp_difference/max": 0.7788830995559692, |
| "sampling/sampling_logp_difference/mean": 0.014648417942225933, |
| "step": 33, |
| "step_time": 73.15154370599839 |
| }, |
| { |
| "clip_ratio/high_max": 0.02234185009729117, |
| "clip_ratio/high_mean": 0.01517453434644267, |
| "clip_ratio/low_mean": 0.013266197114717215, |
| "clip_ratio/low_min": 0.0029219260904937983, |
| "clip_ratio/region_mean": 0.028440731461159885, |
| "entropy": 0.1540502686984837, |
| "epoch": 0.00034, |
| "grad_norm": 1.6532738208770752, |
| "kl": 0.17376246908679605, |
| "learning_rate": 9.428571428571429e-05, |
| "loss": 0.1516, |
| "step": 34, |
| "step_time": 20.732925201999024 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7191.0, |
| "completions/max_terminated_length": 7191.0, |
| "completions/mean_length": 6296.96875, |
| "completions/mean_terminated_length": 6296.96875, |
| "completions/min_length": 1298.0, |
| "completions/min_terminated_length": 1298.0, |
| "entropy": 0.15177559992298484, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.398000478744507, |
| "kl": 0.148225731914863, |
| "learning_rate": 9.714285714285715e-05, |
| "loss": -0.0734, |
| "num_tokens": 4245524.0, |
| "reward": -0.310538113117218, |
| "reward_std": 0.5178384780883789, |
| "rewards/rollout_reward_func/mean": -0.310538113117218, |
| "rewards/rollout_reward_func/std": 0.5584234595298767, |
| "sampling/importance_sampling_ratio/max": 1.9095696210861206, |
| "sampling/importance_sampling_ratio/mean": 1.0012774467468262, |
| "sampling/importance_sampling_ratio/min": 0.49921339750289917, |
| "sampling/sampling_logp_difference/max": 0.6947216987609863, |
| "sampling/sampling_logp_difference/mean": 0.016120275482535362, |
| "step": 35, |
| "step_time": 74.6011399819995 |
| }, |
| { |
| "clip_ratio/high_max": 0.02324301766930148, |
| "clip_ratio/high_mean": 0.01162150883465074, |
| "clip_ratio/low_mean": 0.018352847517235205, |
| "clip_ratio/low_min": 0.0029549773316830397, |
| "clip_ratio/region_mean": 0.029974356439197436, |
| "entropy": 0.14728072145953774, |
| "epoch": 0.00036, |
| "grad_norm": 1.5255354642868042, |
| "kl": 0.20994531828910112, |
| "learning_rate": 0.0001, |
| "loss": -0.0878, |
| "step": 36, |
| "step_time": 20.82440556399979 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7183.0, |
| "completions/max_terminated_length": 7183.0, |
| "completions/mean_length": 6492.21875, |
| "completions/mean_terminated_length": 6492.21875, |
| "completions/min_length": 2157.0, |
| "completions/min_terminated_length": 2157.0, |
| "entropy": 0.13423061231151223, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.7005836963653564, |
| "kl": 0.13645986001938581, |
| "learning_rate": 9.9999999995372e-05, |
| "loss": 0.1703, |
| "num_tokens": 4480206.0, |
| "reward": -0.26134949922561646, |
| "reward_std": 0.6028911471366882, |
| "rewards/rollout_reward_func/mean": -0.26134949922561646, |
| "rewards/rollout_reward_func/std": 0.6724324822425842, |
| "sampling/importance_sampling_ratio/max": 1.5934667587280273, |
| "sampling/importance_sampling_ratio/mean": 1.0009084939956665, |
| "sampling/importance_sampling_ratio/min": 0.5504827499389648, |
| "sampling/sampling_logp_difference/max": 0.5969595909118652, |
| "sampling/sampling_logp_difference/mean": 0.014366455376148224, |
| "step": 37, |
| "step_time": 71.96047617400063 |
| }, |
| { |
| "clip_ratio/high_max": 0.024150839541107416, |
| "clip_ratio/high_mean": 0.014068285818211734, |
| "clip_ratio/low_mean": 0.014680934487842023, |
| "clip_ratio/low_min": 0.0028285159496590495, |
| "clip_ratio/region_mean": 0.028749220189638436, |
| "entropy": 0.13499235128983855, |
| "epoch": 0.00038, |
| "grad_norm": 1.614046573638916, |
| "kl": 0.13809796143323183, |
| "learning_rate": 9.999999998148801e-05, |
| "loss": 0.1472, |
| "step": 38, |
| "step_time": 21.394122580000385 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7197.0, |
| "completions/max_terminated_length": 7197.0, |
| "completions/mean_length": 6897.28125, |
| "completions/mean_terminated_length": 6897.28125, |
| "completions/min_length": 3989.0, |
| "completions/min_terminated_length": 3989.0, |
| "entropy": 0.14411908714100718, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.0081279277801514, |
| "kl": 0.14255051454529166, |
| "learning_rate": 9.999999995834803e-05, |
| "loss": 0.033, |
| "num_tokens": 4728466.0, |
| "reward": -0.7427432537078857, |
| "reward_std": 1.1238763332366943, |
| "rewards/rollout_reward_func/mean": -0.7427432537078857, |
| "rewards/rollout_reward_func/std": 1.6972980499267578, |
| "sampling/importance_sampling_ratio/max": 2.691387414932251, |
| "sampling/importance_sampling_ratio/mean": 1.0012941360473633, |
| "sampling/importance_sampling_ratio/min": 0.5389984846115112, |
| "sampling/sampling_logp_difference/max": 0.9900567531585693, |
| "sampling/sampling_logp_difference/mean": 0.015304097905755043, |
| "step": 39, |
| "step_time": 76.1961692240011 |
| }, |
| { |
| "clip_ratio/high_max": 0.028671345789916813, |
| "clip_ratio/high_mean": 0.016989116091281176, |
| "clip_ratio/low_mean": 0.010039376968052238, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.027028492942918092, |
| "entropy": 0.1379445530474186, |
| "epoch": 0.0004, |
| "grad_norm": 1.33473539352417, |
| "kl": 0.148153408896178, |
| "learning_rate": 9.999999992595206e-05, |
| "loss": 0.014, |
| "step": 40, |
| "step_time": 21.02001015499991 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7191.0, |
| "completions/max_terminated_length": 7191.0, |
| "completions/mean_length": 6105.4375, |
| "completions/mean_terminated_length": 6105.4375, |
| "completions/min_length": 348.0, |
| "completions/min_terminated_length": 348.0, |
| "entropy": 0.1185432756319642, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.8736112117767334, |
| "kl": 0.13703973684459925, |
| "learning_rate": 9.999999988430009e-05, |
| "loss": 0.0395, |
| "num_tokens": 4950737.0, |
| "reward": 0.1191471517086029, |
| "reward_std": 0.6051614880561829, |
| "rewards/rollout_reward_func/mean": 0.1191471517086029, |
| "rewards/rollout_reward_func/std": 0.7786230444908142, |
| "sampling/importance_sampling_ratio/max": 1.5303796529769897, |
| "sampling/importance_sampling_ratio/mean": 0.9996438026428223, |
| "sampling/importance_sampling_ratio/min": 0.2842811048030853, |
| "sampling/sampling_logp_difference/max": 1.2577917575836182, |
| "sampling/sampling_logp_difference/mean": 0.011852031573653221, |
| "step": 41, |
| "step_time": 69.69323256299958 |
| }, |
| { |
| "clip_ratio/high_max": 0.020609080092981458, |
| "clip_ratio/high_mean": 0.011732496321201324, |
| "clip_ratio/low_mean": 0.018367109150858596, |
| "clip_ratio/low_min": 0.008527286583557725, |
| "clip_ratio/region_mean": 0.030099605297436938, |
| "entropy": 0.10515932412818074, |
| "epoch": 0.00042, |
| "grad_norm": 1.2367208003997803, |
| "kl": 0.1550464630126953, |
| "learning_rate": 9.999999983339212e-05, |
| "loss": 0.021, |
| "step": 42, |
| "step_time": 20.80896859000086 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7190.0, |
| "completions/max_terminated_length": 7190.0, |
| "completions/mean_length": 6363.03125, |
| "completions/mean_terminated_length": 6363.03125, |
| "completions/min_length": 485.0, |
| "completions/min_terminated_length": 485.0, |
| "entropy": 0.12465643603354692, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.0174715518951416, |
| "kl": 0.21286533144302666, |
| "learning_rate": 9.999999977322817e-05, |
| "loss": 0.2556, |
| "num_tokens": 5181291.0, |
| "reward": -0.09569768607616425, |
| "reward_std": 0.7106317281723022, |
| "rewards/rollout_reward_func/mean": -0.09569768607616425, |
| "rewards/rollout_reward_func/std": 0.7111215591430664, |
| "sampling/importance_sampling_ratio/max": 1.7430000305175781, |
| "sampling/importance_sampling_ratio/mean": 0.9981288313865662, |
| "sampling/importance_sampling_ratio/min": 0.26334136724472046, |
| "sampling/sampling_logp_difference/max": 1.3343040943145752, |
| "sampling/sampling_logp_difference/mean": 0.01676650159060955, |
| "step": 43, |
| "step_time": 71.26101785300034 |
| }, |
| { |
| "clip_ratio/high_max": 0.01569115009624511, |
| "clip_ratio/high_mean": 0.008822137664537877, |
| "clip_ratio/low_mean": 0.0272232957649976, |
| "clip_ratio/low_min": 0.008712121518328786, |
| "clip_ratio/region_mean": 0.03604543337132782, |
| "entropy": 0.12509216810576618, |
| "epoch": 0.00044, |
| "grad_norm": 1.9798716306686401, |
| "kl": 0.32974541489966214, |
| "learning_rate": 9.999999970380822e-05, |
| "loss": 0.2569, |
| "step": 44, |
| "step_time": 20.445396449999862 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7137.0, |
| "completions/max_terminated_length": 7137.0, |
| "completions/mean_length": 6645.0, |
| "completions/mean_terminated_length": 6645.0, |
| "completions/min_length": 2145.0, |
| "completions/min_terminated_length": 2145.0, |
| "entropy": 0.14243312273174524, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.270914077758789, |
| "kl": 0.1891773445531726, |
| "learning_rate": 9.999999962513227e-05, |
| "loss": 0.191, |
| "num_tokens": 5421251.0, |
| "reward": -0.3059474229812622, |
| "reward_std": 0.5616379976272583, |
| "rewards/rollout_reward_func/mean": -0.3059474229812622, |
| "rewards/rollout_reward_func/std": 0.6127358675003052, |
| "sampling/importance_sampling_ratio/max": 1.5816742181777954, |
| "sampling/importance_sampling_ratio/mean": 0.9985382556915283, |
| "sampling/importance_sampling_ratio/min": 0.3467845618724823, |
| "sampling/sampling_logp_difference/max": 1.059051513671875, |
| "sampling/sampling_logp_difference/mean": 0.014099782332777977, |
| "step": 45, |
| "step_time": 73.49722926599998 |
| }, |
| { |
| "clip_ratio/high_max": 0.03434479812858626, |
| "clip_ratio/high_mean": 0.019554392056306824, |
| "clip_ratio/low_mean": 0.016489103843923658, |
| "clip_ratio/low_min": 0.0020062917610630393, |
| "clip_ratio/region_mean": 0.03604349592933431, |
| "entropy": 0.14761458057910204, |
| "epoch": 0.00046, |
| "grad_norm": 1.3394122123718262, |
| "kl": 0.21296317502856255, |
| "learning_rate": 9.999999953720034e-05, |
| "loss": 0.1677, |
| "step": 46, |
| "step_time": 20.180529893000767 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7188.0, |
| "completions/max_terminated_length": 7188.0, |
| "completions/mean_length": 6372.34375, |
| "completions/mean_terminated_length": 6372.34375, |
| "completions/min_length": 1282.0, |
| "completions/min_terminated_length": 1282.0, |
| "entropy": 0.1546558472327888, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.706822156906128, |
| "kl": 0.22835597558878362, |
| "learning_rate": 9.999999944001239e-05, |
| "loss": 0.2036, |
| "num_tokens": 5652642.0, |
| "reward": 0.008082188665866852, |
| "reward_std": 0.6939152479171753, |
| "rewards/rollout_reward_func/mean": 0.008082188665866852, |
| "rewards/rollout_reward_func/std": 0.7366408109664917, |
| "sampling/importance_sampling_ratio/max": 1.879154086112976, |
| "sampling/importance_sampling_ratio/mean": 0.9997397661209106, |
| "sampling/importance_sampling_ratio/min": 0.3034858703613281, |
| "sampling/sampling_logp_difference/max": 1.192420244216919, |
| "sampling/sampling_logp_difference/mean": 0.0160796120762825, |
| "step": 47, |
| "step_time": 71.3642789200012 |
| }, |
| { |
| "clip_ratio/high_max": 0.01923437899677083, |
| "clip_ratio/high_mean": 0.009617189498385414, |
| "clip_ratio/low_mean": 0.02740418363828212, |
| "clip_ratio/low_min": 0.00842021155403927, |
| "clip_ratio/region_mean": 0.03702137336949818, |
| "entropy": 0.16089062485843897, |
| "epoch": 0.00048, |
| "grad_norm": 1.372217059135437, |
| "kl": 0.23768562963232398, |
| "learning_rate": 9.999999933356847e-05, |
| "loss": 0.1809, |
| "step": 48, |
| "step_time": 20.638286104000144 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7204.0, |
| "completions/max_terminated_length": 7204.0, |
| "completions/mean_length": 5986.96875, |
| "completions/mean_terminated_length": 5986.96875, |
| "completions/min_length": 475.0, |
| "completions/min_terminated_length": 475.0, |
| "entropy": 0.14714862499386072, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3831368684768677, |
| "kl": 0.2443592413328588, |
| "learning_rate": 9.999999921786855e-05, |
| "loss": 0.3008, |
| "num_tokens": 5870897.0, |
| "reward": -0.1703118532896042, |
| "reward_std": 0.5653867125511169, |
| "rewards/rollout_reward_func/mean": -0.1703118532896042, |
| "rewards/rollout_reward_func/std": 0.5801846981048584, |
| "sampling/importance_sampling_ratio/max": 1.7765135765075684, |
| "sampling/importance_sampling_ratio/mean": 1.000416874885559, |
| "sampling/importance_sampling_ratio/min": 0.5828776359558105, |
| "sampling/sampling_logp_difference/max": 0.5746527910232544, |
| "sampling/sampling_logp_difference/mean": 0.012065214104950428, |
| "step": 49, |
| "step_time": 70.26743744700025 |
| }, |
| { |
| "clip_ratio/high_max": 0.020529773319140077, |
| "clip_ratio/high_mean": 0.0111706837778911, |
| "clip_ratio/low_mean": 0.013500820234185085, |
| "clip_ratio/low_min": 0.0018660873756743968, |
| "clip_ratio/region_mean": 0.024671503924764693, |
| "entropy": 0.15120144048705697, |
| "epoch": 0.0005, |
| "grad_norm": 0.8662842512130737, |
| "kl": 0.23750609252601862, |
| "learning_rate": 9.999999909291264e-05, |
| "loss": 0.2818, |
| "step": 50, |
| "step_time": 20.684693471000173 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7176.0, |
| "completions/max_terminated_length": 7176.0, |
| "completions/mean_length": 6356.71875, |
| "completions/mean_terminated_length": 6356.71875, |
| "completions/min_length": 512.0, |
| "completions/min_terminated_length": 512.0, |
| "entropy": 0.2029622569680214, |
| "epoch": 0.00051, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.470454216003418, |
| "kl": 0.32010217709466815, |
| "learning_rate": 9.999999895870074e-05, |
| "loss": -0.2661, |
| "num_tokens": 6100948.0, |
| "reward": -0.25348520278930664, |
| "reward_std": 0.3336632549762726, |
| "rewards/rollout_reward_func/mean": -0.25348520278930664, |
| "rewards/rollout_reward_func/std": 0.4214349091053009, |
| "sampling/importance_sampling_ratio/max": 1.6843327283859253, |
| "sampling/importance_sampling_ratio/mean": 1.000025749206543, |
| "sampling/importance_sampling_ratio/min": 0.5226742029190063, |
| "sampling/sampling_logp_difference/max": 0.6487969160079956, |
| "sampling/sampling_logp_difference/mean": 0.01634291186928749, |
| "step": 51, |
| "step_time": 70.80520131099956 |
| }, |
| { |
| "clip_ratio/high_max": 0.024123376875650138, |
| "clip_ratio/high_mean": 0.01351902013993822, |
| "clip_ratio/low_mean": 0.024863345635822043, |
| "clip_ratio/low_min": 0.013176591077353805, |
| "clip_ratio/region_mean": 0.03838236551382579, |
| "entropy": 0.2084681000560522, |
| "epoch": 0.00052, |
| "grad_norm": 1.7166988849639893, |
| "kl": 0.32335374876856804, |
| "learning_rate": 9.999999881523285e-05, |
| "loss": -0.2854, |
| "step": 52, |
| "step_time": 21.188456840999606 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7198.0, |
| "completions/max_terminated_length": 7198.0, |
| "completions/mean_length": 6701.84375, |
| "completions/mean_terminated_length": 6701.84375, |
| "completions/min_length": 3902.0, |
| "completions/min_terminated_length": 3902.0, |
| "entropy": 0.18233888410031796, |
| "epoch": 0.00053, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.3137943744659424, |
| "kl": 0.31009208457544446, |
| "learning_rate": 9.999999866250896e-05, |
| "loss": 0.146, |
| "num_tokens": 6342384.0, |
| "reward": -0.09742432832717896, |
| "reward_std": 0.7253708839416504, |
| "rewards/rollout_reward_func/mean": -0.09742432832717896, |
| "rewards/rollout_reward_func/std": 0.791815996170044, |
| "sampling/importance_sampling_ratio/max": 1.7338067293167114, |
| "sampling/importance_sampling_ratio/mean": 1.001516580581665, |
| "sampling/importance_sampling_ratio/min": 0.7540294528007507, |
| "sampling/sampling_logp_difference/max": 0.5503194332122803, |
| "sampling/sampling_logp_difference/mean": 0.012293871492147446, |
| "step": 53, |
| "step_time": 74.30354640299993 |
| }, |
| { |
| "clip_ratio/high_max": 0.01733909751055762, |
| "clip_ratio/high_mean": 0.00866954875527881, |
| "clip_ratio/low_mean": 0.014682008099043742, |
| "clip_ratio/low_min": 0.004769332124851644, |
| "clip_ratio/region_mean": 0.023351556912530214, |
| "entropy": 0.1862204186618328, |
| "epoch": 0.00054, |
| "grad_norm": 1.5992038249969482, |
| "kl": 0.45713420305401087, |
| "learning_rate": 9.999999850052907e-05, |
| "loss": 0.13, |
| "step": 54, |
| "step_time": 20.94784750800045 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7205.0, |
| "completions/max_terminated_length": 7205.0, |
| "completions/mean_length": 6518.5625, |
| "completions/mean_terminated_length": 6518.5625, |
| "completions/min_length": 2439.0, |
| "completions/min_terminated_length": 2439.0, |
| "entropy": 0.18665636330842972, |
| "epoch": 0.00055, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.647172212600708, |
| "kl": 0.2535929731093347, |
| "learning_rate": 9.999999832929319e-05, |
| "loss": 0.1066, |
| "num_tokens": 6576834.0, |
| "reward": -0.2756434381008148, |
| "reward_std": 1.2163379192352295, |
| "rewards/rollout_reward_func/mean": -0.2756434381008148, |
| "rewards/rollout_reward_func/std": 1.33612060546875, |
| "sampling/importance_sampling_ratio/max": 1.5734235048294067, |
| "sampling/importance_sampling_ratio/mean": 0.9997575283050537, |
| "sampling/importance_sampling_ratio/min": 0.7102355360984802, |
| "sampling/sampling_logp_difference/max": 0.45325374603271484, |
| "sampling/sampling_logp_difference/mean": 0.01267676055431366, |
| "step": 55, |
| "step_time": 74.89291913599936 |
| }, |
| { |
| "clip_ratio/high_max": 0.022371860803104937, |
| "clip_ratio/high_mean": 0.013946017599664629, |
| "clip_ratio/low_mean": 0.013265578163554892, |
| "clip_ratio/low_min": 0.0047451850259676576, |
| "clip_ratio/region_mean": 0.027211595443077385, |
| "entropy": 0.19314171466976404, |
| "epoch": 0.00056, |
| "grad_norm": 0.9973457455635071, |
| "kl": 0.2640712922438979, |
| "learning_rate": 9.999999814880133e-05, |
| "loss": 0.0836, |
| "step": 56, |
| "step_time": 20.20899101500072 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7190.0, |
| "completions/max_terminated_length": 7190.0, |
| "completions/mean_length": 6503.0, |
| "completions/mean_terminated_length": 6503.0, |
| "completions/min_length": 493.0, |
| "completions/min_terminated_length": 493.0, |
| "entropy": 0.18440700694918633, |
| "epoch": 0.00057, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3433215618133545, |
| "kl": 0.1886869000736624, |
| "learning_rate": 9.999999795905346e-05, |
| "loss": 0.1253, |
| "num_tokens": 6811123.0, |
| "reward": 0.09097956866025925, |
| "reward_std": 0.6929575800895691, |
| "rewards/rollout_reward_func/mean": 0.09097956866025925, |
| "rewards/rollout_reward_func/std": 0.7096734046936035, |
| "sampling/importance_sampling_ratio/max": 1.3313918113708496, |
| "sampling/importance_sampling_ratio/mean": 0.9979821443557739, |
| "sampling/importance_sampling_ratio/min": 0.6041592359542847, |
| "sampling/sampling_logp_difference/max": 0.5039174556732178, |
| "sampling/sampling_logp_difference/mean": 0.01128523238003254, |
| "step": 57, |
| "step_time": 71.78335577100097 |
| }, |
| { |
| "clip_ratio/high_max": 0.020602153497748077, |
| "clip_ratio/high_mean": 0.012533219472970814, |
| "clip_ratio/low_mean": 0.017767465789802372, |
| "clip_ratio/low_min": 0.002756410278379917, |
| "clip_ratio/region_mean": 0.030300685088150203, |
| "entropy": 0.18255082983523607, |
| "epoch": 0.00058, |
| "grad_norm": 0.8210721611976624, |
| "kl": 0.19338385853916407, |
| "learning_rate": 9.999999776004962e-05, |
| "loss": 0.1068, |
| "step": 58, |
| "step_time": 20.446120923000308 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 7193.0, |
| "completions/max_terminated_length": 7193.0, |
| "completions/mean_length": 6131.625, |
| "completions/mean_terminated_length": 6131.625, |
| "completions/min_length": 245.0, |
| "completions/min_terminated_length": 245.0, |
| "entropy": 0.1735478788614273, |
| "epoch": 0.00059, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6428956985473633, |
| "kl": 0.2708520796149969, |
| "learning_rate": 9.999999755178976e-05, |
| "loss": 0.0039, |
| "num_tokens": 7033501.0, |
| "reward": 0.004367034882307053, |
| "reward_std": 0.8082680702209473, |
| "rewards/rollout_reward_func/mean": 0.004367034882307053, |
| "rewards/rollout_reward_func/std": 0.8295721411705017, |
| "sampling/importance_sampling_ratio/max": 2.616551637649536, |
| "sampling/importance_sampling_ratio/mean": 1.0010383129119873, |
| "sampling/importance_sampling_ratio/min": 0.8113043308258057, |
| "sampling/sampling_logp_difference/max": 0.9618573188781738, |
| "sampling/sampling_logp_difference/mean": 0.011064756661653519, |
| "step": 59, |
| "step_time": 70.97496701200043 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 200000, |
| "num_input_tokens_seen": 7033501, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
