{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.00059, "eval_steps": 500, "global_step": 59, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7186.0, "completions/max_terminated_length": 7186.0, "completions/mean_length": 6321.78125, "completions/mean_terminated_length": 6321.78125, "completions/min_length": 1536.0, "completions/min_terminated_length": 1536.0, "entropy": 0.11082597565837204, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 3.0425922870635986, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0232, "num_tokens": 229639.0, "reward": -0.27844929695129395, "reward_std": 0.8033838272094727, "rewards/rollout_reward_func/mean": -0.27844929695129395, "rewards/rollout_reward_func/std": 0.9743078351020813, "sampling/importance_sampling_ratio/max": 1.6097382307052612, "sampling/importance_sampling_ratio/mean": 0.999904990196228, "sampling/importance_sampling_ratio/min": 0.6331427693367004, "sampling/sampling_logp_difference/max": 0.4760715961456299, "sampling/sampling_logp_difference/mean": 0.01278754137456417, "step": 1, "step_time": 72.43192732900013 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.11082597565837204, "epoch": 2e-05, "grad_norm": 3.043464422225952, "kl": 0.0, "learning_rate": 2.8571428571428573e-06, "loss": -0.0232, "step": 2, "step_time": 21.072451215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7133.0, "completions/max_terminated_length": 7133.0, "completions/mean_length": 6761.40625, "completions/mean_terminated_length": 6761.40625, "completions/min_length": 2474.0, "completions/min_terminated_length": 2474.0, "entropy": 0.10073394409846514, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.356334686279297, "kl": 0.0010660493712748575, "learning_rate": 5.7142857142857145e-06, "loss": -0.0396, "num_tokens": 473937.0, "reward": -0.334066778421402, "reward_std": 0.47207579016685486, "rewards/rollout_reward_func/mean": -0.334066778421402, "rewards/rollout_reward_func/std": 0.5433834791183472, "sampling/importance_sampling_ratio/max": 1.6687252521514893, "sampling/importance_sampling_ratio/mean": 0.9993299245834351, "sampling/importance_sampling_ratio/min": 0.4878324866294861, "sampling/sampling_logp_difference/max": 0.7177832126617432, "sampling/sampling_logp_difference/mean": 0.011480587534606457, "step": 3, "step_time": 71.67093075799994 }, { "clip_ratio/high_max": 0.010028428107034415, "clip_ratio/high_mean": 0.0054949832847341895, "clip_ratio/low_mean": 0.0004960317746736109, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0059910150594078004, "entropy": 0.10106482543051243, "epoch": 4e-05, "grad_norm": 2.0703060626983643, "kl": 0.0011230208425558885, "learning_rate": 8.571428571428573e-06, "loss": -0.0386, "step": 4, "step_time": 20.236717101000067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7137.0, "completions/max_terminated_length": 7137.0, "completions/mean_length": 6591.0, "completions/mean_terminated_length": 6591.0, "completions/min_length": 2458.0, "completions/min_terminated_length": 2458.0, "entropy": 0.10171629791148007, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.9237060546875, "kl": 0.003934878988729906, "learning_rate": 1.1428571428571429e-05, "loss": -0.0051, "num_tokens": 711875.0, "reward": -0.33581840991973877, "reward_std": 0.6337194442749023, "rewards/rollout_reward_func/mean": -0.33581840991973877, "rewards/rollout_reward_func/std": 0.671731173992157, "sampling/importance_sampling_ratio/max": 1.6314605474472046, "sampling/importance_sampling_ratio/mean": 0.9984297752380371, "sampling/importance_sampling_ratio/min": 0.14402654767036438, "sampling/sampling_logp_difference/max": 1.9377576112747192, "sampling/sampling_logp_difference/mean": 0.012444065883755684, "step": 5, "step_time": 73.43241403400066 }, { "clip_ratio/high_max": 0.008482495148200542, "clip_ratio/high_mean": 0.004241247574100271, "clip_ratio/low_mean": 0.0020028685103170574, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.006244116055313498, "entropy": 0.10172272333875299, "epoch": 6e-05, "grad_norm": 2.604616403579712, "kl": 0.005101401355204871, "learning_rate": 1.4285714285714285e-05, "loss": -0.007, "step": 6, "step_time": 20.171661792999885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7193.0, "completions/max_terminated_length": 7193.0, "completions/mean_length": 6867.46875, "completions/mean_terminated_length": 6867.46875, "completions/min_length": 3615.0, "completions/min_terminated_length": 3615.0, "entropy": 0.11073760967701674, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 4.576971530914307, "kl": 0.004255090296283015, "learning_rate": 1.7142857142857145e-05, "loss": -0.0014, "num_tokens": 959949.0, "reward": -0.5907129049301147, "reward_std": 0.5521823167800903, "rewards/rollout_reward_func/mean": -0.5907129049301147, "rewards/rollout_reward_func/std": 0.5898951292037964, "sampling/importance_sampling_ratio/max": 2.881687641143799, "sampling/importance_sampling_ratio/mean": 1.0014710426330566, "sampling/importance_sampling_ratio/min": 0.5214266777038574, "sampling/sampling_logp_difference/max": 1.0583760738372803, "sampling/sampling_logp_difference/mean": 0.014161895960569382, "step": 7, "step_time": 74.81416012100021 }, { "clip_ratio/high_max": 0.012535313493572176, "clip_ratio/high_mean": 0.007229195209220052, "clip_ratio/low_mean": 0.005726156698074192, "clip_ratio/low_min": 0.0018418201361782849, "clip_ratio/region_mean": 0.012955351907294244, "entropy": 0.11031321715563536, "epoch": 8e-05, "grad_norm": 2.7158591747283936, "kl": 0.0501496486394899, "learning_rate": 2e-05, "loss": -0.0054, "step": 8, "step_time": 20.444352195000192 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7173.0, "completions/max_terminated_length": 7173.0, "completions/mean_length": 6770.3125, "completions/mean_terminated_length": 6770.3125, "completions/min_length": 4303.0, "completions/min_terminated_length": 4303.0, "entropy": 0.11259338050149381, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 3.6611337661743164, "kl": 0.006731060013407841, "learning_rate": 2.2857142857142858e-05, "loss": 0.0709, "num_tokens": 1204211.0, "reward": -0.43597501516342163, "reward_std": 0.508872389793396, "rewards/rollout_reward_func/mean": -0.43597501516342163, "rewards/rollout_reward_func/std": 0.5624052286148071, "sampling/importance_sampling_ratio/max": 2.5312182903289795, "sampling/importance_sampling_ratio/mean": 1.0003730058670044, "sampling/importance_sampling_ratio/min": 0.5103842616081238, "sampling/sampling_logp_difference/max": 0.9287006855010986, "sampling/sampling_logp_difference/mean": 0.013469953089952469, "step": 9, "step_time": 75.07683681499975 }, { "clip_ratio/high_max": 0.009207025868818164, "clip_ratio/high_mean": 0.004603512934409082, "clip_ratio/low_mean": 0.00531235386733897, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.009915866743540391, "entropy": 0.11247325246222317, "epoch": 0.0001, "grad_norm": 2.5203232765197754, "kl": 0.009506899892585352, "learning_rate": 2.5714285714285714e-05, "loss": 0.0626, "step": 10, "step_time": 20.971426127000313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7140.0, "completions/max_terminated_length": 7140.0, "completions/mean_length": 6344.5625, "completions/mean_terminated_length": 6344.5625, "completions/min_length": 1757.0, "completions/min_terminated_length": 1757.0, "entropy": 0.08536498458124697, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 2.28136944770813, "kl": 0.010524504003114998, "learning_rate": 2.857142857142857e-05, "loss": 0.24, "num_tokens": 1434809.0, "reward": -0.12437915802001953, "reward_std": 0.7249743938446045, "rewards/rollout_reward_func/mean": -0.12437915802001953, "rewards/rollout_reward_func/std": 0.7129427790641785, "sampling/importance_sampling_ratio/max": 1.7484616041183472, "sampling/importance_sampling_ratio/mean": 1.000082015991211, "sampling/importance_sampling_ratio/min": 0.2885062098503113, "sampling/sampling_logp_difference/max": 1.2430386543273926, "sampling/sampling_logp_difference/mean": 0.010702264495193958, "step": 11, "step_time": 71.37631205699972 }, { "clip_ratio/high_max": 0.008433553797658533, "clip_ratio/high_mean": 0.004216776898829266, "clip_ratio/low_mean": 0.005822986277053133, "clip_ratio/low_min": 0.0009920635493472219, "clip_ratio/region_mean": 0.010039763175882399, "entropy": 0.08729252964258194, "epoch": 0.00012, "grad_norm": 1.8221752643585205, "kl": 0.013876965815143194, "learning_rate": 3.142857142857143e-05, "loss": 0.2335, "step": 12, "step_time": 20.700381073000244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7187.0, "completions/max_terminated_length": 7187.0, "completions/mean_length": 6312.21875, "completions/mean_terminated_length": 6312.21875, "completions/min_length": 1046.0, "completions/min_terminated_length": 1046.0, "entropy": 0.11384610936511308, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 2.000776767730713, "kl": 0.018129478237824515, "learning_rate": 3.428571428571429e-05, "loss": 0.0521, "num_tokens": 1665083.0, "reward": -0.37061113119125366, "reward_std": 0.5958520770072937, "rewards/rollout_reward_func/mean": -0.37061113119125366, "rewards/rollout_reward_func/std": 0.6383178234100342, "sampling/importance_sampling_ratio/max": 1.6943004131317139, "sampling/importance_sampling_ratio/mean": 0.9994621872901917, "sampling/importance_sampling_ratio/min": 0.4000489115715027, "sampling/sampling_logp_difference/max": 0.9161684513092041, "sampling/sampling_logp_difference/mean": 0.013028917834162712, "step": 13, "step_time": 71.84389305899981 }, { "clip_ratio/high_max": 0.005226312903687358, "clip_ratio/high_mean": 0.0030866413144394755, "clip_ratio/low_mean": 0.012578878930071369, "clip_ratio/low_min": 0.00592114357277751, "clip_ratio/region_mean": 0.015665520186303183, "entropy": 0.10930443368852139, "epoch": 0.00014, "grad_norm": 1.6448551416397095, "kl": 0.03214951721020043, "learning_rate": 3.7142857142857143e-05, "loss": 0.0459, "step": 14, "step_time": 21.154496373000256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7195.0, "completions/max_terminated_length": 7195.0, "completions/mean_length": 6422.4375, "completions/mean_terminated_length": 6422.4375, "completions/min_length": 1669.0, "completions/min_terminated_length": 1669.0, "entropy": 0.09530685748904943, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 2.60772442817688, "kl": 0.058312947396188974, "learning_rate": 4e-05, "loss": 0.2536, "num_tokens": 1898458.0, "reward": -0.19722016155719757, "reward_std": 0.6420445442199707, "rewards/rollout_reward_func/mean": -0.19722016155719757, "rewards/rollout_reward_func/std": 0.7446529865264893, "sampling/importance_sampling_ratio/max": 2.3931148052215576, "sampling/importance_sampling_ratio/mean": 0.9992523193359375, "sampling/importance_sampling_ratio/min": 0.3422590494155884, "sampling/sampling_logp_difference/max": 1.0721874237060547, "sampling/sampling_logp_difference/mean": 0.015092113055288792, "step": 15, "step_time": 73.59496456700026 }, { "clip_ratio/high_max": 0.015396840986795723, "clip_ratio/high_mean": 0.008179189695511013, "clip_ratio/low_mean": 0.012581566552398726, "clip_ratio/low_min": 0.0029352466226555407, "clip_ratio/region_mean": 0.02076075618970208, "entropy": 0.09704524488188326, "epoch": 0.00016, "grad_norm": 1.5820164680480957, "kl": 0.10026089521124959, "learning_rate": 4.2857142857142856e-05, "loss": 0.2444, "step": 16, "step_time": 20.932767735000198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7189.0, "completions/max_terminated_length": 7189.0, "completions/mean_length": 6465.28125, "completions/mean_terminated_length": 6465.28125, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "entropy": 0.11429568706080317, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 3.674102306365967, "kl": 0.1223891562258359, "learning_rate": 4.5714285714285716e-05, "loss": 0.0103, "num_tokens": 2133460.0, "reward": -0.16014590859413147, "reward_std": 0.9211455583572388, "rewards/rollout_reward_func/mean": -0.16014590859413147, "rewards/rollout_reward_func/std": 0.9030523896217346, "sampling/importance_sampling_ratio/max": 2.774460792541504, "sampling/importance_sampling_ratio/mean": 0.9994904398918152, "sampling/importance_sampling_ratio/min": 0.1321282833814621, "sampling/sampling_logp_difference/max": 2.023982048034668, "sampling/sampling_logp_difference/mean": 0.02061440795660019, "step": 17, "step_time": 72.87764749799976 }, { "clip_ratio/high_max": 0.007797610305715352, "clip_ratio/high_mean": 0.003898805152857676, "clip_ratio/low_mean": 0.013190117082558572, "clip_ratio/low_min": 0.0028285159496590495, "clip_ratio/region_mean": 0.017088922177208588, "entropy": 0.11811477039009333, "epoch": 0.00018, "grad_norm": 2.2869365215301514, "kl": 0.1371490377932787, "learning_rate": 4.8571428571428576e-05, "loss": 0.0051, "step": 18, "step_time": 20.23855368000045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7184.0, "completions/max_terminated_length": 7184.0, "completions/mean_length": 6048.65625, "completions/mean_terminated_length": 6048.65625, "completions/min_length": 729.0, "completions/min_terminated_length": 729.0, "entropy": 0.12088540918193758, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 2.9295918941497803, "kl": 0.10166991280857474, "learning_rate": 5.142857142857143e-05, "loss": 0.2669, "num_tokens": 2354968.0, "reward": -0.27598708868026733, "reward_std": 0.6756124496459961, "rewards/rollout_reward_func/mean": -0.27598708868026733, "rewards/rollout_reward_func/std": 0.7757007479667664, "sampling/importance_sampling_ratio/max": 2.1056318283081055, "sampling/importance_sampling_ratio/mean": 0.9998591542243958, "sampling/importance_sampling_ratio/min": 0.12148962169885635, "sampling/sampling_logp_difference/max": 2.107926368713379, "sampling/sampling_logp_difference/mean": 0.02129209041595459, "step": 19, "step_time": 72.15802609599996 }, { "clip_ratio/high_max": 0.016534669615793973, "clip_ratio/high_mean": 0.009214304416673258, "clip_ratio/low_mean": 0.010016241867560893, "clip_ratio/low_min": 0.0009191176504828036, "clip_ratio/region_mean": 0.019230546342441812, "entropy": 0.12869162112474442, "epoch": 0.0002, "grad_norm": 1.8469160795211792, "kl": 0.10747615271247923, "learning_rate": 5.428571428571428e-05, "loss": 0.2522, "step": 20, "step_time": 20.24046250399965 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7178.0, "completions/max_terminated_length": 7178.0, "completions/mean_length": 6320.125, "completions/mean_terminated_length": 6320.125, "completions/min_length": 1352.0, "completions/min_terminated_length": 1352.0, "entropy": 0.12233590893447399, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 2.423140048980713, "kl": 0.08687845710664988, "learning_rate": 5.714285714285714e-05, "loss": 0.03, "num_tokens": 2585024.0, "reward": -0.4235602617263794, "reward_std": 0.35052525997161865, "rewards/rollout_reward_func/mean": -0.4235602617263794, "rewards/rollout_reward_func/std": 0.36363738775253296, "sampling/importance_sampling_ratio/max": 1.8418291807174683, "sampling/importance_sampling_ratio/mean": 1.0002148151397705, "sampling/importance_sampling_ratio/min": 0.37045231461524963, "sampling/sampling_logp_difference/max": 0.9930305480957031, "sampling/sampling_logp_difference/mean": 0.01798679307103157, "step": 21, "step_time": 75.22016325100026 }, { "clip_ratio/high_max": 0.014240824559237808, "clip_ratio/high_mean": 0.00900533315143548, "clip_ratio/low_mean": 0.004223970463499427, "clip_ratio/low_min": 0.002718516974709928, "clip_ratio/region_mean": 0.013229303614934906, "entropy": 0.1283457549288869, "epoch": 0.00022, "grad_norm": 1.4343130588531494, "kl": 0.08600935456342995, "learning_rate": 6e-05, "loss": 0.0166, "step": 22, "step_time": 20.456357018999142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7186.0, "completions/max_terminated_length": 7186.0, "completions/mean_length": 6497.09375, "completions/mean_terminated_length": 6497.09375, "completions/min_length": 1703.0, "completions/min_terminated_length": 1703.0, "entropy": 0.14600737858563662, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.981393814086914, "kl": 0.1366494270041585, "learning_rate": 6.285714285714286e-05, "loss": 0.1462, "num_tokens": 2820351.0, "reward": -0.16027802228927612, "reward_std": 0.7403393983840942, "rewards/rollout_reward_func/mean": -0.16027802228927612, "rewards/rollout_reward_func/std": 0.8203603029251099, "sampling/importance_sampling_ratio/max": 1.553918719291687, "sampling/importance_sampling_ratio/mean": 0.9995216131210327, "sampling/importance_sampling_ratio/min": 0.4670841097831726, "sampling/sampling_logp_difference/max": 0.7612459063529968, "sampling/sampling_logp_difference/mean": 0.02016393467783928, "step": 23, "step_time": 77.5452739830007 }, { "clip_ratio/high_max": 0.01864305092021823, "clip_ratio/high_mean": 0.010268495068885386, "clip_ratio/low_mean": 0.005839857796672732, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.016108352894661948, "entropy": 0.1439510751515627, "epoch": 0.00024, "grad_norm": 1.6414893865585327, "kl": 0.1360741090029478, "learning_rate": 6.571428571428571e-05, "loss": 0.1268, "step": 24, "step_time": 21.017881847999433 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7192.0, "completions/max_terminated_length": 7192.0, "completions/mean_length": 6739.3125, "completions/mean_terminated_length": 6739.3125, "completions/min_length": 1137.0, "completions/min_terminated_length": 1137.0, "entropy": 0.13592258002609015, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 1.7049397230148315, "kl": 0.09707080130465329, "learning_rate": 6.857142857142858e-05, "loss": 0.0225, "num_tokens": 3062812.0, "reward": -0.2555980086326599, "reward_std": 0.46265262365341187, "rewards/rollout_reward_func/mean": -0.2555980086326599, "rewards/rollout_reward_func/std": 0.5201680660247803, "sampling/importance_sampling_ratio/max": 1.5951086282730103, "sampling/importance_sampling_ratio/mean": 1.0003236532211304, "sampling/importance_sampling_ratio/min": 0.4008389711380005, "sampling/sampling_logp_difference/max": 0.9141955375671387, "sampling/sampling_logp_difference/mean": 0.015921704471111298, "step": 25, "step_time": 74.81312160099947 }, { "clip_ratio/high_max": 0.010763560771010816, "clip_ratio/high_mean": 0.0063089781906455755, "clip_ratio/low_mean": 0.00724734173854813, "clip_ratio/low_min": 0.000961538462433964, "clip_ratio/region_mean": 0.013556319929193705, "entropy": 0.13424453372135758, "epoch": 0.00026, "grad_norm": 1.216122031211853, "kl": 0.10550768894609064, "learning_rate": 7.142857142857143e-05, "loss": 0.0038, "step": 26, "step_time": 21.94968546100108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7189.0, "completions/max_terminated_length": 7189.0, "completions/mean_length": 6695.4375, "completions/mean_terminated_length": 6695.4375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.15573087381199002, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 1.8572553396224976, "kl": 0.14214458037167788, "learning_rate": 7.428571428571429e-05, "loss": -0.0045, "num_tokens": 3303798.0, "reward": -0.17485392093658447, "reward_std": 0.7181835174560547, "rewards/rollout_reward_func/mean": -0.17485392093658447, "rewards/rollout_reward_func/std": 0.7982522249221802, "sampling/importance_sampling_ratio/max": 2.1118197441101074, "sampling/importance_sampling_ratio/mean": 0.9986248016357422, "sampling/importance_sampling_ratio/min": 0.4841018319129944, "sampling/sampling_logp_difference/max": 0.7475500106811523, "sampling/sampling_logp_difference/mean": 0.016640285030007362, "step": 27, "step_time": 76.9669843910001 }, { "clip_ratio/high_max": 0.020623119547963142, "clip_ratio/high_mean": 0.010771118570119143, "clip_ratio/low_mean": 0.008652619959320873, "clip_ratio/low_min": 0.0010416667209938169, "clip_ratio/region_mean": 0.019423738471232355, "entropy": 0.1612294539809227, "epoch": 0.00028, "grad_norm": 1.346195936203003, "kl": 0.16462914063595235, "learning_rate": 7.714285714285715e-05, "loss": -0.0238, "step": 28, "step_time": 21.173690784000428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7199.0, "completions/max_terminated_length": 7199.0, "completions/mean_length": 6783.84375, "completions/mean_terminated_length": 6783.84375, "completions/min_length": 1525.0, "completions/min_terminated_length": 1525.0, "entropy": 0.1497128251940012, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 2.307813882827759, "kl": 0.15235059848055243, "learning_rate": 8e-05, "loss": 0.2215, "num_tokens": 3547486.0, "reward": -0.2910870611667633, "reward_std": 0.45635536313056946, "rewards/rollout_reward_func/mean": -0.2910870611667633, "rewards/rollout_reward_func/std": 0.4955015480518341, "sampling/importance_sampling_ratio/max": 2.331606864929199, "sampling/importance_sampling_ratio/mean": 0.9988598227500916, "sampling/importance_sampling_ratio/min": 0.3406290113925934, "sampling/sampling_logp_difference/max": 1.0769612789154053, "sampling/sampling_logp_difference/mean": 0.018017400056123734, "step": 29, "step_time": 76.03286529600018 }, { "clip_ratio/high_max": 0.020109005854465067, "clip_ratio/high_mean": 0.011001472536008805, "clip_ratio/low_mean": 0.011510160518810153, "clip_ratio/low_min": 0.004589171556290239, "clip_ratio/region_mean": 0.02251163314213045, "entropy": 0.15249854000285268, "epoch": 0.0003, "grad_norm": 1.2684451341629028, "kl": 0.14053832553327084, "learning_rate": 8.285714285714287e-05, "loss": 0.2104, "step": 30, "step_time": 20.506643704000908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7174.0, "completions/max_terminated_length": 7174.0, "completions/mean_length": 6390.03125, "completions/mean_terminated_length": 6390.03125, "completions/min_length": 505.0, "completions/min_terminated_length": 505.0, "entropy": 0.16762764751911163, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 2.429037094116211, "kl": 0.17197725968435407, "learning_rate": 8.571428571428571e-05, "loss": 0.1598, "num_tokens": 3778917.0, "reward": -0.5515357255935669, "reward_std": 0.5762243270874023, "rewards/rollout_reward_func/mean": -0.5515357255935669, "rewards/rollout_reward_func/std": 0.5761792063713074, "sampling/importance_sampling_ratio/max": 1.4742530584335327, "sampling/importance_sampling_ratio/mean": 0.9995973110198975, "sampling/importance_sampling_ratio/min": 0.5588556528091431, "sampling/sampling_logp_difference/max": 0.5818639993667603, "sampling/sampling_logp_difference/mean": 0.01619536057114601, "step": 31, "step_time": 73.20797732400115 }, { "clip_ratio/high_max": 0.029806217935401946, "clip_ratio/high_mean": 0.016797483171103522, "clip_ratio/low_mean": 0.011243915010709316, "clip_ratio/low_min": 0.0018382353009656072, "clip_ratio/region_mean": 0.028041398210916668, "entropy": 0.1692466689273715, "epoch": 0.00032, "grad_norm": 1.6384068727493286, "kl": 0.17134067474398762, "learning_rate": 8.857142857142857e-05, "loss": 0.1416, "step": 32, "step_time": 20.21315849200164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7197.0, "completions/max_terminated_length": 7197.0, "completions/mean_length": 6609.71875, "completions/mean_terminated_length": 6609.71875, "completions/min_length": 2501.0, "completions/min_terminated_length": 2501.0, "entropy": 0.15834833681583405, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 2.319058656692505, "kl": 0.1458935912232846, "learning_rate": 9.142857142857143e-05, "loss": 0.1679, "num_tokens": 4016883.0, "reward": -0.26157355308532715, "reward_std": 1.0493861436843872, "rewards/rollout_reward_func/mean": -0.26157355308532715, "rewards/rollout_reward_func/std": 1.119583249092102, "sampling/importance_sampling_ratio/max": 1.5931501388549805, "sampling/importance_sampling_ratio/mean": 0.9999998807907104, "sampling/importance_sampling_ratio/min": 0.4589182734489441, "sampling/sampling_logp_difference/max": 0.7788830995559692, "sampling/sampling_logp_difference/mean": 0.014648417942225933, "step": 33, "step_time": 73.15154370599839 }, { "clip_ratio/high_max": 0.02234185009729117, "clip_ratio/high_mean": 0.01517453434644267, "clip_ratio/low_mean": 0.013266197114717215, "clip_ratio/low_min": 0.0029219260904937983, "clip_ratio/region_mean": 0.028440731461159885, "entropy": 0.1540502686984837, "epoch": 0.00034, "grad_norm": 1.6532738208770752, "kl": 0.17376246908679605, "learning_rate": 9.428571428571429e-05, "loss": 0.1516, "step": 34, "step_time": 20.732925201999024 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7191.0, "completions/max_terminated_length": 7191.0, "completions/mean_length": 6296.96875, "completions/mean_terminated_length": 6296.96875, "completions/min_length": 1298.0, "completions/min_terminated_length": 1298.0, "entropy": 0.15177559992298484, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 2.398000478744507, "kl": 0.148225731914863, "learning_rate": 9.714285714285715e-05, "loss": -0.0734, "num_tokens": 4245524.0, "reward": -0.310538113117218, "reward_std": 0.5178384780883789, "rewards/rollout_reward_func/mean": -0.310538113117218, "rewards/rollout_reward_func/std": 0.5584234595298767, "sampling/importance_sampling_ratio/max": 1.9095696210861206, "sampling/importance_sampling_ratio/mean": 1.0012774467468262, "sampling/importance_sampling_ratio/min": 0.49921339750289917, "sampling/sampling_logp_difference/max": 0.6947216987609863, "sampling/sampling_logp_difference/mean": 0.016120275482535362, "step": 35, "step_time": 74.6011399819995 }, { "clip_ratio/high_max": 0.02324301766930148, "clip_ratio/high_mean": 0.01162150883465074, "clip_ratio/low_mean": 0.018352847517235205, "clip_ratio/low_min": 0.0029549773316830397, "clip_ratio/region_mean": 0.029974356439197436, "entropy": 0.14728072145953774, "epoch": 0.00036, "grad_norm": 1.5255354642868042, "kl": 0.20994531828910112, "learning_rate": 0.0001, "loss": -0.0878, "step": 36, "step_time": 20.82440556399979 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7183.0, "completions/max_terminated_length": 7183.0, "completions/mean_length": 6492.21875, "completions/mean_terminated_length": 6492.21875, "completions/min_length": 2157.0, "completions/min_terminated_length": 2157.0, "entropy": 0.13423061231151223, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 2.7005836963653564, "kl": 0.13645986001938581, "learning_rate": 9.9999999995372e-05, "loss": 0.1703, "num_tokens": 4480206.0, "reward": -0.26134949922561646, "reward_std": 0.6028911471366882, "rewards/rollout_reward_func/mean": -0.26134949922561646, "rewards/rollout_reward_func/std": 0.6724324822425842, "sampling/importance_sampling_ratio/max": 1.5934667587280273, "sampling/importance_sampling_ratio/mean": 1.0009084939956665, "sampling/importance_sampling_ratio/min": 0.5504827499389648, "sampling/sampling_logp_difference/max": 0.5969595909118652, "sampling/sampling_logp_difference/mean": 0.014366455376148224, "step": 37, "step_time": 71.96047617400063 }, { "clip_ratio/high_max": 0.024150839541107416, "clip_ratio/high_mean": 0.014068285818211734, "clip_ratio/low_mean": 0.014680934487842023, "clip_ratio/low_min": 0.0028285159496590495, "clip_ratio/region_mean": 0.028749220189638436, "entropy": 0.13499235128983855, "epoch": 0.00038, "grad_norm": 1.614046573638916, "kl": 0.13809796143323183, "learning_rate": 9.999999998148801e-05, "loss": 0.1472, "step": 38, "step_time": 21.394122580000385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7197.0, "completions/max_terminated_length": 7197.0, "completions/mean_length": 6897.28125, "completions/mean_terminated_length": 6897.28125, "completions/min_length": 3989.0, "completions/min_terminated_length": 3989.0, "entropy": 0.14411908714100718, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 2.0081279277801514, "kl": 0.14255051454529166, "learning_rate": 9.999999995834803e-05, "loss": 0.033, "num_tokens": 4728466.0, "reward": -0.7427432537078857, "reward_std": 1.1238763332366943, "rewards/rollout_reward_func/mean": -0.7427432537078857, "rewards/rollout_reward_func/std": 1.6972980499267578, "sampling/importance_sampling_ratio/max": 2.691387414932251, "sampling/importance_sampling_ratio/mean": 1.0012941360473633, "sampling/importance_sampling_ratio/min": 0.5389984846115112, "sampling/sampling_logp_difference/max": 0.9900567531585693, "sampling/sampling_logp_difference/mean": 0.015304097905755043, "step": 39, "step_time": 76.1961692240011 }, { "clip_ratio/high_max": 0.028671345789916813, "clip_ratio/high_mean": 0.016989116091281176, "clip_ratio/low_mean": 0.010039376968052238, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.027028492942918092, "entropy": 0.1379445530474186, "epoch": 0.0004, "grad_norm": 1.33473539352417, "kl": 0.148153408896178, "learning_rate": 9.999999992595206e-05, "loss": 0.014, "step": 40, "step_time": 21.02001015499991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7191.0, "completions/max_terminated_length": 7191.0, "completions/mean_length": 6105.4375, "completions/mean_terminated_length": 6105.4375, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "entropy": 0.1185432756319642, "epoch": 0.00041, "frac_reward_zero_std": 0.0, "grad_norm": 1.8736112117767334, "kl": 0.13703973684459925, "learning_rate": 9.999999988430009e-05, "loss": 0.0395, "num_tokens": 4950737.0, "reward": 0.1191471517086029, "reward_std": 0.6051614880561829, "rewards/rollout_reward_func/mean": 0.1191471517086029, "rewards/rollout_reward_func/std": 0.7786230444908142, "sampling/importance_sampling_ratio/max": 1.5303796529769897, "sampling/importance_sampling_ratio/mean": 0.9996438026428223, "sampling/importance_sampling_ratio/min": 0.2842811048030853, "sampling/sampling_logp_difference/max": 1.2577917575836182, "sampling/sampling_logp_difference/mean": 0.011852031573653221, "step": 41, "step_time": 69.69323256299958 }, { "clip_ratio/high_max": 0.020609080092981458, "clip_ratio/high_mean": 0.011732496321201324, "clip_ratio/low_mean": 0.018367109150858596, "clip_ratio/low_min": 0.008527286583557725, "clip_ratio/region_mean": 0.030099605297436938, "entropy": 0.10515932412818074, "epoch": 0.00042, "grad_norm": 1.2367208003997803, "kl": 0.1550464630126953, "learning_rate": 9.999999983339212e-05, "loss": 0.021, "step": 42, "step_time": 20.80896859000086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7190.0, "completions/max_terminated_length": 7190.0, "completions/mean_length": 6363.03125, "completions/mean_terminated_length": 6363.03125, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "entropy": 0.12465643603354692, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 2.0174715518951416, "kl": 0.21286533144302666, "learning_rate": 9.999999977322817e-05, "loss": 0.2556, "num_tokens": 5181291.0, "reward": -0.09569768607616425, "reward_std": 0.7106317281723022, "rewards/rollout_reward_func/mean": -0.09569768607616425, "rewards/rollout_reward_func/std": 0.7111215591430664, "sampling/importance_sampling_ratio/max": 1.7430000305175781, "sampling/importance_sampling_ratio/mean": 0.9981288313865662, "sampling/importance_sampling_ratio/min": 0.26334136724472046, "sampling/sampling_logp_difference/max": 1.3343040943145752, "sampling/sampling_logp_difference/mean": 0.01676650159060955, "step": 43, "step_time": 71.26101785300034 }, { "clip_ratio/high_max": 0.01569115009624511, "clip_ratio/high_mean": 0.008822137664537877, "clip_ratio/low_mean": 0.0272232957649976, "clip_ratio/low_min": 0.008712121518328786, "clip_ratio/region_mean": 0.03604543337132782, "entropy": 0.12509216810576618, "epoch": 0.00044, "grad_norm": 1.9798716306686401, "kl": 0.32974541489966214, "learning_rate": 9.999999970380822e-05, "loss": 0.2569, "step": 44, "step_time": 20.445396449999862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7137.0, "completions/max_terminated_length": 7137.0, "completions/mean_length": 6645.0, "completions/mean_terminated_length": 6645.0, "completions/min_length": 2145.0, "completions/min_terminated_length": 2145.0, "entropy": 0.14243312273174524, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 2.270914077758789, "kl": 0.1891773445531726, "learning_rate": 9.999999962513227e-05, "loss": 0.191, "num_tokens": 5421251.0, "reward": -0.3059474229812622, "reward_std": 0.5616379976272583, "rewards/rollout_reward_func/mean": -0.3059474229812622, "rewards/rollout_reward_func/std": 0.6127358675003052, "sampling/importance_sampling_ratio/max": 1.5816742181777954, "sampling/importance_sampling_ratio/mean": 0.9985382556915283, "sampling/importance_sampling_ratio/min": 0.3467845618724823, "sampling/sampling_logp_difference/max": 1.059051513671875, "sampling/sampling_logp_difference/mean": 0.014099782332777977, "step": 45, "step_time": 73.49722926599998 }, { "clip_ratio/high_max": 0.03434479812858626, "clip_ratio/high_mean": 0.019554392056306824, "clip_ratio/low_mean": 0.016489103843923658, "clip_ratio/low_min": 0.0020062917610630393, "clip_ratio/region_mean": 0.03604349592933431, "entropy": 0.14761458057910204, "epoch": 0.00046, "grad_norm": 1.3394122123718262, "kl": 0.21296317502856255, "learning_rate": 9.999999953720034e-05, "loss": 0.1677, "step": 46, "step_time": 20.180529893000767 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7188.0, "completions/max_terminated_length": 7188.0, "completions/mean_length": 6372.34375, "completions/mean_terminated_length": 6372.34375, "completions/min_length": 1282.0, "completions/min_terminated_length": 1282.0, "entropy": 0.1546558472327888, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 1.706822156906128, "kl": 0.22835597558878362, "learning_rate": 9.999999944001239e-05, "loss": 0.2036, "num_tokens": 5652642.0, "reward": 0.008082188665866852, "reward_std": 0.6939152479171753, "rewards/rollout_reward_func/mean": 0.008082188665866852, "rewards/rollout_reward_func/std": 0.7366408109664917, "sampling/importance_sampling_ratio/max": 1.879154086112976, "sampling/importance_sampling_ratio/mean": 0.9997397661209106, "sampling/importance_sampling_ratio/min": 0.3034858703613281, "sampling/sampling_logp_difference/max": 1.192420244216919, "sampling/sampling_logp_difference/mean": 0.0160796120762825, "step": 47, "step_time": 71.3642789200012 }, { "clip_ratio/high_max": 0.01923437899677083, "clip_ratio/high_mean": 0.009617189498385414, "clip_ratio/low_mean": 0.02740418363828212, "clip_ratio/low_min": 0.00842021155403927, "clip_ratio/region_mean": 0.03702137336949818, "entropy": 0.16089062485843897, "epoch": 0.00048, "grad_norm": 1.372217059135437, "kl": 0.23768562963232398, "learning_rate": 9.999999933356847e-05, "loss": 0.1809, "step": 48, "step_time": 20.638286104000144 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7204.0, "completions/max_terminated_length": 7204.0, "completions/mean_length": 5986.96875, "completions/mean_terminated_length": 5986.96875, "completions/min_length": 475.0, "completions/min_terminated_length": 475.0, "entropy": 0.14714862499386072, "epoch": 0.00049, "frac_reward_zero_std": 0.0, "grad_norm": 1.3831368684768677, "kl": 0.2443592413328588, "learning_rate": 9.999999921786855e-05, "loss": 0.3008, "num_tokens": 5870897.0, "reward": -0.1703118532896042, "reward_std": 0.5653867125511169, "rewards/rollout_reward_func/mean": -0.1703118532896042, "rewards/rollout_reward_func/std": 0.5801846981048584, "sampling/importance_sampling_ratio/max": 1.7765135765075684, "sampling/importance_sampling_ratio/mean": 1.000416874885559, "sampling/importance_sampling_ratio/min": 0.5828776359558105, "sampling/sampling_logp_difference/max": 0.5746527910232544, "sampling/sampling_logp_difference/mean": 0.012065214104950428, "step": 49, "step_time": 70.26743744700025 }, { "clip_ratio/high_max": 0.020529773319140077, "clip_ratio/high_mean": 0.0111706837778911, "clip_ratio/low_mean": 0.013500820234185085, "clip_ratio/low_min": 0.0018660873756743968, "clip_ratio/region_mean": 0.024671503924764693, "entropy": 0.15120144048705697, "epoch": 0.0005, "grad_norm": 0.8662842512130737, "kl": 0.23750609252601862, "learning_rate": 9.999999909291264e-05, "loss": 0.2818, "step": 50, "step_time": 20.684693471000173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7176.0, "completions/max_terminated_length": 7176.0, "completions/mean_length": 6356.71875, "completions/mean_terminated_length": 6356.71875, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "entropy": 0.2029622569680214, "epoch": 0.00051, "frac_reward_zero_std": 0.0, "grad_norm": 2.470454216003418, "kl": 0.32010217709466815, "learning_rate": 9.999999895870074e-05, "loss": -0.2661, "num_tokens": 6100948.0, "reward": -0.25348520278930664, "reward_std": 0.3336632549762726, "rewards/rollout_reward_func/mean": -0.25348520278930664, "rewards/rollout_reward_func/std": 0.4214349091053009, "sampling/importance_sampling_ratio/max": 1.6843327283859253, "sampling/importance_sampling_ratio/mean": 1.000025749206543, "sampling/importance_sampling_ratio/min": 0.5226742029190063, "sampling/sampling_logp_difference/max": 0.6487969160079956, "sampling/sampling_logp_difference/mean": 0.01634291186928749, "step": 51, "step_time": 70.80520131099956 }, { "clip_ratio/high_max": 0.024123376875650138, "clip_ratio/high_mean": 0.01351902013993822, "clip_ratio/low_mean": 0.024863345635822043, "clip_ratio/low_min": 0.013176591077353805, "clip_ratio/region_mean": 0.03838236551382579, "entropy": 0.2084681000560522, "epoch": 0.00052, "grad_norm": 1.7166988849639893, "kl": 0.32335374876856804, "learning_rate": 9.999999881523285e-05, "loss": -0.2854, "step": 52, "step_time": 21.188456840999606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7198.0, "completions/max_terminated_length": 7198.0, "completions/mean_length": 6701.84375, "completions/mean_terminated_length": 6701.84375, "completions/min_length": 3902.0, "completions/min_terminated_length": 3902.0, "entropy": 0.18233888410031796, "epoch": 0.00053, "frac_reward_zero_std": 0.0, "grad_norm": 2.3137943744659424, "kl": 0.31009208457544446, "learning_rate": 9.999999866250896e-05, "loss": 0.146, "num_tokens": 6342384.0, "reward": -0.09742432832717896, "reward_std": 0.7253708839416504, "rewards/rollout_reward_func/mean": -0.09742432832717896, "rewards/rollout_reward_func/std": 0.791815996170044, "sampling/importance_sampling_ratio/max": 1.7338067293167114, "sampling/importance_sampling_ratio/mean": 1.001516580581665, "sampling/importance_sampling_ratio/min": 0.7540294528007507, "sampling/sampling_logp_difference/max": 0.5503194332122803, "sampling/sampling_logp_difference/mean": 0.012293871492147446, "step": 53, "step_time": 74.30354640299993 }, { "clip_ratio/high_max": 0.01733909751055762, "clip_ratio/high_mean": 0.00866954875527881, "clip_ratio/low_mean": 0.014682008099043742, "clip_ratio/low_min": 0.004769332124851644, "clip_ratio/region_mean": 0.023351556912530214, "entropy": 0.1862204186618328, "epoch": 0.00054, "grad_norm": 1.5992038249969482, "kl": 0.45713420305401087, "learning_rate": 9.999999850052907e-05, "loss": 0.13, "step": 54, "step_time": 20.94784750800045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7205.0, "completions/max_terminated_length": 7205.0, "completions/mean_length": 6518.5625, "completions/mean_terminated_length": 6518.5625, "completions/min_length": 2439.0, "completions/min_terminated_length": 2439.0, "entropy": 0.18665636330842972, "epoch": 0.00055, "frac_reward_zero_std": 0.0, "grad_norm": 1.647172212600708, "kl": 0.2535929731093347, "learning_rate": 9.999999832929319e-05, "loss": 0.1066, "num_tokens": 6576834.0, "reward": -0.2756434381008148, "reward_std": 1.2163379192352295, "rewards/rollout_reward_func/mean": -0.2756434381008148, "rewards/rollout_reward_func/std": 1.33612060546875, "sampling/importance_sampling_ratio/max": 1.5734235048294067, "sampling/importance_sampling_ratio/mean": 0.9997575283050537, "sampling/importance_sampling_ratio/min": 0.7102355360984802, "sampling/sampling_logp_difference/max": 0.45325374603271484, "sampling/sampling_logp_difference/mean": 0.01267676055431366, "step": 55, "step_time": 74.89291913599936 }, { "clip_ratio/high_max": 0.022371860803104937, "clip_ratio/high_mean": 0.013946017599664629, "clip_ratio/low_mean": 0.013265578163554892, "clip_ratio/low_min": 0.0047451850259676576, "clip_ratio/region_mean": 0.027211595443077385, "entropy": 0.19314171466976404, "epoch": 0.00056, "grad_norm": 0.9973457455635071, "kl": 0.2640712922438979, "learning_rate": 9.999999814880133e-05, "loss": 0.0836, "step": 56, "step_time": 20.20899101500072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7190.0, "completions/max_terminated_length": 7190.0, "completions/mean_length": 6503.0, "completions/mean_terminated_length": 6503.0, "completions/min_length": 493.0, "completions/min_terminated_length": 493.0, "entropy": 0.18440700694918633, "epoch": 0.00057, "frac_reward_zero_std": 0.0, "grad_norm": 1.3433215618133545, "kl": 0.1886869000736624, "learning_rate": 9.999999795905346e-05, "loss": 0.1253, "num_tokens": 6811123.0, "reward": 0.09097956866025925, "reward_std": 0.6929575800895691, "rewards/rollout_reward_func/mean": 0.09097956866025925, "rewards/rollout_reward_func/std": 0.7096734046936035, "sampling/importance_sampling_ratio/max": 1.3313918113708496, "sampling/importance_sampling_ratio/mean": 0.9979821443557739, "sampling/importance_sampling_ratio/min": 0.6041592359542847, "sampling/sampling_logp_difference/max": 0.5039174556732178, "sampling/sampling_logp_difference/mean": 0.01128523238003254, "step": 57, "step_time": 71.78335577100097 }, { "clip_ratio/high_max": 0.020602153497748077, "clip_ratio/high_mean": 0.012533219472970814, "clip_ratio/low_mean": 0.017767465789802372, "clip_ratio/low_min": 0.002756410278379917, "clip_ratio/region_mean": 0.030300685088150203, "entropy": 0.18255082983523607, "epoch": 0.00058, "grad_norm": 0.8210721611976624, "kl": 0.19338385853916407, "learning_rate": 9.999999776004962e-05, "loss": 0.1068, "step": 58, "step_time": 20.446120923000308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 7193.0, "completions/max_terminated_length": 7193.0, "completions/mean_length": 6131.625, "completions/mean_terminated_length": 6131.625, "completions/min_length": 245.0, "completions/min_terminated_length": 245.0, "entropy": 0.1735478788614273, "epoch": 0.00059, "frac_reward_zero_std": 0.0, "grad_norm": 1.6428956985473633, "kl": 0.2708520796149969, "learning_rate": 9.999999755178976e-05, "loss": 0.0039, "num_tokens": 7033501.0, "reward": 0.004367034882307053, "reward_std": 0.8082680702209473, "rewards/rollout_reward_func/mean": 0.004367034882307053, "rewards/rollout_reward_func/std": 0.8295721411705017, "sampling/importance_sampling_ratio/max": 2.616551637649536, "sampling/importance_sampling_ratio/mean": 1.0010383129119873, "sampling/importance_sampling_ratio/min": 0.8113043308258057, "sampling/sampling_logp_difference/max": 0.9618573188781738, "sampling/sampling_logp_difference/mean": 0.011064756661653519, "step": 59, "step_time": 70.97496701200043 } ], "logging_steps": 1.0, "max_steps": 200000, "num_input_tokens_seen": 7033501, "num_train_epochs": 2, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }