{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.0005, "eval_steps": 500, "global_step": 50, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 14431.0, "completions/max_terminated_length": 13090.0, "completions/mean_length": 11799.78125, "completions/mean_terminated_length": 7162.27294921875, "completions/min_length": 235.0, "completions/min_terminated_length": 235.0, "entropy": 0.13893846329301596, "epoch": 1e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.7887574434280396, "kl": 0.0, "learning_rate": 0.0, "loss": -0.0596, "num_tokens": 404630.0, "reward": -0.7717239856719971, "reward_std": 0.42467162013053894, "rewards/rollout_reward_func/mean": -0.7717239856719971, "rewards/rollout_reward_func/std": 0.5858266353607178, "sampling/importance_sampling_ratio/max": 1.7099345922470093, "sampling/importance_sampling_ratio/mean": 0.9991271495819092, "sampling/importance_sampling_ratio/min": 0.4281526207923889, "sampling/sampling_logp_difference/max": 0.8482755422592163, "sampling/sampling_logp_difference/mean": 0.012976451776921749, "step": 1, "step_time": 130.9108755219977 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "entropy": 0.13893846329301596, "epoch": 2e-05, "grad_norm": 1.804929494857788, "kl": 0.0, "learning_rate": 2.8571428571428573e-06, "loss": -0.0596, "step": 2, "step_time": 44.13144713700058 }, { "clip_ratio/high_max": 0.0022670181933790445, "clip_ratio/high_mean": 0.0011335090966895223, "clip_ratio/low_mean": 0.0029946123831905425, "clip_ratio/low_min": 0.0005040322430431843, "clip_ratio/region_mean": 0.004128121479880065, "completions/clipped_ratio": 0.6875, "completions/max_length": 14439.0, "completions/max_terminated_length": 11449.0, "completions/mean_length": 11617.5, "completions/mean_terminated_length": 5924.5, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "entropy": 0.12676118151284754, "epoch": 3e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.4913136959075928, "kl": 0.0009995185955631314, "learning_rate": 5.7142857142857145e-06, "loss": 0.0137, "num_tokens": 803817.0, "reward": -0.8155950307846069, "reward_std": 0.4161786139011383, "rewards/rollout_reward_func/mean": -0.8155950307846069, "rewards/rollout_reward_func/std": 0.4885525703430176, "sampling/importance_sampling_ratio/max": 2.0640246868133545, "sampling/importance_sampling_ratio/mean": 0.9992372393608093, "sampling/importance_sampling_ratio/min": 0.41666513681411743, "sampling/sampling_logp_difference/max": 0.8754724264144897, "sampling/sampling_logp_difference/mean": 0.011403773911297321, "step": 3, "step_time": 126.63972502899833 }, { "clip_ratio/high_max": 0.005151324032340199, "clip_ratio/high_mean": 0.0025756620161700994, "clip_ratio/low_mean": 0.001626400655368343, "clip_ratio/low_min": 0.0005040322430431843, "clip_ratio/region_mean": 0.004202062671538442, "entropy": 0.12616439966950566, "epoch": 4e-05, "grad_norm": 1.4003232717514038, "kl": 0.001313611121076974, "learning_rate": 8.571428571428573e-06, "loss": 0.0134, "step": 4, "step_time": 44.62073808600053 }, { "clip_ratio/high_max": 0.0016215514624491334, "clip_ratio/high_mean": 0.0008107757312245667, "clip_ratio/low_mean": 0.003547743574017659, "clip_ratio/low_min": 0.0010995370685122907, "clip_ratio/region_mean": 0.0043585193052422255, "completions/clipped_ratio": 0.8125, "completions/max_length": 14535.0, "completions/max_terminated_length": 11037.0, "completions/mean_length": 12972.78125, "completions/mean_terminated_length": 7317.5, "completions/min_length": 4349.0, "completions/min_terminated_length": 4349.0, "entropy": 0.1461638123728335, "epoch": 5e-05, "frac_reward_zero_std": 0.0, "grad_norm": 2.107853889465332, "kl": 0.001256127497072157, "learning_rate": 1.1428571428571429e-05, "loss": 0.1588, "num_tokens": 1245837.0, "reward": -0.6968419551849365, "reward_std": 0.4676758050918579, "rewards/rollout_reward_func/mean": -0.6968419551849365, "rewards/rollout_reward_func/std": 0.5711839199066162, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 1.001021385192871, "sampling/importance_sampling_ratio/min": 0.5821155905723572, "sampling/sampling_logp_difference/max": 1.1031980514526367, "sampling/sampling_logp_difference/mean": 0.01345288660377264, "step": 5, "step_time": 131.89395336500274 }, { "clip_ratio/high_max": 0.0021937466226518154, "clip_ratio/high_mean": 0.0010968733113259077, "clip_ratio/low_mean": 0.0032513330806978047, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004348206392023712, "entropy": 0.14492363901808858, "epoch": 6e-05, "grad_norm": 1.6609207391738892, "kl": 0.001644153301640472, "learning_rate": 1.4285714285714285e-05, "loss": 0.1578, "step": 6, "step_time": 46.381975520000196 }, { "clip_ratio/high_max": 0.0031376838451251388, "clip_ratio/high_mean": 0.0015688419225625694, "clip_ratio/low_mean": 0.0033995709964074194, "clip_ratio/low_min": 0.0015243901871144772, "clip_ratio/region_mean": 0.004968412918969989, "completions/clipped_ratio": 0.5625, "completions/max_length": 14446.0, "completions/max_terminated_length": 12650.0, "completions/mean_length": 11378.28125, "completions/mean_terminated_length": 7822.64306640625, "completions/min_length": 1771.0, "completions/min_terminated_length": 1771.0, "entropy": 0.0922874582465738, "epoch": 7e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.8714547157287598, "kl": 0.0019052933557759388, "learning_rate": 1.7142857142857145e-05, "loss": 0.3544, "num_tokens": 1637608.0, "reward": -0.4466933012008667, "reward_std": 0.8536279201507568, "rewards/rollout_reward_func/mean": -0.4466933012008667, "rewards/rollout_reward_func/std": 0.8547493815422058, "sampling/importance_sampling_ratio/max": 2.0912468433380127, "sampling/importance_sampling_ratio/mean": 0.9991166591644287, "sampling/importance_sampling_ratio/min": 0.48319441080093384, "sampling/sampling_logp_difference/max": 0.7377604246139526, "sampling/sampling_logp_difference/mean": 0.01108577847480774, "step": 7, "step_time": 126.40769893900142 }, { "clip_ratio/high_max": 0.004281101922970265, "clip_ratio/high_mean": 0.002941832965007052, "clip_ratio/low_mean": 0.004934519121889025, "clip_ratio/low_min": 0.001998567197006196, "clip_ratio/region_mean": 0.007876352086896077, "entropy": 0.08806704939343035, "epoch": 8e-05, "grad_norm": 1.634592890739441, "kl": 0.003069787417189218, "learning_rate": 2e-05, "loss": 0.347, "step": 8, "step_time": 44.41796817000068 }, { "clip_ratio/high_max": 0.0016580428346060216, "clip_ratio/high_mean": 0.0008290214173030108, "clip_ratio/low_mean": 0.00299244187772274, "clip_ratio/low_min": 0.0006009615608491004, "clip_ratio/region_mean": 0.003821463295025751, "completions/clipped_ratio": 0.65625, "completions/max_length": 14526.0, "completions/max_terminated_length": 13961.0, "completions/mean_length": 11837.40625, "completions/mean_terminated_length": 7239.818359375, "completions/min_length": 967.0, "completions/min_terminated_length": 967.0, "entropy": 0.10492791910655797, "epoch": 9e-05, "frac_reward_zero_std": 0.0, "grad_norm": 1.7168275117874146, "kl": 0.004868662523222156, "learning_rate": 2.2857142857142858e-05, "loss": -0.0376, "num_tokens": 2043692.0, "reward": -0.6215145587921143, "reward_std": 0.6429688930511475, "rewards/rollout_reward_func/mean": -0.6215145587921143, "rewards/rollout_reward_func/std": 0.7194304466247559, "sampling/importance_sampling_ratio/max": 1.6615937948226929, "sampling/importance_sampling_ratio/mean": 1.0002217292785645, "sampling/importance_sampling_ratio/min": 0.47998860478401184, "sampling/sampling_logp_difference/max": 0.7339929938316345, "sampling/sampling_logp_difference/mean": 0.010960595682263374, "step": 9, "step_time": 124.44129867599804 }, { "clip_ratio/high_max": 0.005017199146095663, "clip_ratio/high_mean": 0.0025085995730478317, "clip_ratio/low_mean": 0.00814974692184478, "clip_ratio/low_min": 0.002315932302735746, "clip_ratio/region_mean": 0.010658346494892612, "entropy": 0.09843971265945584, "epoch": 0.0001, "grad_norm": 1.3552314043045044, "kl": 0.01046636993123684, "learning_rate": 2.5714285714285714e-05, "loss": -0.0446, "step": 10, "step_time": 45.56444605700199 }, { "clip_ratio/high_max": 0.0035993335768580437, "clip_ratio/high_mean": 0.0017996667884290218, "clip_ratio/low_mean": 0.003951315797166899, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.005750982585595921, "completions/clipped_ratio": 0.65625, "completions/max_length": 14459.0, "completions/max_terminated_length": 13816.0, "completions/mean_length": 11661.625, "completions/mean_terminated_length": 6824.5458984375, "completions/min_length": 339.0, "completions/min_terminated_length": 339.0, "entropy": 0.06328993663191795, "epoch": 0.00011, "frac_reward_zero_std": 0.0, "grad_norm": 1.7052122354507446, "kl": 0.020232660688634496, "learning_rate": 2.857142857142857e-05, "loss": -0.0762, "num_tokens": 2444128.0, "reward": -0.7770951390266418, "reward_std": 0.4045039117336273, "rewards/rollout_reward_func/mean": -0.7770951390266418, "rewards/rollout_reward_func/std": 0.4794606566429138, "sampling/importance_sampling_ratio/max": 2.010491371154785, "sampling/importance_sampling_ratio/mean": 0.9987582564353943, "sampling/importance_sampling_ratio/min": 0.159025177359581, "sampling/sampling_logp_difference/max": 1.8386926651000977, "sampling/sampling_logp_difference/mean": 0.008627503179013729, "step": 11, "step_time": 121.11756841299757 }, { "clip_ratio/high_max": 0.005686733347829431, "clip_ratio/high_mean": 0.0028433666739147156, "clip_ratio/low_mean": 0.012177724187495187, "clip_ratio/low_min": 0.006776289606932551, "clip_ratio/region_mean": 0.015021090890513733, "entropy": 0.06035556551069021, "epoch": 0.00012, "grad_norm": 1.6369805335998535, "kl": 0.06257348190410994, "learning_rate": 3.142857142857143e-05, "loss": -0.0747, "step": 12, "step_time": 44.37280418500268 }, { "clip_ratio/high_max": 0.006585154333151877, "clip_ratio/high_mean": 0.003569125838112086, "clip_ratio/low_mean": 0.00462483472074382, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.008193960617063567, "completions/clipped_ratio": 0.8125, "completions/max_length": 14486.0, "completions/max_terminated_length": 13435.0, "completions/mean_length": 12772.375, "completions/mean_terminated_length": 6521.1669921875, "completions/min_length": 815.0, "completions/min_terminated_length": 815.0, "entropy": 0.09321935242041945, "epoch": 0.00013, "frac_reward_zero_std": 0.0, "grad_norm": 1.849172592163086, "kl": 0.03665067162364721, "learning_rate": 3.428571428571429e-05, "loss": -0.1546, "num_tokens": 2880635.0, "reward": -0.927730917930603, "reward_std": 0.10113543272018433, "rewards/rollout_reward_func/mean": -0.927730917930603, "rewards/rollout_reward_func/std": 0.10532300174236298, "sampling/importance_sampling_ratio/max": 1.936368465423584, "sampling/importance_sampling_ratio/mean": 0.9986484050750732, "sampling/importance_sampling_ratio/min": 0.18729014694690704, "sampling/sampling_logp_difference/max": 1.6750962734222412, "sampling/sampling_logp_difference/mean": 0.01844533160328865, "step": 13, "step_time": 130.46155179900052 }, { "clip_ratio/high_max": 0.007653064152691513, "clip_ratio/high_mean": 0.0043634974281303585, "clip_ratio/low_mean": 0.014142459142021835, "clip_ratio/low_min": 0.0029499504598788917, "clip_ratio/region_mean": 0.018505956570152193, "entropy": 0.09064974845387042, "epoch": 0.00014, "grad_norm": 8.18207836151123, "kl": 0.2905037718010135, "learning_rate": 3.7142857142857143e-05, "loss": -0.1459, "step": 14, "step_time": 44.158161802002724 }, { "clip_ratio/high_max": 0.0027650215197354555, "clip_ratio/high_mean": 0.0013825107598677278, "clip_ratio/low_mean": 0.004234670544974506, "clip_ratio/low_min": 0.0012649551499634981, "clip_ratio/region_mean": 0.005617181275738403, "completions/clipped_ratio": 0.8125, "completions/max_length": 14400.0, "completions/max_terminated_length": 9012.0, "completions/mean_length": 12419.84375, "completions/mean_terminated_length": 4916.33349609375, "completions/min_length": 1128.0, "completions/min_terminated_length": 1128.0, "entropy": 0.07450041663832963, "epoch": 0.00015, "frac_reward_zero_std": 0.0, "grad_norm": 2.2030179500579834, "kl": 0.07086092534882482, "learning_rate": 4e-05, "loss": 0.1292, "num_tokens": 3305559.0, "reward": -0.7803157567977905, "reward_std": 0.42063942551612854, "rewards/rollout_reward_func/mean": -0.7803157567977905, "rewards/rollout_reward_func/std": 0.4826337397098541, "sampling/importance_sampling_ratio/max": 2.0322628021240234, "sampling/importance_sampling_ratio/mean": 0.9985103607177734, "sampling/importance_sampling_ratio/min": 0.24719201028347015, "sampling/sampling_logp_difference/max": 1.397589921951294, "sampling/sampling_logp_difference/mean": 0.015935054048895836, "step": 15, "step_time": 129.98408898099842 }, { "clip_ratio/high_max": 0.008143757819198072, "clip_ratio/high_mean": 0.004071878909599036, "clip_ratio/low_mean": 0.014440719707636163, "clip_ratio/low_min": 0.0063073321944102645, "clip_ratio/region_mean": 0.01851259867544286, "entropy": 0.07352668652310967, "epoch": 0.00016, "grad_norm": 1.1360507011413574, "kl": 0.08157501685491297, "learning_rate": 4.2857142857142856e-05, "loss": 0.1258, "step": 16, "step_time": 44.95600875300079 }, { "clip_ratio/high_max": 0.000589622650295496, "clip_ratio/high_mean": 0.000294811325147748, "clip_ratio/low_mean": 0.004054940363857895, "clip_ratio/low_min": 0.0010419560712762177, "clip_ratio/region_mean": 0.004349751689005643, "completions/clipped_ratio": 0.5625, "completions/max_length": 14400.0, "completions/max_terminated_length": 13862.0, "completions/mean_length": 11858.40625, "completions/mean_terminated_length": 8890.857421875, "completions/min_length": 2159.0, "completions/min_terminated_length": 2159.0, "entropy": 0.07483385875821114, "epoch": 0.00017, "frac_reward_zero_std": 0.0, "grad_norm": 1.7257509231567383, "kl": 0.05066429602447897, "learning_rate": 4.5714285714285716e-05, "loss": 0.334, "num_tokens": 3712600.0, "reward": -0.43021440505981445, "reward_std": 0.8870621919631958, "rewards/rollout_reward_func/mean": -0.43021440505981445, "rewards/rollout_reward_func/std": 0.8449047803878784, "sampling/importance_sampling_ratio/max": 2.4000132083892822, "sampling/importance_sampling_ratio/mean": 1.002233624458313, "sampling/importance_sampling_ratio/min": 0.1551634818315506, "sampling/sampling_logp_difference/max": 1.8632760047912598, "sampling/sampling_logp_difference/mean": 0.01753704622387886, "step": 17, "step_time": 132.86553408099644 }, { "clip_ratio/high_max": 0.006706475804094225, "clip_ratio/high_mean": 0.003931941610062495, "clip_ratio/low_mean": 0.0171583968331106, "clip_ratio/low_min": 0.010398818471003324, "clip_ratio/region_mean": 0.021090338443173096, "entropy": 0.06936636054888368, "epoch": 0.00018, "grad_norm": 1.771362543106079, "kl": 0.10733210120815784, "learning_rate": 4.8571428571428576e-05, "loss": 0.3285, "step": 18, "step_time": 44.16366357000152 }, { "clip_ratio/high_max": 0.0006443298771046102, "clip_ratio/high_mean": 0.0003221649385523051, "clip_ratio/low_mean": 0.0072666899068281054, "clip_ratio/low_min": 0.0009964113123714924, "clip_ratio/region_mean": 0.00758885481627658, "completions/clipped_ratio": 0.625, "completions/max_length": 14492.0, "completions/max_terminated_length": 10697.0, "completions/mean_length": 11052.375, "completions/mean_terminated_length": 5732.83349609375, "completions/min_length": 1322.0, "completions/min_terminated_length": 1322.0, "entropy": 0.06488334969617426, "epoch": 0.00019, "frac_reward_zero_std": 0.0, "grad_norm": 1.4038617610931396, "kl": 0.08611430588643998, "learning_rate": 5.142857142857143e-05, "loss": 0.5711, "num_tokens": 4093752.0, "reward": -0.4095195233821869, "reward_std": 0.8482590317726135, "rewards/rollout_reward_func/mean": -0.4095195233821869, "rewards/rollout_reward_func/std": 0.8331766724586487, "sampling/importance_sampling_ratio/max": 2.3997995853424072, "sampling/importance_sampling_ratio/mean": 0.9988876581192017, "sampling/importance_sampling_ratio/min": 0.19376768171787262, "sampling/sampling_logp_difference/max": 1.6410953998565674, "sampling/sampling_logp_difference/mean": 0.016171647235751152, "step": 19, "step_time": 128.7513532999983 }, { "clip_ratio/high_max": 0.009648803621530533, "clip_ratio/high_mean": 0.004824401810765266, "clip_ratio/low_mean": 0.014935461309505627, "clip_ratio/low_min": 0.006239572481717914, "clip_ratio/region_mean": 0.019759863178478554, "entropy": 0.06792194582521915, "epoch": 0.0002, "grad_norm": 1.5674505233764648, "kl": 0.07957532908767462, "learning_rate": 5.428571428571428e-05, "loss": 0.5691, "step": 20, "step_time": 44.17661613099699 }, { "clip_ratio/high_max": 0.0010775862028822303, "clip_ratio/high_mean": 0.0005387931014411151, "clip_ratio/low_mean": 0.002510265534510836, "clip_ratio/low_min": 0.0006067961221560836, "clip_ratio/region_mean": 0.003049058635951951, "completions/clipped_ratio": 0.53125, "completions/max_length": 14388.0, "completions/max_terminated_length": 13834.0, "completions/mean_length": 11314.09375, "completions/mean_terminated_length": 8011.1337890625, "completions/min_length": 1359.0, "completions/min_terminated_length": 1359.0, "entropy": 0.06659986882004887, "epoch": 0.00021, "frac_reward_zero_std": 0.0, "grad_norm": 1.3160443305969238, "kl": 0.04374655510764569, "learning_rate": 5.714285714285714e-05, "loss": 0.2579, "num_tokens": 4483294.0, "reward": -0.7569143772125244, "reward_std": 0.38748466968536377, "rewards/rollout_reward_func/mean": -0.7569143772125244, "rewards/rollout_reward_func/std": 0.47438445687294006, "sampling/importance_sampling_ratio/max": 2.5243661403656006, "sampling/importance_sampling_ratio/mean": 1.0004966259002686, "sampling/importance_sampling_ratio/min": 0.44545236229896545, "sampling/sampling_logp_difference/max": 0.9259899854660034, "sampling/sampling_logp_difference/mean": 0.011256169527769089, "step": 21, "step_time": 125.32715417199688 }, { "clip_ratio/high_max": 0.007965195982251316, "clip_ratio/high_mean": 0.004264129471266642, "clip_ratio/low_mean": 0.01166028477018699, "clip_ratio/low_min": 0.003762419626582414, "clip_ratio/region_mean": 0.015924414212349802, "entropy": 0.06479066435713321, "epoch": 0.00022, "grad_norm": 0.8557514548301697, "kl": 0.0534607790177688, "learning_rate": 6e-05, "loss": 0.2505, "step": 22, "step_time": 44.035087997997834 }, { "clip_ratio/high_max": 0.0011444397969171405, "clip_ratio/high_mean": 0.0005722198984585702, "clip_ratio/low_mean": 0.002795889973640442, "clip_ratio/low_min": 0.0005040322430431843, "clip_ratio/region_mean": 0.0033681098429951817, "completions/clipped_ratio": 0.65625, "completions/max_length": 14447.0, "completions/max_terminated_length": 13549.0, "completions/mean_length": 11808.34375, "completions/mean_terminated_length": 7141.36376953125, "completions/min_length": 784.0, "completions/min_terminated_length": 784.0, "entropy": 0.07080131443217397, "epoch": 0.00023, "frac_reward_zero_std": 0.0, "grad_norm": 1.2227104902267456, "kl": 0.10710712906438857, "learning_rate": 6.285714285714286e-05, "loss": 0.357, "num_tokens": 4888221.0, "reward": -0.49963146448135376, "reward_std": 0.8173227310180664, "rewards/rollout_reward_func/mean": -0.49963146448135376, "rewards/rollout_reward_func/std": 0.8128207921981812, "sampling/importance_sampling_ratio/max": 1.9062927961349487, "sampling/importance_sampling_ratio/mean": 1.0000483989715576, "sampling/importance_sampling_ratio/min": 0.30003389716148376, "sampling/sampling_logp_difference/max": 1.203859806060791, "sampling/sampling_logp_difference/mean": 0.011820203624665737, "step": 23, "step_time": 134.11656286700236 }, { "clip_ratio/high_max": 0.0063895476050674915, "clip_ratio/high_mean": 0.0031947738025337458, "clip_ratio/low_mean": 0.00737490743631497, "clip_ratio/low_min": 0.0014960957923904061, "clip_ratio/region_mean": 0.010569681267952546, "entropy": 0.07403647690080106, "epoch": 0.00024, "grad_norm": 0.9927102327346802, "kl": 0.07292677101213485, "learning_rate": 6.571428571428571e-05, "loss": 0.3453, "step": 24, "step_time": 44.88870704800138 }, { "clip_ratio/high_max": 0.002575038291979581, "clip_ratio/high_mean": 0.0012875191459897906, "clip_ratio/low_mean": 0.0031292334315367043, "clip_ratio/low_min": 0.0010516082984395325, "clip_ratio/region_mean": 0.004416752519318834, "completions/clipped_ratio": 0.6875, "completions/max_length": 14441.0, "completions/max_terminated_length": 13590.0, "completions/mean_length": 11539.0625, "completions/mean_terminated_length": 5616.10009765625, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.09310276620090008, "epoch": 0.00025, "frac_reward_zero_std": 0.0, "grad_norm": 1.282138466835022, "kl": 0.07473902264609933, "learning_rate": 6.857142857142858e-05, "loss": -0.0119, "num_tokens": 5284116.0, "reward": -0.7307593822479248, "reward_std": 0.4555254876613617, "rewards/rollout_reward_func/mean": -0.7307593822479248, "rewards/rollout_reward_func/std": 0.5767623782157898, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 1.0000531673431396, "sampling/importance_sampling_ratio/min": 0.18337896466255188, "sampling/sampling_logp_difference/max": 1.6962004899978638, "sampling/sampling_logp_difference/mean": 0.014519629999995232, "step": 25, "step_time": 128.11358543400092 }, { "clip_ratio/high_max": 0.006235250388272107, "clip_ratio/high_mean": 0.0041256895638071, "clip_ratio/low_mean": 0.019248695025453344, "clip_ratio/low_min": 0.006461259792558849, "clip_ratio/region_mean": 0.023374384531052783, "entropy": 0.09371388074941933, "epoch": 0.00026, "grad_norm": 1.1407207250595093, "kl": 0.10249935300089419, "learning_rate": 7.142857142857143e-05, "loss": -0.0179, "step": 26, "step_time": 44.78832636999505 }, { "clip_ratio/high_max": 0.001961233268957585, "clip_ratio/high_mean": 0.0009806166344787925, "clip_ratio/low_mean": 0.0014239389565773308, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0024045555910561234, "completions/clipped_ratio": 0.5, "completions/max_length": 14387.0, "completions/max_terminated_length": 13921.0, "completions/mean_length": 10618.1875, "completions/mean_terminated_length": 7001.0, "completions/min_length": 1065.0, "completions/min_terminated_length": 1065.0, "entropy": 0.08063321514055133, "epoch": 0.00027, "frac_reward_zero_std": 0.0, "grad_norm": 3.7377917766571045, "kl": 0.46857869252562523, "learning_rate": 7.428571428571429e-05, "loss": 0.0274, "num_tokens": 5650556.0, "reward": -0.4988600015640259, "reward_std": 0.6857542991638184, "rewards/rollout_reward_func/mean": -0.4988600015640259, "rewards/rollout_reward_func/std": 0.8108665347099304, "sampling/importance_sampling_ratio/max": 1.5844895839691162, "sampling/importance_sampling_ratio/mean": 0.9988585710525513, "sampling/importance_sampling_ratio/min": 0.14879053831100464, "sampling/sampling_logp_difference/max": 1.9052157402038574, "sampling/sampling_logp_difference/mean": 0.010426906868815422, "step": 27, "step_time": 117.60450818400022 }, { "clip_ratio/high_max": 0.011021792015526444, "clip_ratio/high_mean": 0.006221123243449256, "clip_ratio/low_mean": 0.007159021100960672, "clip_ratio/low_min": 0.0011492071789689362, "clip_ratio/region_mean": 0.01338014446082525, "entropy": 0.08160684490576386, "epoch": 0.00028, "grad_norm": 1.8652974367141724, "kl": 0.0501517970697023, "learning_rate": 7.714285714285715e-05, "loss": 0.0131, "step": 28, "step_time": 43.69357772600051 }, { "clip_ratio/high_max": 0.005480502266436815, "clip_ratio/high_mean": 0.0027402511332184076, "clip_ratio/low_mean": 0.0009808092145249248, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0037210604059509933, "completions/clipped_ratio": 0.5, "completions/max_length": 14524.0, "completions/max_terminated_length": 13789.0, "completions/mean_length": 10754.5, "completions/mean_terminated_length": 7253.3125, "completions/min_length": 1385.0, "completions/min_terminated_length": 1385.0, "entropy": 0.08766714809462428, "epoch": 0.00029, "frac_reward_zero_std": 0.0, "grad_norm": 0.9512283205986023, "kl": 0.09233958297409117, "learning_rate": 8e-05, "loss": 0.3863, "num_tokens": 6021181.0, "reward": -0.43986791372299194, "reward_std": 0.8378865718841553, "rewards/rollout_reward_func/mean": -0.43986791372299194, "rewards/rollout_reward_func/std": 0.8483731150627136, "sampling/importance_sampling_ratio/max": 1.6999077796936035, "sampling/importance_sampling_ratio/mean": 0.9990843534469604, "sampling/importance_sampling_ratio/min": 0.4758984446525574, "sampling/sampling_logp_difference/max": 0.7425508499145508, "sampling/sampling_logp_difference/mean": 0.010583655908703804, "step": 29, "step_time": 122.50815704600063 }, { "clip_ratio/high_max": 0.011935517424717546, "clip_ratio/high_mean": 0.006662203115411103, "clip_ratio/low_mean": 0.008739105775021017, "clip_ratio/low_min": 0.004148350621107966, "clip_ratio/region_mean": 0.01540130889043212, "entropy": 0.09500867640599608, "epoch": 0.0003, "grad_norm": 0.7339725494384766, "kl": 0.0838697346043773, "learning_rate": 8.285714285714287e-05, "loss": 0.3791, "step": 30, "step_time": 44.16028004800319 }, { "clip_ratio/high_max": 0.002299328800290823, "clip_ratio/high_mean": 0.0011496644001454115, "clip_ratio/low_mean": 0.0013620644458569586, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00251172884600237, "completions/clipped_ratio": 0.71875, "completions/max_length": 14550.0, "completions/max_terminated_length": 13983.0, "completions/mean_length": 12700.71875, "completions/mean_terminated_length": 8789.888671875, "completions/min_length": 3262.0, "completions/min_terminated_length": 3262.0, "entropy": 0.11977054877206683, "epoch": 0.00031, "frac_reward_zero_std": 0.0, "grad_norm": 1.086565613746643, "kl": 0.05921647744253278, "learning_rate": 8.571428571428571e-05, "loss": 0.0947, "num_tokens": 6454382.0, "reward": -0.7849480509757996, "reward_std": 0.38423284888267517, "rewards/rollout_reward_func/mean": -0.7849480509757996, "rewards/rollout_reward_func/std": 0.48005345463752747, "sampling/importance_sampling_ratio/max": 1.8024439811706543, "sampling/importance_sampling_ratio/mean": 1.000292181968689, "sampling/importance_sampling_ratio/min": 0.5424980521202087, "sampling/sampling_logp_difference/max": 0.6115708351135254, "sampling/sampling_logp_difference/mean": 0.01138882152736187, "step": 31, "step_time": 127.06237872199927 }, { "clip_ratio/high_max": 0.012531265849247575, "clip_ratio/high_mean": 0.006265632924623787, "clip_ratio/low_mean": 0.00913364501320757, "clip_ratio/low_min": 0.0016989448922686279, "clip_ratio/region_mean": 0.015399277937831357, "entropy": 0.1310350843705237, "epoch": 0.00032, "grad_norm": 0.8760910034179688, "kl": 0.06428600556682795, "learning_rate": 8.857142857142857e-05, "loss": 0.0848, "step": 32, "step_time": 45.15639938999993 }, { "clip_ratio/high_max": 0.001622461248189211, "clip_ratio/high_mean": 0.0008112306240946054, "clip_ratio/low_mean": 0.0012673057353822514, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.002078536359476857, "completions/clipped_ratio": 0.5, "completions/max_length": 14485.0, "completions/max_terminated_length": 13438.0, "completions/mean_length": 11201.46875, "completions/mean_terminated_length": 8163.75, "completions/min_length": 1137.0, "completions/min_terminated_length": 1137.0, "entropy": 0.11091703665442765, "epoch": 0.00033, "frac_reward_zero_std": 0.0, "grad_norm": 0.763496994972229, "kl": 0.08535079064313322, "learning_rate": 9.142857142857143e-05, "loss": 0.187, "num_tokens": 6839240.0, "reward": -0.606515109539032, "reward_std": 0.644769549369812, "rewards/rollout_reward_func/mean": -0.606515109539032, "rewards/rollout_reward_func/std": 0.7157695293426514, "sampling/importance_sampling_ratio/max": 1.9776629209518433, "sampling/importance_sampling_ratio/mean": 1.0004677772521973, "sampling/importance_sampling_ratio/min": 0.6363774538040161, "sampling/sampling_logp_difference/max": 0.6819157600402832, "sampling/sampling_logp_difference/mean": 0.010047465562820435, "step": 33, "step_time": 122.12401338600102 }, { "clip_ratio/high_max": 0.003852536785416305, "clip_ratio/high_mean": 0.0024514784454368055, "clip_ratio/low_mean": 0.010322274029022083, "clip_ratio/low_min": 0.0054294930887408555, "clip_ratio/region_mean": 0.012773752474458888, "entropy": 0.11235238378867507, "epoch": 0.00034, "grad_norm": 0.9141661524772644, "kl": 0.08446309305145405, "learning_rate": 9.428571428571429e-05, "loss": 0.1825, "step": 34, "step_time": 43.25774885099963 }, { "clip_ratio/high_max": 0.002067690948024392, "clip_ratio/high_mean": 0.001033845474012196, "clip_ratio/low_mean": 0.0013292695221025497, "clip_ratio/low_min": 0.0005208333604969084, "clip_ratio/region_mean": 0.0023631149961147457, "completions/clipped_ratio": 0.625, "completions/max_length": 14504.0, "completions/max_terminated_length": 13750.0, "completions/mean_length": 12662.125, "completions/mean_terminated_length": 10072.6669921875, "completions/min_length": 2136.0, "completions/min_terminated_length": 2136.0, "entropy": 0.13832575432024896, "epoch": 0.00035, "frac_reward_zero_std": 0.0, "grad_norm": 0.8794277906417847, "kl": 0.08255716518033296, "learning_rate": 9.714285714285715e-05, "loss": 0.0863, "num_tokens": 7271283.0, "reward": -0.5410826206207275, "reward_std": 0.7768348455429077, "rewards/rollout_reward_func/mean": -0.5410826206207275, "rewards/rollout_reward_func/std": 0.7597488760948181, "sampling/importance_sampling_ratio/max": 1.5629594326019287, "sampling/importance_sampling_ratio/mean": 1.0001167058944702, "sampling/importance_sampling_ratio/min": 0.5588953495025635, "sampling/sampling_logp_difference/max": 0.5817930698394775, "sampling/sampling_logp_difference/mean": 0.009778056293725967, "step": 35, "step_time": 130.91799312700277 }, { "clip_ratio/high_max": 0.011173836537636817, "clip_ratio/high_mean": 0.005586918268818408, "clip_ratio/low_mean": 0.00869341439101845, "clip_ratio/low_min": 0.004238906840328127, "clip_ratio/region_mean": 0.01428033280535601, "entropy": 0.15124066965654492, "epoch": 0.00036, "grad_norm": 0.7353250980377197, "kl": 0.08487725246232003, "learning_rate": 0.0001, "loss": 0.0769, "step": 36, "step_time": 45.46909211699858 }, { "clip_ratio/high_max": 0.0015989625826478004, "clip_ratio/high_mean": 0.0007994812913239002, "clip_ratio/low_mean": 0.0020003379322588444, "clip_ratio/low_min": 0.0005165289039723575, "clip_ratio/region_mean": 0.0027998192235827446, "completions/clipped_ratio": 0.625, "completions/max_length": 14481.0, "completions/max_terminated_length": 13367.0, "completions/mean_length": 11635.15625, "completions/mean_terminated_length": 7297.1669921875, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "entropy": 0.13707713410258293, "epoch": 0.00037, "frac_reward_zero_std": 0.0, "grad_norm": 0.8795878887176514, "kl": 0.09341590362600982, "learning_rate": 9.918053502751771e-05, "loss": 0.1423, "num_tokens": 7670326.0, "reward": -0.6904894113540649, "reward_std": 0.6018991470336914, "rewards/rollout_reward_func/mean": -0.6904894113540649, "rewards/rollout_reward_func/std": 0.6566587686538696, "sampling/importance_sampling_ratio/max": 1.4667878150939941, "sampling/importance_sampling_ratio/mean": 0.999151349067688, "sampling/importance_sampling_ratio/min": 0.3191526234149933, "sampling/sampling_logp_difference/max": 1.1420859098434448, "sampling/sampling_logp_difference/mean": 0.01049021165817976, "step": 37, "step_time": 128.31693721999363 }, { "clip_ratio/high_max": 0.012058351479936391, "clip_ratio/high_mean": 0.007017562194960192, "clip_ratio/low_mean": 0.010125949047505856, "clip_ratio/low_min": 0.002969218941871077, "clip_ratio/region_mean": 0.01714351132977754, "entropy": 0.13681618124246597, "epoch": 0.00038, "grad_norm": 0.6959795951843262, "kl": 0.09944733651354909, "learning_rate": 9.675795466159755e-05, "loss": 0.13, "step": 38, "step_time": 45.151127613000426 }, { "clip_ratio/high_max": 0.001359933929052204, "clip_ratio/high_mean": 0.000679966964526102, "clip_ratio/low_mean": 0.001585537102073431, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0022655040957033634, "completions/clipped_ratio": 0.59375, "completions/max_length": 14393.0, "completions/max_terminated_length": 13179.0, "completions/mean_length": 11368.53125, "completions/mean_terminated_length": 7292.30810546875, "completions/min_length": 1023.0, "completions/min_terminated_length": 1023.0, "entropy": 0.11355275707319379, "epoch": 0.00039, "frac_reward_zero_std": 0.0, "grad_norm": 0.8372141122817993, "kl": 0.12875043111853302, "learning_rate": 9.283813728906054e-05, "loss": -0.0014, "num_tokens": 8061296.0, "reward": -0.6022458076477051, "reward_std": 0.6515352129936218, "rewards/rollout_reward_func/mean": -0.6022458076477051, "rewards/rollout_reward_func/std": 0.7095517516136169, "sampling/importance_sampling_ratio/max": 1.636765480041504, "sampling/importance_sampling_ratio/mean": 0.9989644289016724, "sampling/importance_sampling_ratio/min": 0.47850245237350464, "sampling/sampling_logp_difference/max": 0.7370939254760742, "sampling/sampling_logp_difference/mean": 0.010086959227919579, "step": 39, "step_time": 126.83774206699854 }, { "clip_ratio/high_max": 0.008263030322268605, "clip_ratio/high_mean": 0.004131515161134303, "clip_ratio/low_mean": 0.014172348921420053, "clip_ratio/low_min": 0.0036224325885996222, "clip_ratio/region_mean": 0.018303864169865847, "entropy": 0.10010521637741476, "epoch": 0.0004, "grad_norm": 0.7000178098678589, "kl": 0.14327447954565287, "learning_rate": 8.759239773845718e-05, "loss": -0.0115, "step": 40, "step_time": 44.916688461000376 }, { "clip_ratio/high_max": 0.001846958533860743, "clip_ratio/high_mean": 0.0009234792669303715, "clip_ratio/low_mean": 0.002370692673139274, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0032941719400696456, "completions/clipped_ratio": 0.65625, "completions/max_length": 14506.0, "completions/max_terminated_length": 13335.0, "completions/mean_length": 11941.9375, "completions/mean_terminated_length": 7638.63671875, "completions/min_length": 1452.0, "completions/min_terminated_length": 1452.0, "entropy": 0.11166324256919324, "epoch": 0.00041, "frac_reward_zero_std": 0.0, "grad_norm": 1.0560439825057983, "kl": 0.10882639326155186, "learning_rate": 8.125000000000001e-05, "loss": 0.5418, "num_tokens": 8470213.0, "reward": -0.4550766348838806, "reward_std": 0.7941820621490479, "rewards/rollout_reward_func/mean": -0.4550766348838806, "rewards/rollout_reward_func/std": 0.7887347340583801, "sampling/importance_sampling_ratio/max": 1.8251365423202515, "sampling/importance_sampling_ratio/mean": 1.0004825592041016, "sampling/importance_sampling_ratio/min": 0.5376880168914795, "sampling/sampling_logp_difference/max": 0.6204767227172852, "sampling/sampling_logp_difference/mean": 0.011826084926724434, "step": 41, "step_time": 126.00419519500247 }, { "clip_ratio/high_max": 0.007851478818338364, "clip_ratio/high_mean": 0.003925739409169182, "clip_ratio/low_mean": 0.014391002681804821, "clip_ratio/low_min": 0.005810233065858483, "clip_ratio/region_mean": 0.018316742061870173, "entropy": 0.10689377831295133, "epoch": 0.00042, "grad_norm": 1.3130351305007935, "kl": 0.1170152414124459, "learning_rate": 7.408813728906053e-05, "loss": 0.5353, "step": 42, "step_time": 45.83305146299972 }, { "clip_ratio/high_max": 0.002037170110270381, "clip_ratio/high_mean": 0.0010185850551351905, "clip_ratio/low_mean": 0.0016175404307432473, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026361254567746073, "completions/clipped_ratio": 0.53125, "completions/max_length": 14370.0, "completions/max_terminated_length": 13214.0, "completions/mean_length": 11265.375, "completions/mean_terminated_length": 7982.400390625, "completions/min_length": 782.0, "completions/min_terminated_length": 782.0, "entropy": 0.09073930280283093, "epoch": 0.00043, "frac_reward_zero_std": 0.0, "grad_norm": 1.7020624876022339, "kl": 0.13664298865478486, "learning_rate": 6.6419817372537e-05, "loss": 0.1856, "num_tokens": 8857416.0, "reward": -0.37660786509513855, "reward_std": 0.8688997030258179, "rewards/rollout_reward_func/mean": -0.37660786509513855, "rewards/rollout_reward_func/std": 0.8781781196594238, "sampling/importance_sampling_ratio/max": 1.9817417860031128, "sampling/importance_sampling_ratio/mean": 1.0001003742218018, "sampling/importance_sampling_ratio/min": 0.4231337010860443, "sampling/sampling_logp_difference/max": 0.8600671291351318, "sampling/sampling_logp_difference/mean": 0.009302067570388317, "step": 43, "step_time": 121.0720823820011 }, { "clip_ratio/high_max": 0.007119149784557521, "clip_ratio/high_mean": 0.0035595748922787607, "clip_ratio/low_mean": 0.01176984136691317, "clip_ratio/low_min": 0.002642020583152771, "clip_ratio/region_mean": 0.01532941620098427, "entropy": 0.08315122197382152, "epoch": 0.00044, "grad_norm": 5.769225597381592, "kl": 0.6573639016132802, "learning_rate": 5.858018262746301e-05, "loss": 0.1936, "step": 44, "step_time": 44.69906600800459 }, { "clip_ratio/high_max": 0.0023281591129489243, "clip_ratio/high_mean": 0.0011640795564744622, "clip_ratio/low_mean": 0.002193144930060953, "clip_ratio/low_min": 0.0005530973430722952, "clip_ratio/region_mean": 0.0033572244574315846, "completions/clipped_ratio": 0.53125, "completions/max_length": 14504.0, "completions/max_terminated_length": 13747.0, "completions/mean_length": 11228.96875, "completions/mean_terminated_length": 7874.400390625, "completions/min_length": 1631.0, "completions/min_terminated_length": 1631.0, "entropy": 0.08939703018404543, "epoch": 0.00045, "frac_reward_zero_std": 0.0, "grad_norm": 1.0448198318481445, "kl": 0.11512639001011848, "learning_rate": 5.0911862710939485e-05, "loss": 0.0475, "num_tokens": 9243782.0, "reward": -0.49534285068511963, "reward_std": 0.7042239904403687, "rewards/rollout_reward_func/mean": -0.49534285068511963, "rewards/rollout_reward_func/std": 0.8103145360946655, "sampling/importance_sampling_ratio/max": 1.6891045570373535, "sampling/importance_sampling_ratio/mean": 1.0011885166168213, "sampling/importance_sampling_ratio/min": 0.5372036695480347, "sampling/sampling_logp_difference/max": 0.6213779449462891, "sampling/sampling_logp_difference/mean": 0.0091001121327281, "step": 45, "step_time": 120.37874379399727 }, { "clip_ratio/high_max": 0.008119428472127765, "clip_ratio/high_mean": 0.004059714236063883, "clip_ratio/low_mean": 0.008602545305620879, "clip_ratio/low_min": 0.0037590391584672034, "clip_ratio/region_mean": 0.01266225945437327, "entropy": 0.08823490096256137, "epoch": 0.00046, "grad_norm": 0.8318017721176147, "kl": 0.120926967356354, "learning_rate": 4.375000000000001e-05, "loss": 0.0362, "step": 46, "step_time": 44.49746013800177 }, { "clip_ratio/high_max": 0.003532699483912438, "clip_ratio/high_mean": 0.001766349741956219, "clip_ratio/low_mean": 0.0016052687860792503, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0033716185280354694, "completions/clipped_ratio": 0.65625, "completions/max_length": 14315.0, "completions/max_terminated_length": 13481.0, "completions/mean_length": 11687.5625, "completions/mean_terminated_length": 7038.7275390625, "completions/min_length": 1557.0, "completions/min_terminated_length": 1557.0, "entropy": 0.07309431000612676, "epoch": 0.00047, "frac_reward_zero_std": 0.0, "grad_norm": 0.8768713474273682, "kl": 0.08262627921067178, "learning_rate": 3.740760226154283e-05, "loss": 0.3828, "num_tokens": 9644940.0, "reward": -0.4843946695327759, "reward_std": 0.8082762360572815, "rewards/rollout_reward_func/mean": -0.4843946695327759, "rewards/rollout_reward_func/std": 0.8041583895683289, "sampling/importance_sampling_ratio/max": 1.4661790132522583, "sampling/importance_sampling_ratio/mean": 1.000061273574829, "sampling/importance_sampling_ratio/min": 0.3773066997528076, "sampling/sampling_logp_difference/max": 0.9746968746185303, "sampling/sampling_logp_difference/mean": 0.007087397389113903, "step": 47, "step_time": 121.44068199299727 }, { "clip_ratio/high_max": 0.01053907914320007, "clip_ratio/high_mean": 0.005612946202745661, "clip_ratio/low_mean": 0.00564750301418826, "clip_ratio/low_min": 0.0020543700957205147, "clip_ratio/region_mean": 0.01126044918783009, "entropy": 0.07405747356824577, "epoch": 0.00048, "grad_norm": 0.811284601688385, "kl": 0.08346198999788612, "learning_rate": 3.2161862710939476e-05, "loss": 0.374, "step": 48, "step_time": 44.960870664997856 }, { "clip_ratio/high_max": 0.002270299184601754, "clip_ratio/high_mean": 0.001135149592300877, "clip_ratio/low_mean": 0.003153837809804827, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.004288987343898043, "completions/clipped_ratio": 0.625, "completions/max_length": 14530.0, "completions/max_terminated_length": 13232.0, "completions/mean_length": 11548.625, "completions/mean_terminated_length": 7039.5, "completions/min_length": 1709.0, "completions/min_terminated_length": 1709.0, "entropy": 0.08768759481608868, "epoch": 0.00049, "frac_reward_zero_std": 0.0, "grad_norm": 1.6150716543197632, "kl": 0.3609654267784208, "learning_rate": 2.8242045338402468e-05, "loss": 0.4855, "num_tokens": 10041134.0, "reward": -0.5441310405731201, "reward_std": 0.6545704007148743, "rewards/rollout_reward_func/mean": -0.5441310405731201, "rewards/rollout_reward_func/std": 0.7639203667640686, "sampling/importance_sampling_ratio/max": 1.7008589506149292, "sampling/importance_sampling_ratio/mean": 1.0008265972137451, "sampling/importance_sampling_ratio/min": 0.45911118388175964, "sampling/sampling_logp_difference/max": 0.7784628868103027, "sampling/sampling_logp_difference/mean": 0.010290013626217842, "step": 49, "step_time": 124.03581126300378 }, { "clip_ratio/high_max": 0.004471153952181339, "clip_ratio/high_mean": 0.0022355769760906696, "clip_ratio/low_mean": 0.007084499462507665, "clip_ratio/low_min": 0.0021499572903849185, "clip_ratio/region_mean": 0.009320076409494504, "entropy": 0.08996679773554206, "epoch": 0.0005, "grad_norm": 0.8601014018058777, "kl": 0.24557857040781528, "learning_rate": 2.5819464972482288e-05, "loss": 0.4747, "step": 50, "step_time": 45.77899998500652 }, { "epoch": 0.0005, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.60875, "eval_completions/max_length": 13600.67375, "eval_completions/max_terminated_length": 5307.3125, "eval_completions/mean_length": 11798.84625, "eval_completions/mean_terminated_length": 4963.2975, "eval_completions/min_length": 9997.01875, "eval_completions/min_terminated_length": 4619.2825, "eval_entropy": 0.09010161134880036, "eval_frac_reward_zero_std": 1.0, "eval_kl": 0.13719033321132884, "eval_loss": 0.00026262112078256905, "eval_num_tokens": 10041134.0, "eval_reward": -0.47466856505721805, "eval_reward_std": 0.0, "eval_rewards/rollout_reward_func/mean": -0.47466856505721805, "eval_rewards/rollout_reward_func/std": 0.509479589665134, "eval_runtime": 4806.419, "eval_samples_per_second": 0.042, "eval_sampling/importance_sampling_ratio/max": 1.2884997601807118, "eval_sampling/importance_sampling_ratio/mean": 1.000060096681118, "eval_sampling/importance_sampling_ratio/min": 0.7314270212873817, "eval_sampling/sampling_logp_difference/max": 0.38731383727863433, "eval_sampling/sampling_logp_difference/mean": 0.00951688139612088, "eval_steps_per_second": 0.021, "step": 50 } ], "logging_steps": 1.0, "max_steps": 50, "num_input_tokens_seen": 10041134, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }