| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0005, |
| "eval_steps": 500, |
| "global_step": 50, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14431.0, |
| "completions/max_terminated_length": 13090.0, |
| "completions/mean_length": 11799.78125, |
| "completions/mean_terminated_length": 7162.27294921875, |
| "completions/min_length": 235.0, |
| "completions/min_terminated_length": 235.0, |
| "entropy": 0.13893846329301596, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7887574434280396, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0596, |
| "num_tokens": 404630.0, |
| "reward": -0.7717239856719971, |
| "reward_std": 0.42467162013053894, |
| "rewards/rollout_reward_func/mean": -0.7717239856719971, |
| "rewards/rollout_reward_func/std": 0.5858266353607178, |
| "sampling/importance_sampling_ratio/max": 1.7099345922470093, |
| "sampling/importance_sampling_ratio/mean": 0.9991271495819092, |
| "sampling/importance_sampling_ratio/min": 0.4281526207923889, |
| "sampling/sampling_logp_difference/max": 0.8482755422592163, |
| "sampling/sampling_logp_difference/mean": 0.012976451776921749, |
| "step": 1, |
| "step_time": 130.9108755219977 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.13893846329301596, |
| "epoch": 2e-05, |
| "grad_norm": 1.804929494857788, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": -0.0596, |
| "step": 2, |
| "step_time": 44.13144713700058 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022670181933790445, |
| "clip_ratio/high_mean": 0.0011335090966895223, |
| "clip_ratio/low_mean": 0.0029946123831905425, |
| "clip_ratio/low_min": 0.0005040322430431843, |
| "clip_ratio/region_mean": 0.004128121479880065, |
| "completions/clipped_ratio": 0.6875, |
| "completions/max_length": 14439.0, |
| "completions/max_terminated_length": 11449.0, |
| "completions/mean_length": 11617.5, |
| "completions/mean_terminated_length": 5924.5, |
| "completions/min_length": 323.0, |
| "completions/min_terminated_length": 323.0, |
| "entropy": 0.12676118151284754, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.4913136959075928, |
| "kl": 0.0009995185955631314, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": 0.0137, |
| "num_tokens": 803817.0, |
| "reward": -0.8155950307846069, |
| "reward_std": 0.4161786139011383, |
| "rewards/rollout_reward_func/mean": -0.8155950307846069, |
| "rewards/rollout_reward_func/std": 0.4885525703430176, |
| "sampling/importance_sampling_ratio/max": 2.0640246868133545, |
| "sampling/importance_sampling_ratio/mean": 0.9992372393608093, |
| "sampling/importance_sampling_ratio/min": 0.41666513681411743, |
| "sampling/sampling_logp_difference/max": 0.8754724264144897, |
| "sampling/sampling_logp_difference/mean": 0.011403773911297321, |
| "step": 3, |
| "step_time": 126.63972502899833 |
| }, |
| { |
| "clip_ratio/high_max": 0.005151324032340199, |
| "clip_ratio/high_mean": 0.0025756620161700994, |
| "clip_ratio/low_mean": 0.001626400655368343, |
| "clip_ratio/low_min": 0.0005040322430431843, |
| "clip_ratio/region_mean": 0.004202062671538442, |
| "entropy": 0.12616439966950566, |
| "epoch": 4e-05, |
| "grad_norm": 1.4003232717514038, |
| "kl": 0.001313611121076974, |
| "learning_rate": 8.571428571428573e-06, |
| "loss": 0.0134, |
| "step": 4, |
| "step_time": 44.62073808600053 |
| }, |
| { |
| "clip_ratio/high_max": 0.0016215514624491334, |
| "clip_ratio/high_mean": 0.0008107757312245667, |
| "clip_ratio/low_mean": 0.003547743574017659, |
| "clip_ratio/low_min": 0.0010995370685122907, |
| "clip_ratio/region_mean": 0.0043585193052422255, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 14535.0, |
| "completions/max_terminated_length": 11037.0, |
| "completions/mean_length": 12972.78125, |
| "completions/mean_terminated_length": 7317.5, |
| "completions/min_length": 4349.0, |
| "completions/min_terminated_length": 4349.0, |
| "entropy": 0.1461638123728335, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.107853889465332, |
| "kl": 0.001256127497072157, |
| "learning_rate": 1.1428571428571429e-05, |
| "loss": 0.1588, |
| "num_tokens": 1245837.0, |
| "reward": -0.6968419551849365, |
| "reward_std": 0.4676758050918579, |
| "rewards/rollout_reward_func/mean": -0.6968419551849365, |
| "rewards/rollout_reward_func/std": 0.5711839199066162, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.001021385192871, |
| "sampling/importance_sampling_ratio/min": 0.5821155905723572, |
| "sampling/sampling_logp_difference/max": 1.1031980514526367, |
| "sampling/sampling_logp_difference/mean": 0.01345288660377264, |
| "step": 5, |
| "step_time": 131.89395336500274 |
| }, |
| { |
| "clip_ratio/high_max": 0.0021937466226518154, |
| "clip_ratio/high_mean": 0.0010968733113259077, |
| "clip_ratio/low_mean": 0.0032513330806978047, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004348206392023712, |
| "entropy": 0.14492363901808858, |
| "epoch": 6e-05, |
| "grad_norm": 1.6609207391738892, |
| "kl": 0.001644153301640472, |
| "learning_rate": 1.4285714285714285e-05, |
| "loss": 0.1578, |
| "step": 6, |
| "step_time": 46.381975520000196 |
| }, |
| { |
| "clip_ratio/high_max": 0.0031376838451251388, |
| "clip_ratio/high_mean": 0.0015688419225625694, |
| "clip_ratio/low_mean": 0.0033995709964074194, |
| "clip_ratio/low_min": 0.0015243901871144772, |
| "clip_ratio/region_mean": 0.004968412918969989, |
| "completions/clipped_ratio": 0.5625, |
| "completions/max_length": 14446.0, |
| "completions/max_terminated_length": 12650.0, |
| "completions/mean_length": 11378.28125, |
| "completions/mean_terminated_length": 7822.64306640625, |
| "completions/min_length": 1771.0, |
| "completions/min_terminated_length": 1771.0, |
| "entropy": 0.0922874582465738, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.8714547157287598, |
| "kl": 0.0019052933557759388, |
| "learning_rate": 1.7142857142857145e-05, |
| "loss": 0.3544, |
| "num_tokens": 1637608.0, |
| "reward": -0.4466933012008667, |
| "reward_std": 0.8536279201507568, |
| "rewards/rollout_reward_func/mean": -0.4466933012008667, |
| "rewards/rollout_reward_func/std": 0.8547493815422058, |
| "sampling/importance_sampling_ratio/max": 2.0912468433380127, |
| "sampling/importance_sampling_ratio/mean": 0.9991166591644287, |
| "sampling/importance_sampling_ratio/min": 0.48319441080093384, |
| "sampling/sampling_logp_difference/max": 0.7377604246139526, |
| "sampling/sampling_logp_difference/mean": 0.01108577847480774, |
| "step": 7, |
| "step_time": 126.40769893900142 |
| }, |
| { |
| "clip_ratio/high_max": 0.004281101922970265, |
| "clip_ratio/high_mean": 0.002941832965007052, |
| "clip_ratio/low_mean": 0.004934519121889025, |
| "clip_ratio/low_min": 0.001998567197006196, |
| "clip_ratio/region_mean": 0.007876352086896077, |
| "entropy": 0.08806704939343035, |
| "epoch": 8e-05, |
| "grad_norm": 1.634592890739441, |
| "kl": 0.003069787417189218, |
| "learning_rate": 2e-05, |
| "loss": 0.347, |
| "step": 8, |
| "step_time": 44.41796817000068 |
| }, |
| { |
| "clip_ratio/high_max": 0.0016580428346060216, |
| "clip_ratio/high_mean": 0.0008290214173030108, |
| "clip_ratio/low_mean": 0.00299244187772274, |
| "clip_ratio/low_min": 0.0006009615608491004, |
| "clip_ratio/region_mean": 0.003821463295025751, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14526.0, |
| "completions/max_terminated_length": 13961.0, |
| "completions/mean_length": 11837.40625, |
| "completions/mean_terminated_length": 7239.818359375, |
| "completions/min_length": 967.0, |
| "completions/min_terminated_length": 967.0, |
| "entropy": 0.10492791910655797, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7168275117874146, |
| "kl": 0.004868662523222156, |
| "learning_rate": 2.2857142857142858e-05, |
| "loss": -0.0376, |
| "num_tokens": 2043692.0, |
| "reward": -0.6215145587921143, |
| "reward_std": 0.6429688930511475, |
| "rewards/rollout_reward_func/mean": -0.6215145587921143, |
| "rewards/rollout_reward_func/std": 0.7194304466247559, |
| "sampling/importance_sampling_ratio/max": 1.6615937948226929, |
| "sampling/importance_sampling_ratio/mean": 1.0002217292785645, |
| "sampling/importance_sampling_ratio/min": 0.47998860478401184, |
| "sampling/sampling_logp_difference/max": 0.7339929938316345, |
| "sampling/sampling_logp_difference/mean": 0.010960595682263374, |
| "step": 9, |
| "step_time": 124.44129867599804 |
| }, |
| { |
| "clip_ratio/high_max": 0.005017199146095663, |
| "clip_ratio/high_mean": 0.0025085995730478317, |
| "clip_ratio/low_mean": 0.00814974692184478, |
| "clip_ratio/low_min": 0.002315932302735746, |
| "clip_ratio/region_mean": 0.010658346494892612, |
| "entropy": 0.09843971265945584, |
| "epoch": 0.0001, |
| "grad_norm": 1.3552314043045044, |
| "kl": 0.01046636993123684, |
| "learning_rate": 2.5714285714285714e-05, |
| "loss": -0.0446, |
| "step": 10, |
| "step_time": 45.56444605700199 |
| }, |
| { |
| "clip_ratio/high_max": 0.0035993335768580437, |
| "clip_ratio/high_mean": 0.0017996667884290218, |
| "clip_ratio/low_mean": 0.003951315797166899, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.005750982585595921, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14459.0, |
| "completions/max_terminated_length": 13816.0, |
| "completions/mean_length": 11661.625, |
| "completions/mean_terminated_length": 6824.5458984375, |
| "completions/min_length": 339.0, |
| "completions/min_terminated_length": 339.0, |
| "entropy": 0.06328993663191795, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7052122354507446, |
| "kl": 0.020232660688634496, |
| "learning_rate": 2.857142857142857e-05, |
| "loss": -0.0762, |
| "num_tokens": 2444128.0, |
| "reward": -0.7770951390266418, |
| "reward_std": 0.4045039117336273, |
| "rewards/rollout_reward_func/mean": -0.7770951390266418, |
| "rewards/rollout_reward_func/std": 0.4794606566429138, |
| "sampling/importance_sampling_ratio/max": 2.010491371154785, |
| "sampling/importance_sampling_ratio/mean": 0.9987582564353943, |
| "sampling/importance_sampling_ratio/min": 0.159025177359581, |
| "sampling/sampling_logp_difference/max": 1.8386926651000977, |
| "sampling/sampling_logp_difference/mean": 0.008627503179013729, |
| "step": 11, |
| "step_time": 121.11756841299757 |
| }, |
| { |
| "clip_ratio/high_max": 0.005686733347829431, |
| "clip_ratio/high_mean": 0.0028433666739147156, |
| "clip_ratio/low_mean": 0.012177724187495187, |
| "clip_ratio/low_min": 0.006776289606932551, |
| "clip_ratio/region_mean": 0.015021090890513733, |
| "entropy": 0.06035556551069021, |
| "epoch": 0.00012, |
| "grad_norm": 1.6369805335998535, |
| "kl": 0.06257348190410994, |
| "learning_rate": 3.142857142857143e-05, |
| "loss": -0.0747, |
| "step": 12, |
| "step_time": 44.37280418500268 |
| }, |
| { |
| "clip_ratio/high_max": 0.006585154333151877, |
| "clip_ratio/high_mean": 0.003569125838112086, |
| "clip_ratio/low_mean": 0.00462483472074382, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.008193960617063567, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 14486.0, |
| "completions/max_terminated_length": 13435.0, |
| "completions/mean_length": 12772.375, |
| "completions/mean_terminated_length": 6521.1669921875, |
| "completions/min_length": 815.0, |
| "completions/min_terminated_length": 815.0, |
| "entropy": 0.09321935242041945, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.849172592163086, |
| "kl": 0.03665067162364721, |
| "learning_rate": 3.428571428571429e-05, |
| "loss": -0.1546, |
| "num_tokens": 2880635.0, |
| "reward": -0.927730917930603, |
| "reward_std": 0.10113543272018433, |
| "rewards/rollout_reward_func/mean": -0.927730917930603, |
| "rewards/rollout_reward_func/std": 0.10532300174236298, |
| "sampling/importance_sampling_ratio/max": 1.936368465423584, |
| "sampling/importance_sampling_ratio/mean": 0.9986484050750732, |
| "sampling/importance_sampling_ratio/min": 0.18729014694690704, |
| "sampling/sampling_logp_difference/max": 1.6750962734222412, |
| "sampling/sampling_logp_difference/mean": 0.01844533160328865, |
| "step": 13, |
| "step_time": 130.46155179900052 |
| }, |
| { |
| "clip_ratio/high_max": 0.007653064152691513, |
| "clip_ratio/high_mean": 0.0043634974281303585, |
| "clip_ratio/low_mean": 0.014142459142021835, |
| "clip_ratio/low_min": 0.0029499504598788917, |
| "clip_ratio/region_mean": 0.018505956570152193, |
| "entropy": 0.09064974845387042, |
| "epoch": 0.00014, |
| "grad_norm": 8.18207836151123, |
| "kl": 0.2905037718010135, |
| "learning_rate": 3.7142857142857143e-05, |
| "loss": -0.1459, |
| "step": 14, |
| "step_time": 44.158161802002724 |
| }, |
| { |
| "clip_ratio/high_max": 0.0027650215197354555, |
| "clip_ratio/high_mean": 0.0013825107598677278, |
| "clip_ratio/low_mean": 0.004234670544974506, |
| "clip_ratio/low_min": 0.0012649551499634981, |
| "clip_ratio/region_mean": 0.005617181275738403, |
| "completions/clipped_ratio": 0.8125, |
| "completions/max_length": 14400.0, |
| "completions/max_terminated_length": 9012.0, |
| "completions/mean_length": 12419.84375, |
| "completions/mean_terminated_length": 4916.33349609375, |
| "completions/min_length": 1128.0, |
| "completions/min_terminated_length": 1128.0, |
| "entropy": 0.07450041663832963, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.2030179500579834, |
| "kl": 0.07086092534882482, |
| "learning_rate": 4e-05, |
| "loss": 0.1292, |
| "num_tokens": 3305559.0, |
| "reward": -0.7803157567977905, |
| "reward_std": 0.42063942551612854, |
| "rewards/rollout_reward_func/mean": -0.7803157567977905, |
| "rewards/rollout_reward_func/std": 0.4826337397098541, |
| "sampling/importance_sampling_ratio/max": 2.0322628021240234, |
| "sampling/importance_sampling_ratio/mean": 0.9985103607177734, |
| "sampling/importance_sampling_ratio/min": 0.24719201028347015, |
| "sampling/sampling_logp_difference/max": 1.397589921951294, |
| "sampling/sampling_logp_difference/mean": 0.015935054048895836, |
| "step": 15, |
| "step_time": 129.98408898099842 |
| }, |
| { |
| "clip_ratio/high_max": 0.008143757819198072, |
| "clip_ratio/high_mean": 0.004071878909599036, |
| "clip_ratio/low_mean": 0.014440719707636163, |
| "clip_ratio/low_min": 0.0063073321944102645, |
| "clip_ratio/region_mean": 0.01851259867544286, |
| "entropy": 0.07352668652310967, |
| "epoch": 0.00016, |
| "grad_norm": 1.1360507011413574, |
| "kl": 0.08157501685491297, |
| "learning_rate": 4.2857142857142856e-05, |
| "loss": 0.1258, |
| "step": 16, |
| "step_time": 44.95600875300079 |
| }, |
| { |
| "clip_ratio/high_max": 0.000589622650295496, |
| "clip_ratio/high_mean": 0.000294811325147748, |
| "clip_ratio/low_mean": 0.004054940363857895, |
| "clip_ratio/low_min": 0.0010419560712762177, |
| "clip_ratio/region_mean": 0.004349751689005643, |
| "completions/clipped_ratio": 0.5625, |
| "completions/max_length": 14400.0, |
| "completions/max_terminated_length": 13862.0, |
| "completions/mean_length": 11858.40625, |
| "completions/mean_terminated_length": 8890.857421875, |
| "completions/min_length": 2159.0, |
| "completions/min_terminated_length": 2159.0, |
| "entropy": 0.07483385875821114, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7257509231567383, |
| "kl": 0.05066429602447897, |
| "learning_rate": 4.5714285714285716e-05, |
| "loss": 0.334, |
| "num_tokens": 3712600.0, |
| "reward": -0.43021440505981445, |
| "reward_std": 0.8870621919631958, |
| "rewards/rollout_reward_func/mean": -0.43021440505981445, |
| "rewards/rollout_reward_func/std": 0.8449047803878784, |
| "sampling/importance_sampling_ratio/max": 2.4000132083892822, |
| "sampling/importance_sampling_ratio/mean": 1.002233624458313, |
| "sampling/importance_sampling_ratio/min": 0.1551634818315506, |
| "sampling/sampling_logp_difference/max": 1.8632760047912598, |
| "sampling/sampling_logp_difference/mean": 0.01753704622387886, |
| "step": 17, |
| "step_time": 132.86553408099644 |
| }, |
| { |
| "clip_ratio/high_max": 0.006706475804094225, |
| "clip_ratio/high_mean": 0.003931941610062495, |
| "clip_ratio/low_mean": 0.0171583968331106, |
| "clip_ratio/low_min": 0.010398818471003324, |
| "clip_ratio/region_mean": 0.021090338443173096, |
| "entropy": 0.06936636054888368, |
| "epoch": 0.00018, |
| "grad_norm": 1.771362543106079, |
| "kl": 0.10733210120815784, |
| "learning_rate": 4.8571428571428576e-05, |
| "loss": 0.3285, |
| "step": 18, |
| "step_time": 44.16366357000152 |
| }, |
| { |
| "clip_ratio/high_max": 0.0006443298771046102, |
| "clip_ratio/high_mean": 0.0003221649385523051, |
| "clip_ratio/low_mean": 0.0072666899068281054, |
| "clip_ratio/low_min": 0.0009964113123714924, |
| "clip_ratio/region_mean": 0.00758885481627658, |
| "completions/clipped_ratio": 0.625, |
| "completions/max_length": 14492.0, |
| "completions/max_terminated_length": 10697.0, |
| "completions/mean_length": 11052.375, |
| "completions/mean_terminated_length": 5732.83349609375, |
| "completions/min_length": 1322.0, |
| "completions/min_terminated_length": 1322.0, |
| "entropy": 0.06488334969617426, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.4038617610931396, |
| "kl": 0.08611430588643998, |
| "learning_rate": 5.142857142857143e-05, |
| "loss": 0.5711, |
| "num_tokens": 4093752.0, |
| "reward": -0.4095195233821869, |
| "reward_std": 0.8482590317726135, |
| "rewards/rollout_reward_func/mean": -0.4095195233821869, |
| "rewards/rollout_reward_func/std": 0.8331766724586487, |
| "sampling/importance_sampling_ratio/max": 2.3997995853424072, |
| "sampling/importance_sampling_ratio/mean": 0.9988876581192017, |
| "sampling/importance_sampling_ratio/min": 0.19376768171787262, |
| "sampling/sampling_logp_difference/max": 1.6410953998565674, |
| "sampling/sampling_logp_difference/mean": 0.016171647235751152, |
| "step": 19, |
| "step_time": 128.7513532999983 |
| }, |
| { |
| "clip_ratio/high_max": 0.009648803621530533, |
| "clip_ratio/high_mean": 0.004824401810765266, |
| "clip_ratio/low_mean": 0.014935461309505627, |
| "clip_ratio/low_min": 0.006239572481717914, |
| "clip_ratio/region_mean": 0.019759863178478554, |
| "entropy": 0.06792194582521915, |
| "epoch": 0.0002, |
| "grad_norm": 1.5674505233764648, |
| "kl": 0.07957532908767462, |
| "learning_rate": 5.428571428571428e-05, |
| "loss": 0.5691, |
| "step": 20, |
| "step_time": 44.17661613099699 |
| }, |
| { |
| "clip_ratio/high_max": 0.0010775862028822303, |
| "clip_ratio/high_mean": 0.0005387931014411151, |
| "clip_ratio/low_mean": 0.002510265534510836, |
| "clip_ratio/low_min": 0.0006067961221560836, |
| "clip_ratio/region_mean": 0.003049058635951951, |
| "completions/clipped_ratio": 0.53125, |
| "completions/max_length": 14388.0, |
| "completions/max_terminated_length": 13834.0, |
| "completions/mean_length": 11314.09375, |
| "completions/mean_terminated_length": 8011.1337890625, |
| "completions/min_length": 1359.0, |
| "completions/min_terminated_length": 1359.0, |
| "entropy": 0.06659986882004887, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3160443305969238, |
| "kl": 0.04374655510764569, |
| "learning_rate": 5.714285714285714e-05, |
| "loss": 0.2579, |
| "num_tokens": 4483294.0, |
| "reward": -0.7569143772125244, |
| "reward_std": 0.38748466968536377, |
| "rewards/rollout_reward_func/mean": -0.7569143772125244, |
| "rewards/rollout_reward_func/std": 0.47438445687294006, |
| "sampling/importance_sampling_ratio/max": 2.5243661403656006, |
| "sampling/importance_sampling_ratio/mean": 1.0004966259002686, |
| "sampling/importance_sampling_ratio/min": 0.44545236229896545, |
| "sampling/sampling_logp_difference/max": 0.9259899854660034, |
| "sampling/sampling_logp_difference/mean": 0.011256169527769089, |
| "step": 21, |
| "step_time": 125.32715417199688 |
| }, |
| { |
| "clip_ratio/high_max": 0.007965195982251316, |
| "clip_ratio/high_mean": 0.004264129471266642, |
| "clip_ratio/low_mean": 0.01166028477018699, |
| "clip_ratio/low_min": 0.003762419626582414, |
| "clip_ratio/region_mean": 0.015924414212349802, |
| "entropy": 0.06479066435713321, |
| "epoch": 0.00022, |
| "grad_norm": 0.8557514548301697, |
| "kl": 0.0534607790177688, |
| "learning_rate": 6e-05, |
| "loss": 0.2505, |
| "step": 22, |
| "step_time": 44.035087997997834 |
| }, |
| { |
| "clip_ratio/high_max": 0.0011444397969171405, |
| "clip_ratio/high_mean": 0.0005722198984585702, |
| "clip_ratio/low_mean": 0.002795889973640442, |
| "clip_ratio/low_min": 0.0005040322430431843, |
| "clip_ratio/region_mean": 0.0033681098429951817, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14447.0, |
| "completions/max_terminated_length": 13549.0, |
| "completions/mean_length": 11808.34375, |
| "completions/mean_terminated_length": 7141.36376953125, |
| "completions/min_length": 784.0, |
| "completions/min_terminated_length": 784.0, |
| "entropy": 0.07080131443217397, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2227104902267456, |
| "kl": 0.10710712906438857, |
| "learning_rate": 6.285714285714286e-05, |
| "loss": 0.357, |
| "num_tokens": 4888221.0, |
| "reward": -0.49963146448135376, |
| "reward_std": 0.8173227310180664, |
| "rewards/rollout_reward_func/mean": -0.49963146448135376, |
| "rewards/rollout_reward_func/std": 0.8128207921981812, |
| "sampling/importance_sampling_ratio/max": 1.9062927961349487, |
| "sampling/importance_sampling_ratio/mean": 1.0000483989715576, |
| "sampling/importance_sampling_ratio/min": 0.30003389716148376, |
| "sampling/sampling_logp_difference/max": 1.203859806060791, |
| "sampling/sampling_logp_difference/mean": 0.011820203624665737, |
| "step": 23, |
| "step_time": 134.11656286700236 |
| }, |
| { |
| "clip_ratio/high_max": 0.0063895476050674915, |
| "clip_ratio/high_mean": 0.0031947738025337458, |
| "clip_ratio/low_mean": 0.00737490743631497, |
| "clip_ratio/low_min": 0.0014960957923904061, |
| "clip_ratio/region_mean": 0.010569681267952546, |
| "entropy": 0.07403647690080106, |
| "epoch": 0.00024, |
| "grad_norm": 0.9927102327346802, |
| "kl": 0.07292677101213485, |
| "learning_rate": 6.571428571428571e-05, |
| "loss": 0.3453, |
| "step": 24, |
| "step_time": 44.88870704800138 |
| }, |
| { |
| "clip_ratio/high_max": 0.002575038291979581, |
| "clip_ratio/high_mean": 0.0012875191459897906, |
| "clip_ratio/low_mean": 0.0031292334315367043, |
| "clip_ratio/low_min": 0.0010516082984395325, |
| "clip_ratio/region_mean": 0.004416752519318834, |
| "completions/clipped_ratio": 0.6875, |
| "completions/max_length": 14441.0, |
| "completions/max_terminated_length": 13590.0, |
| "completions/mean_length": 11539.0625, |
| "completions/mean_terminated_length": 5616.10009765625, |
| "completions/min_length": 699.0, |
| "completions/min_terminated_length": 699.0, |
| "entropy": 0.09310276620090008, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.282138466835022, |
| "kl": 0.07473902264609933, |
| "learning_rate": 6.857142857142858e-05, |
| "loss": -0.0119, |
| "num_tokens": 5284116.0, |
| "reward": -0.7307593822479248, |
| "reward_std": 0.4555254876613617, |
| "rewards/rollout_reward_func/mean": -0.7307593822479248, |
| "rewards/rollout_reward_func/std": 0.5767623782157898, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0000531673431396, |
| "sampling/importance_sampling_ratio/min": 0.18337896466255188, |
| "sampling/sampling_logp_difference/max": 1.6962004899978638, |
| "sampling/sampling_logp_difference/mean": 0.014519629999995232, |
| "step": 25, |
| "step_time": 128.11358543400092 |
| }, |
| { |
| "clip_ratio/high_max": 0.006235250388272107, |
| "clip_ratio/high_mean": 0.0041256895638071, |
| "clip_ratio/low_mean": 0.019248695025453344, |
| "clip_ratio/low_min": 0.006461259792558849, |
| "clip_ratio/region_mean": 0.023374384531052783, |
| "entropy": 0.09371388074941933, |
| "epoch": 0.00026, |
| "grad_norm": 1.1407207250595093, |
| "kl": 0.10249935300089419, |
| "learning_rate": 7.142857142857143e-05, |
| "loss": -0.0179, |
| "step": 26, |
| "step_time": 44.78832636999505 |
| }, |
| { |
| "clip_ratio/high_max": 0.001961233268957585, |
| "clip_ratio/high_mean": 0.0009806166344787925, |
| "clip_ratio/low_mean": 0.0014239389565773308, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0024045555910561234, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 14387.0, |
| "completions/max_terminated_length": 13921.0, |
| "completions/mean_length": 10618.1875, |
| "completions/mean_terminated_length": 7001.0, |
| "completions/min_length": 1065.0, |
| "completions/min_terminated_length": 1065.0, |
| "entropy": 0.08063321514055133, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 3.7377917766571045, |
| "kl": 0.46857869252562523, |
| "learning_rate": 7.428571428571429e-05, |
| "loss": 0.0274, |
| "num_tokens": 5650556.0, |
| "reward": -0.4988600015640259, |
| "reward_std": 0.6857542991638184, |
| "rewards/rollout_reward_func/mean": -0.4988600015640259, |
| "rewards/rollout_reward_func/std": 0.8108665347099304, |
| "sampling/importance_sampling_ratio/max": 1.5844895839691162, |
| "sampling/importance_sampling_ratio/mean": 0.9988585710525513, |
| "sampling/importance_sampling_ratio/min": 0.14879053831100464, |
| "sampling/sampling_logp_difference/max": 1.9052157402038574, |
| "sampling/sampling_logp_difference/mean": 0.010426906868815422, |
| "step": 27, |
| "step_time": 117.60450818400022 |
| }, |
| { |
| "clip_ratio/high_max": 0.011021792015526444, |
| "clip_ratio/high_mean": 0.006221123243449256, |
| "clip_ratio/low_mean": 0.007159021100960672, |
| "clip_ratio/low_min": 0.0011492071789689362, |
| "clip_ratio/region_mean": 0.01338014446082525, |
| "entropy": 0.08160684490576386, |
| "epoch": 0.00028, |
| "grad_norm": 1.8652974367141724, |
| "kl": 0.0501517970697023, |
| "learning_rate": 7.714285714285715e-05, |
| "loss": 0.0131, |
| "step": 28, |
| "step_time": 43.69357772600051 |
| }, |
| { |
| "clip_ratio/high_max": 0.005480502266436815, |
| "clip_ratio/high_mean": 0.0027402511332184076, |
| "clip_ratio/low_mean": 0.0009808092145249248, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0037210604059509933, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 14524.0, |
| "completions/max_terminated_length": 13789.0, |
| "completions/mean_length": 10754.5, |
| "completions/mean_terminated_length": 7253.3125, |
| "completions/min_length": 1385.0, |
| "completions/min_terminated_length": 1385.0, |
| "entropy": 0.08766714809462428, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.9512283205986023, |
| "kl": 0.09233958297409117, |
| "learning_rate": 8e-05, |
| "loss": 0.3863, |
| "num_tokens": 6021181.0, |
| "reward": -0.43986791372299194, |
| "reward_std": 0.8378865718841553, |
| "rewards/rollout_reward_func/mean": -0.43986791372299194, |
| "rewards/rollout_reward_func/std": 0.8483731150627136, |
| "sampling/importance_sampling_ratio/max": 1.6999077796936035, |
| "sampling/importance_sampling_ratio/mean": 0.9990843534469604, |
| "sampling/importance_sampling_ratio/min": 0.4758984446525574, |
| "sampling/sampling_logp_difference/max": 0.7425508499145508, |
| "sampling/sampling_logp_difference/mean": 0.010583655908703804, |
| "step": 29, |
| "step_time": 122.50815704600063 |
| }, |
| { |
| "clip_ratio/high_max": 0.011935517424717546, |
| "clip_ratio/high_mean": 0.006662203115411103, |
| "clip_ratio/low_mean": 0.008739105775021017, |
| "clip_ratio/low_min": 0.004148350621107966, |
| "clip_ratio/region_mean": 0.01540130889043212, |
| "entropy": 0.09500867640599608, |
| "epoch": 0.0003, |
| "grad_norm": 0.7339725494384766, |
| "kl": 0.0838697346043773, |
| "learning_rate": 8.285714285714287e-05, |
| "loss": 0.3791, |
| "step": 30, |
| "step_time": 44.16028004800319 |
| }, |
| { |
| "clip_ratio/high_max": 0.002299328800290823, |
| "clip_ratio/high_mean": 0.0011496644001454115, |
| "clip_ratio/low_mean": 0.0013620644458569586, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00251172884600237, |
| "completions/clipped_ratio": 0.71875, |
| "completions/max_length": 14550.0, |
| "completions/max_terminated_length": 13983.0, |
| "completions/mean_length": 12700.71875, |
| "completions/mean_terminated_length": 8789.888671875, |
| "completions/min_length": 3262.0, |
| "completions/min_terminated_length": 3262.0, |
| "entropy": 0.11977054877206683, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.086565613746643, |
| "kl": 0.05921647744253278, |
| "learning_rate": 8.571428571428571e-05, |
| "loss": 0.0947, |
| "num_tokens": 6454382.0, |
| "reward": -0.7849480509757996, |
| "reward_std": 0.38423284888267517, |
| "rewards/rollout_reward_func/mean": -0.7849480509757996, |
| "rewards/rollout_reward_func/std": 0.48005345463752747, |
| "sampling/importance_sampling_ratio/max": 1.8024439811706543, |
| "sampling/importance_sampling_ratio/mean": 1.000292181968689, |
| "sampling/importance_sampling_ratio/min": 0.5424980521202087, |
| "sampling/sampling_logp_difference/max": 0.6115708351135254, |
| "sampling/sampling_logp_difference/mean": 0.01138882152736187, |
| "step": 31, |
| "step_time": 127.06237872199927 |
| }, |
| { |
| "clip_ratio/high_max": 0.012531265849247575, |
| "clip_ratio/high_mean": 0.006265632924623787, |
| "clip_ratio/low_mean": 0.00913364501320757, |
| "clip_ratio/low_min": 0.0016989448922686279, |
| "clip_ratio/region_mean": 0.015399277937831357, |
| "entropy": 0.1310350843705237, |
| "epoch": 0.00032, |
| "grad_norm": 0.8760910034179688, |
| "kl": 0.06428600556682795, |
| "learning_rate": 8.857142857142857e-05, |
| "loss": 0.0848, |
| "step": 32, |
| "step_time": 45.15639938999993 |
| }, |
| { |
| "clip_ratio/high_max": 0.001622461248189211, |
| "clip_ratio/high_mean": 0.0008112306240946054, |
| "clip_ratio/low_mean": 0.0012673057353822514, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.002078536359476857, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 14485.0, |
| "completions/max_terminated_length": 13438.0, |
| "completions/mean_length": 11201.46875, |
| "completions/mean_terminated_length": 8163.75, |
| "completions/min_length": 1137.0, |
| "completions/min_terminated_length": 1137.0, |
| "entropy": 0.11091703665442765, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.763496994972229, |
| "kl": 0.08535079064313322, |
| "learning_rate": 9.142857142857143e-05, |
| "loss": 0.187, |
| "num_tokens": 6839240.0, |
| "reward": -0.606515109539032, |
| "reward_std": 0.644769549369812, |
| "rewards/rollout_reward_func/mean": -0.606515109539032, |
| "rewards/rollout_reward_func/std": 0.7157695293426514, |
| "sampling/importance_sampling_ratio/max": 1.9776629209518433, |
| "sampling/importance_sampling_ratio/mean": 1.0004677772521973, |
| "sampling/importance_sampling_ratio/min": 0.6363774538040161, |
| "sampling/sampling_logp_difference/max": 0.6819157600402832, |
| "sampling/sampling_logp_difference/mean": 0.010047465562820435, |
| "step": 33, |
| "step_time": 122.12401338600102 |
| }, |
| { |
| "clip_ratio/high_max": 0.003852536785416305, |
| "clip_ratio/high_mean": 0.0024514784454368055, |
| "clip_ratio/low_mean": 0.010322274029022083, |
| "clip_ratio/low_min": 0.0054294930887408555, |
| "clip_ratio/region_mean": 0.012773752474458888, |
| "entropy": 0.11235238378867507, |
| "epoch": 0.00034, |
| "grad_norm": 0.9141661524772644, |
| "kl": 0.08446309305145405, |
| "learning_rate": 9.428571428571429e-05, |
| "loss": 0.1825, |
| "step": 34, |
| "step_time": 43.25774885099963 |
| }, |
| { |
| "clip_ratio/high_max": 0.002067690948024392, |
| "clip_ratio/high_mean": 0.001033845474012196, |
| "clip_ratio/low_mean": 0.0013292695221025497, |
| "clip_ratio/low_min": 0.0005208333604969084, |
| "clip_ratio/region_mean": 0.0023631149961147457, |
| "completions/clipped_ratio": 0.625, |
| "completions/max_length": 14504.0, |
| "completions/max_terminated_length": 13750.0, |
| "completions/mean_length": 12662.125, |
| "completions/mean_terminated_length": 10072.6669921875, |
| "completions/min_length": 2136.0, |
| "completions/min_terminated_length": 2136.0, |
| "entropy": 0.13832575432024896, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8794277906417847, |
| "kl": 0.08255716518033296, |
| "learning_rate": 9.714285714285715e-05, |
| "loss": 0.0863, |
| "num_tokens": 7271283.0, |
| "reward": -0.5410826206207275, |
| "reward_std": 0.7768348455429077, |
| "rewards/rollout_reward_func/mean": -0.5410826206207275, |
| "rewards/rollout_reward_func/std": 0.7597488760948181, |
| "sampling/importance_sampling_ratio/max": 1.5629594326019287, |
| "sampling/importance_sampling_ratio/mean": 1.0001167058944702, |
| "sampling/importance_sampling_ratio/min": 0.5588953495025635, |
| "sampling/sampling_logp_difference/max": 0.5817930698394775, |
| "sampling/sampling_logp_difference/mean": 0.009778056293725967, |
| "step": 35, |
| "step_time": 130.91799312700277 |
| }, |
| { |
| "clip_ratio/high_max": 0.011173836537636817, |
| "clip_ratio/high_mean": 0.005586918268818408, |
| "clip_ratio/low_mean": 0.00869341439101845, |
| "clip_ratio/low_min": 0.004238906840328127, |
| "clip_ratio/region_mean": 0.01428033280535601, |
| "entropy": 0.15124066965654492, |
| "epoch": 0.00036, |
| "grad_norm": 0.7353250980377197, |
| "kl": 0.08487725246232003, |
| "learning_rate": 0.0001, |
| "loss": 0.0769, |
| "step": 36, |
| "step_time": 45.46909211699858 |
| }, |
| { |
| "clip_ratio/high_max": 0.0015989625826478004, |
| "clip_ratio/high_mean": 0.0007994812913239002, |
| "clip_ratio/low_mean": 0.0020003379322588444, |
| "clip_ratio/low_min": 0.0005165289039723575, |
| "clip_ratio/region_mean": 0.0027998192235827446, |
| "completions/clipped_ratio": 0.625, |
| "completions/max_length": 14481.0, |
| "completions/max_terminated_length": 13367.0, |
| "completions/mean_length": 11635.15625, |
| "completions/mean_terminated_length": 7297.1669921875, |
| "completions/min_length": 439.0, |
| "completions/min_terminated_length": 439.0, |
| "entropy": 0.13707713410258293, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8795878887176514, |
| "kl": 0.09341590362600982, |
| "learning_rate": 9.918053502751771e-05, |
| "loss": 0.1423, |
| "num_tokens": 7670326.0, |
| "reward": -0.6904894113540649, |
| "reward_std": 0.6018991470336914, |
| "rewards/rollout_reward_func/mean": -0.6904894113540649, |
| "rewards/rollout_reward_func/std": 0.6566587686538696, |
| "sampling/importance_sampling_ratio/max": 1.4667878150939941, |
| "sampling/importance_sampling_ratio/mean": 0.999151349067688, |
| "sampling/importance_sampling_ratio/min": 0.3191526234149933, |
| "sampling/sampling_logp_difference/max": 1.1420859098434448, |
| "sampling/sampling_logp_difference/mean": 0.01049021165817976, |
| "step": 37, |
| "step_time": 128.31693721999363 |
| }, |
| { |
| "clip_ratio/high_max": 0.012058351479936391, |
| "clip_ratio/high_mean": 0.007017562194960192, |
| "clip_ratio/low_mean": 0.010125949047505856, |
| "clip_ratio/low_min": 0.002969218941871077, |
| "clip_ratio/region_mean": 0.01714351132977754, |
| "entropy": 0.13681618124246597, |
| "epoch": 0.00038, |
| "grad_norm": 0.6959795951843262, |
| "kl": 0.09944733651354909, |
| "learning_rate": 9.675795466159755e-05, |
| "loss": 0.13, |
| "step": 38, |
| "step_time": 45.151127613000426 |
| }, |
| { |
| "clip_ratio/high_max": 0.001359933929052204, |
| "clip_ratio/high_mean": 0.000679966964526102, |
| "clip_ratio/low_mean": 0.001585537102073431, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0022655040957033634, |
| "completions/clipped_ratio": 0.59375, |
| "completions/max_length": 14393.0, |
| "completions/max_terminated_length": 13179.0, |
| "completions/mean_length": 11368.53125, |
| "completions/mean_terminated_length": 7292.30810546875, |
| "completions/min_length": 1023.0, |
| "completions/min_terminated_length": 1023.0, |
| "entropy": 0.11355275707319379, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8372141122817993, |
| "kl": 0.12875043111853302, |
| "learning_rate": 9.283813728906054e-05, |
| "loss": -0.0014, |
| "num_tokens": 8061296.0, |
| "reward": -0.6022458076477051, |
| "reward_std": 0.6515352129936218, |
| "rewards/rollout_reward_func/mean": -0.6022458076477051, |
| "rewards/rollout_reward_func/std": 0.7095517516136169, |
| "sampling/importance_sampling_ratio/max": 1.636765480041504, |
| "sampling/importance_sampling_ratio/mean": 0.9989644289016724, |
| "sampling/importance_sampling_ratio/min": 0.47850245237350464, |
| "sampling/sampling_logp_difference/max": 0.7370939254760742, |
| "sampling/sampling_logp_difference/mean": 0.010086959227919579, |
| "step": 39, |
| "step_time": 126.83774206699854 |
| }, |
| { |
| "clip_ratio/high_max": 0.008263030322268605, |
| "clip_ratio/high_mean": 0.004131515161134303, |
| "clip_ratio/low_mean": 0.014172348921420053, |
| "clip_ratio/low_min": 0.0036224325885996222, |
| "clip_ratio/region_mean": 0.018303864169865847, |
| "entropy": 0.10010521637741476, |
| "epoch": 0.0004, |
| "grad_norm": 0.7000178098678589, |
| "kl": 0.14327447954565287, |
| "learning_rate": 8.759239773845718e-05, |
| "loss": -0.0115, |
| "step": 40, |
| "step_time": 44.916688461000376 |
| }, |
| { |
| "clip_ratio/high_max": 0.001846958533860743, |
| "clip_ratio/high_mean": 0.0009234792669303715, |
| "clip_ratio/low_mean": 0.002370692673139274, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0032941719400696456, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14506.0, |
| "completions/max_terminated_length": 13335.0, |
| "completions/mean_length": 11941.9375, |
| "completions/mean_terminated_length": 7638.63671875, |
| "completions/min_length": 1452.0, |
| "completions/min_terminated_length": 1452.0, |
| "entropy": 0.11166324256919324, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0560439825057983, |
| "kl": 0.10882639326155186, |
| "learning_rate": 8.125000000000001e-05, |
| "loss": 0.5418, |
| "num_tokens": 8470213.0, |
| "reward": -0.4550766348838806, |
| "reward_std": 0.7941820621490479, |
| "rewards/rollout_reward_func/mean": -0.4550766348838806, |
| "rewards/rollout_reward_func/std": 0.7887347340583801, |
| "sampling/importance_sampling_ratio/max": 1.8251365423202515, |
| "sampling/importance_sampling_ratio/mean": 1.0004825592041016, |
| "sampling/importance_sampling_ratio/min": 0.5376880168914795, |
| "sampling/sampling_logp_difference/max": 0.6204767227172852, |
| "sampling/sampling_logp_difference/mean": 0.011826084926724434, |
| "step": 41, |
| "step_time": 126.00419519500247 |
| }, |
| { |
| "clip_ratio/high_max": 0.007851478818338364, |
| "clip_ratio/high_mean": 0.003925739409169182, |
| "clip_ratio/low_mean": 0.014391002681804821, |
| "clip_ratio/low_min": 0.005810233065858483, |
| "clip_ratio/region_mean": 0.018316742061870173, |
| "entropy": 0.10689377831295133, |
| "epoch": 0.00042, |
| "grad_norm": 1.3130351305007935, |
| "kl": 0.1170152414124459, |
| "learning_rate": 7.408813728906053e-05, |
| "loss": 0.5353, |
| "step": 42, |
| "step_time": 45.83305146299972 |
| }, |
| { |
| "clip_ratio/high_max": 0.002037170110270381, |
| "clip_ratio/high_mean": 0.0010185850551351905, |
| "clip_ratio/low_mean": 0.0016175404307432473, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0026361254567746073, |
| "completions/clipped_ratio": 0.53125, |
| "completions/max_length": 14370.0, |
| "completions/max_terminated_length": 13214.0, |
| "completions/mean_length": 11265.375, |
| "completions/mean_terminated_length": 7982.400390625, |
| "completions/min_length": 782.0, |
| "completions/min_terminated_length": 782.0, |
| "entropy": 0.09073930280283093, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.7020624876022339, |
| "kl": 0.13664298865478486, |
| "learning_rate": 6.6419817372537e-05, |
| "loss": 0.1856, |
| "num_tokens": 8857416.0, |
| "reward": -0.37660786509513855, |
| "reward_std": 0.8688997030258179, |
| "rewards/rollout_reward_func/mean": -0.37660786509513855, |
| "rewards/rollout_reward_func/std": 0.8781781196594238, |
| "sampling/importance_sampling_ratio/max": 1.9817417860031128, |
| "sampling/importance_sampling_ratio/mean": 1.0001003742218018, |
| "sampling/importance_sampling_ratio/min": 0.4231337010860443, |
| "sampling/sampling_logp_difference/max": 0.8600671291351318, |
| "sampling/sampling_logp_difference/mean": 0.009302067570388317, |
| "step": 43, |
| "step_time": 121.0720823820011 |
| }, |
| { |
| "clip_ratio/high_max": 0.007119149784557521, |
| "clip_ratio/high_mean": 0.0035595748922787607, |
| "clip_ratio/low_mean": 0.01176984136691317, |
| "clip_ratio/low_min": 0.002642020583152771, |
| "clip_ratio/region_mean": 0.01532941620098427, |
| "entropy": 0.08315122197382152, |
| "epoch": 0.00044, |
| "grad_norm": 5.769225597381592, |
| "kl": 0.6573639016132802, |
| "learning_rate": 5.858018262746301e-05, |
| "loss": 0.1936, |
| "step": 44, |
| "step_time": 44.69906600800459 |
| }, |
| { |
| "clip_ratio/high_max": 0.0023281591129489243, |
| "clip_ratio/high_mean": 0.0011640795564744622, |
| "clip_ratio/low_mean": 0.002193144930060953, |
| "clip_ratio/low_min": 0.0005530973430722952, |
| "clip_ratio/region_mean": 0.0033572244574315846, |
| "completions/clipped_ratio": 0.53125, |
| "completions/max_length": 14504.0, |
| "completions/max_terminated_length": 13747.0, |
| "completions/mean_length": 11228.96875, |
| "completions/mean_terminated_length": 7874.400390625, |
| "completions/min_length": 1631.0, |
| "completions/min_terminated_length": 1631.0, |
| "entropy": 0.08939703018404543, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0448198318481445, |
| "kl": 0.11512639001011848, |
| "learning_rate": 5.0911862710939485e-05, |
| "loss": 0.0475, |
| "num_tokens": 9243782.0, |
| "reward": -0.49534285068511963, |
| "reward_std": 0.7042239904403687, |
| "rewards/rollout_reward_func/mean": -0.49534285068511963, |
| "rewards/rollout_reward_func/std": 0.8103145360946655, |
| "sampling/importance_sampling_ratio/max": 1.6891045570373535, |
| "sampling/importance_sampling_ratio/mean": 1.0011885166168213, |
| "sampling/importance_sampling_ratio/min": 0.5372036695480347, |
| "sampling/sampling_logp_difference/max": 0.6213779449462891, |
| "sampling/sampling_logp_difference/mean": 0.0091001121327281, |
| "step": 45, |
| "step_time": 120.37874379399727 |
| }, |
| { |
| "clip_ratio/high_max": 0.008119428472127765, |
| "clip_ratio/high_mean": 0.004059714236063883, |
| "clip_ratio/low_mean": 0.008602545305620879, |
| "clip_ratio/low_min": 0.0037590391584672034, |
| "clip_ratio/region_mean": 0.01266225945437327, |
| "entropy": 0.08823490096256137, |
| "epoch": 0.00046, |
| "grad_norm": 0.8318017721176147, |
| "kl": 0.120926967356354, |
| "learning_rate": 4.375000000000001e-05, |
| "loss": 0.0362, |
| "step": 46, |
| "step_time": 44.49746013800177 |
| }, |
| { |
| "clip_ratio/high_max": 0.003532699483912438, |
| "clip_ratio/high_mean": 0.001766349741956219, |
| "clip_ratio/low_mean": 0.0016052687860792503, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0033716185280354694, |
| "completions/clipped_ratio": 0.65625, |
| "completions/max_length": 14315.0, |
| "completions/max_terminated_length": 13481.0, |
| "completions/mean_length": 11687.5625, |
| "completions/mean_terminated_length": 7038.7275390625, |
| "completions/min_length": 1557.0, |
| "completions/min_terminated_length": 1557.0, |
| "entropy": 0.07309431000612676, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8768713474273682, |
| "kl": 0.08262627921067178, |
| "learning_rate": 3.740760226154283e-05, |
| "loss": 0.3828, |
| "num_tokens": 9644940.0, |
| "reward": -0.4843946695327759, |
| "reward_std": 0.8082762360572815, |
| "rewards/rollout_reward_func/mean": -0.4843946695327759, |
| "rewards/rollout_reward_func/std": 0.8041583895683289, |
| "sampling/importance_sampling_ratio/max": 1.4661790132522583, |
| "sampling/importance_sampling_ratio/mean": 1.000061273574829, |
| "sampling/importance_sampling_ratio/min": 0.3773066997528076, |
| "sampling/sampling_logp_difference/max": 0.9746968746185303, |
| "sampling/sampling_logp_difference/mean": 0.007087397389113903, |
| "step": 47, |
| "step_time": 121.44068199299727 |
| }, |
| { |
| "clip_ratio/high_max": 0.01053907914320007, |
| "clip_ratio/high_mean": 0.005612946202745661, |
| "clip_ratio/low_mean": 0.00564750301418826, |
| "clip_ratio/low_min": 0.0020543700957205147, |
| "clip_ratio/region_mean": 0.01126044918783009, |
| "entropy": 0.07405747356824577, |
| "epoch": 0.00048, |
| "grad_norm": 0.811284601688385, |
| "kl": 0.08346198999788612, |
| "learning_rate": 3.2161862710939476e-05, |
| "loss": 0.374, |
| "step": 48, |
| "step_time": 44.960870664997856 |
| }, |
| { |
| "clip_ratio/high_max": 0.002270299184601754, |
| "clip_ratio/high_mean": 0.001135149592300877, |
| "clip_ratio/low_mean": 0.003153837809804827, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004288987343898043, |
| "completions/clipped_ratio": 0.625, |
| "completions/max_length": 14530.0, |
| "completions/max_terminated_length": 13232.0, |
| "completions/mean_length": 11548.625, |
| "completions/mean_terminated_length": 7039.5, |
| "completions/min_length": 1709.0, |
| "completions/min_terminated_length": 1709.0, |
| "entropy": 0.08768759481608868, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6150716543197632, |
| "kl": 0.3609654267784208, |
| "learning_rate": 2.8242045338402468e-05, |
| "loss": 0.4855, |
| "num_tokens": 10041134.0, |
| "reward": -0.5441310405731201, |
| "reward_std": 0.6545704007148743, |
| "rewards/rollout_reward_func/mean": -0.5441310405731201, |
| "rewards/rollout_reward_func/std": 0.7639203667640686, |
| "sampling/importance_sampling_ratio/max": 1.7008589506149292, |
| "sampling/importance_sampling_ratio/mean": 1.0008265972137451, |
| "sampling/importance_sampling_ratio/min": 0.45911118388175964, |
| "sampling/sampling_logp_difference/max": 0.7784628868103027, |
| "sampling/sampling_logp_difference/mean": 0.010290013626217842, |
| "step": 49, |
| "step_time": 124.03581126300378 |
| }, |
| { |
| "clip_ratio/high_max": 0.004471153952181339, |
| "clip_ratio/high_mean": 0.0022355769760906696, |
| "clip_ratio/low_mean": 0.007084499462507665, |
| "clip_ratio/low_min": 0.0021499572903849185, |
| "clip_ratio/region_mean": 0.009320076409494504, |
| "entropy": 0.08996679773554206, |
| "epoch": 0.0005, |
| "grad_norm": 0.8601014018058777, |
| "kl": 0.24557857040781528, |
| "learning_rate": 2.5819464972482288e-05, |
| "loss": 0.4747, |
| "step": 50, |
| "step_time": 45.77899998500652 |
| }, |
| { |
| "epoch": 0.0005, |
| "eval_clip_ratio/high_max": 0.0, |
| "eval_clip_ratio/high_mean": 0.0, |
| "eval_clip_ratio/low_mean": 0.0, |
| "eval_clip_ratio/low_min": 0.0, |
| "eval_clip_ratio/region_mean": 0.0, |
| "eval_completions/clipped_ratio": 0.60875, |
| "eval_completions/max_length": 13600.67375, |
| "eval_completions/max_terminated_length": 5307.3125, |
| "eval_completions/mean_length": 11798.84625, |
| "eval_completions/mean_terminated_length": 4963.2975, |
| "eval_completions/min_length": 9997.01875, |
| "eval_completions/min_terminated_length": 4619.2825, |
| "eval_entropy": 0.09010161134880036, |
| "eval_frac_reward_zero_std": 1.0, |
| "eval_kl": 0.13719033321132884, |
| "eval_loss": 0.00026262112078256905, |
| "eval_num_tokens": 10041134.0, |
| "eval_reward": -0.47466856505721805, |
| "eval_reward_std": 0.0, |
| "eval_rewards/rollout_reward_func/mean": -0.47466856505721805, |
| "eval_rewards/rollout_reward_func/std": 0.509479589665134, |
| "eval_runtime": 4806.419, |
| "eval_samples_per_second": 0.042, |
| "eval_sampling/importance_sampling_ratio/max": 1.2884997601807118, |
| "eval_sampling/importance_sampling_ratio/mean": 1.000060096681118, |
| "eval_sampling/importance_sampling_ratio/min": 0.7314270212873817, |
| "eval_sampling/sampling_logp_difference/max": 0.38731383727863433, |
| "eval_sampling/sampling_logp_difference/mean": 0.00951688139612088, |
| "eval_steps_per_second": 0.021, |
| "step": 50 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 50, |
| "num_input_tokens_seen": 10041134, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
