| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.45745654162854527, |
| "eval_steps": 500, |
| "global_step": 500, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1080.75, |
| "completions/max_terminated_length": 1080.75, |
| "completions/mean_length": 206.859375, |
| "completions/mean_terminated_length": 206.859375, |
| "completions/min_length": 72.75, |
| "completions/min_terminated_length": 72.75, |
| "entropy": 0.01571042463183403, |
| "epoch": 0.0036596523330283625, |
| "frac_reward_zero_std": 0.6484375, |
| "grad_norm": 0.1166667682172164, |
| "kl": 6.741156056477848e-06, |
| "learning_rate": 1.8181818181818183e-06, |
| "loss": 0.019, |
| "num_tokens": 595592.0, |
| "reward": 2.5668949484825134, |
| "reward_std": 0.6298200786113739, |
| "rewards/reward_fn/mean": 2.5668949484825134, |
| "rewards/reward_fn/std": 0.6298201084136963, |
| "step": 4, |
| "step_time": 260.7198786311783 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1112.5, |
| "completions/max_terminated_length": 1112.5, |
| "completions/mean_length": 207.083984375, |
| "completions/mean_terminated_length": 207.083984375, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.014931437617633492, |
| "epoch": 0.007319304666056725, |
| "frac_reward_zero_std": 0.67578125, |
| "grad_norm": 0.07972757343469568, |
| "kl": 7.538596197065317e-05, |
| "learning_rate": 3.6363636363636366e-06, |
| "loss": 0.0116, |
| "num_tokens": 1191286.0, |
| "reward": 2.6111356019973755, |
| "reward_std": 0.5753327757120132, |
| "rewards/reward_fn/mean": 2.6111356019973755, |
| "rewards/reward_fn/std": 0.5753328055143356, |
| "step": 8, |
| "step_time": 270.5837429687381 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1084.5, |
| "completions/max_terminated_length": 1084.5, |
| "completions/mean_length": 200.9501953125, |
| "completions/mean_terminated_length": 200.9501953125, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.015009274939075112, |
| "epoch": 0.010978956999085087, |
| "frac_reward_zero_std": 0.75390625, |
| "grad_norm": 0.08571915975811008, |
| "kl": 0.0005235518705433151, |
| "learning_rate": 4.999989462079163e-06, |
| "loss": 0.0121, |
| "num_tokens": 1780099.0, |
| "reward": 2.6977345943450928, |
| "reward_std": 0.44414132833480835, |
| "rewards/reward_fn/mean": 2.6977345943450928, |
| "rewards/reward_fn/std": 0.44414133578538895, |
| "step": 12, |
| "step_time": 261.6133712823503 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 919.75, |
| "completions/max_terminated_length": 919.75, |
| "completions/mean_length": 201.7529296875, |
| "completions/mean_terminated_length": 201.7529296875, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.015052395581733435, |
| "epoch": 0.01463860933211345, |
| "frac_reward_zero_std": 0.765625, |
| "grad_norm": 0.06477325550449937, |
| "kl": 0.0006825914342698525, |
| "learning_rate": 4.999736556420954e-06, |
| "loss": 0.018, |
| "num_tokens": 2371954.0, |
| "reward": 2.7144583463668823, |
| "reward_std": 0.42256640642881393, |
| "rewards/reward_fn/mean": 2.7144583463668823, |
| "rewards/reward_fn/std": 0.42256640642881393, |
| "step": 16, |
| "step_time": 214.89796239417046 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1129.75, |
| "completions/max_terminated_length": 1129.75, |
| "completions/mean_length": 207.7060546875, |
| "completions/mean_terminated_length": 207.7060546875, |
| "completions/min_length": 79.25, |
| "completions/min_terminated_length": 79.25, |
| "entropy": 0.013794319529552013, |
| "epoch": 0.018298261665141813, |
| "frac_reward_zero_std": 0.8046875, |
| "grad_norm": 0.07172710130306945, |
| "kl": 0.001074870842785458, |
| "learning_rate": 4.9991464763837844e-06, |
| "loss": 0.0119, |
| "num_tokens": 2971833.0, |
| "reward": 2.7160686254501343, |
| "reward_std": 0.4207172691822052, |
| "rewards/reward_fn/mean": 2.7160686254501343, |
| "rewards/reward_fn/std": 0.4207172840833664, |
| "step": 20, |
| "step_time": 275.1132181570865 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 974.5, |
| "completions/max_terminated_length": 974.5, |
| "completions/mean_length": 207.3681640625, |
| "completions/mean_terminated_length": 207.3681640625, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.014441891689784825, |
| "epoch": 0.021957913998170174, |
| "frac_reward_zero_std": 0.734375, |
| "grad_norm": 0.08257356486413421, |
| "kl": 0.0034124660814995877, |
| "learning_rate": 4.99821930155999e-06, |
| "loss": -0.0014, |
| "num_tokens": 3569890.0, |
| "reward": 2.6976758241653442, |
| "reward_std": 0.4469035342335701, |
| "rewards/reward_fn/mean": 2.6976758241653442, |
| "rewards/reward_fn/std": 0.4469035491347313, |
| "step": 24, |
| "step_time": 229.607499552425 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 985.5, |
| "completions/max_terminated_length": 985.5, |
| "completions/mean_length": 187.828125, |
| "completions/mean_terminated_length": 187.828125, |
| "completions/min_length": 75.0, |
| "completions/min_terminated_length": 75.0, |
| "entropy": 0.01451652345713228, |
| "epoch": 0.025617566331198535, |
| "frac_reward_zero_std": 0.7734375, |
| "grad_norm": 0.1178542444151902, |
| "kl": 0.011817638587672263, |
| "learning_rate": 4.996955157010586e-06, |
| "loss": 0.0146, |
| "num_tokens": 4150210.0, |
| "reward": 2.7241913080215454, |
| "reward_std": 0.42349882423877716, |
| "rewards/reward_fn/mean": 2.7241913080215454, |
| "rewards/reward_fn/std": 0.42349884659051895, |
| "step": 28, |
| "step_time": 236.96254792809486 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 964.0, |
| "completions/max_terminated_length": 964.0, |
| "completions/mean_length": 180.63671875, |
| "completions/mean_terminated_length": 180.63671875, |
| "completions/min_length": 78.25, |
| "completions/min_terminated_length": 78.25, |
| "entropy": 0.014597299275919795, |
| "epoch": 0.0292772186642269, |
| "frac_reward_zero_std": 0.80078125, |
| "grad_norm": 0.191243062362765, |
| "kl": 0.03357400168897584, |
| "learning_rate": 4.995354213248404e-06, |
| "loss": 0.002, |
| "num_tokens": 4728162.0, |
| "reward": 2.7361614108085632, |
| "reward_std": 0.40531516820192337, |
| "rewards/reward_fn/mean": 2.7361614108085632, |
| "rewards/reward_fn/std": 0.40531518310308456, |
| "step": 32, |
| "step_time": 227.9491332215257 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1252.5, |
| "completions/max_terminated_length": 1252.5, |
| "completions/mean_length": 198.6748046875, |
| "completions/mean_terminated_length": 198.6748046875, |
| "completions/min_length": 77.25, |
| "completions/min_terminated_length": 77.25, |
| "entropy": 0.014511767774820328, |
| "epoch": 0.03293687099725526, |
| "frac_reward_zero_std": 0.765625, |
| "grad_norm": 1.2528338790788236, |
| "kl": 0.02709985862020403, |
| "learning_rate": 4.9934166862150855e-06, |
| "loss": -0.0043, |
| "num_tokens": 5325577.0, |
| "reward": 2.7323471903800964, |
| "reward_std": 0.4196270480751991, |
| "rewards/reward_fn/mean": 2.7323471903800964, |
| "rewards/reward_fn/std": 0.4196270480751991, |
| "step": 36, |
| "step_time": 311.6690187435597 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1034.5, |
| "completions/max_terminated_length": 1034.5, |
| "completions/mean_length": 190.1337890625, |
| "completions/mean_terminated_length": 190.1337890625, |
| "completions/min_length": 73.5, |
| "completions/min_terminated_length": 73.5, |
| "entropy": 0.013505439972504973, |
| "epoch": 0.036596523330283626, |
| "frac_reward_zero_std": 0.80078125, |
| "grad_norm": 0.08402394326328486, |
| "kl": 0.020394871709868312, |
| "learning_rate": 4.991142837251961e-06, |
| "loss": -0.0138, |
| "num_tokens": 5913470.0, |
| "reward": 2.760911762714386, |
| "reward_std": 0.3911248818039894, |
| "rewards/reward_fn/mean": 2.760911762714386, |
| "rewards/reward_fn/std": 0.3911249041557312, |
| "step": 40, |
| "step_time": 245.01106920232996 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 986.5, |
| "completions/max_terminated_length": 986.5, |
| "completions/mean_length": 197.076171875, |
| "completions/mean_terminated_length": 197.076171875, |
| "completions/min_length": 73.25, |
| "completions/min_terminated_length": 73.25, |
| "entropy": 0.013148889294825494, |
| "epoch": 0.04025617566331199, |
| "frac_reward_zero_std": 0.78515625, |
| "grad_norm": 0.07640337001807694, |
| "kl": 0.02207769750384614, |
| "learning_rate": 4.988532973064793e-06, |
| "loss": -0.016, |
| "num_tokens": 6509836.0, |
| "reward": 2.7576756477355957, |
| "reward_std": 0.40184224396944046, |
| "rewards/reward_fn/mean": 2.7576756477355957, |
| "rewards/reward_fn/std": 0.40184228122234344, |
| "step": 44, |
| "step_time": 232.39970854343846 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1041.0, |
| "completions/max_terminated_length": 1041.0, |
| "completions/mean_length": 201.873046875, |
| "completions/mean_terminated_length": 201.873046875, |
| "completions/min_length": 83.75, |
| "completions/min_terminated_length": 83.75, |
| "entropy": 0.012962576001882553, |
| "epoch": 0.04391582799634035, |
| "frac_reward_zero_std": 0.79296875, |
| "grad_norm": 0.09402744049682173, |
| "kl": 0.015623664075974375, |
| "learning_rate": 4.985587445682414e-06, |
| "loss": -0.0103, |
| "num_tokens": 7112210.0, |
| "reward": 2.761009693145752, |
| "reward_std": 0.3792940303683281, |
| "rewards/reward_fn/mean": 2.761009693145752, |
| "rewards/reward_fn/std": 0.3792940229177475, |
| "step": 48, |
| "step_time": 248.20540295541286 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 850.0, |
| "completions/max_terminated_length": 850.0, |
| "completions/mean_length": 180.35546875, |
| "completions/mean_terminated_length": 180.35546875, |
| "completions/min_length": 70.25, |
| "completions/min_terminated_length": 70.25, |
| "entropy": 0.012066802359186113, |
| "epoch": 0.04757548032936871, |
| "frac_reward_zero_std": 0.80078125, |
| "grad_norm": 0.05676690796949128, |
| "kl": 0.02138438867405057, |
| "learning_rate": 4.982306652409236e-06, |
| "loss": -0.0238, |
| "num_tokens": 7691458.0, |
| "reward": 2.7505001425743103, |
| "reward_std": 0.4136660769581795, |
| "rewards/reward_fn/mean": 2.7505001425743103, |
| "rewards/reward_fn/std": 0.4136660695075989, |
| "step": 52, |
| "step_time": 195.69965509325266 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1014.5, |
| "completions/max_terminated_length": 1014.5, |
| "completions/mean_length": 203.6640625, |
| "completions/mean_terminated_length": 203.6640625, |
| "completions/min_length": 81.25, |
| "completions/min_terminated_length": 81.25, |
| "entropy": 0.012359246786218137, |
| "epoch": 0.05123513266239707, |
| "frac_reward_zero_std": 0.81640625, |
| "grad_norm": 0.1820463265067798, |
| "kl": 0.018120157648809254, |
| "learning_rate": 4.978691035771666e-06, |
| "loss": -0.0187, |
| "num_tokens": 8298814.0, |
| "reward": 2.7694973945617676, |
| "reward_std": 0.40114688873291016, |
| "rewards/reward_fn/mean": 2.7694973945617676, |
| "rewards/reward_fn/std": 0.40114689618349075, |
| "step": 56, |
| "step_time": 240.87798455543816 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1240.75, |
| "completions/max_terminated_length": 1240.75, |
| "completions/mean_length": 209.6689453125, |
| "completions/mean_terminated_length": 209.6689453125, |
| "completions/min_length": 78.0, |
| "completions/min_terminated_length": 78.0, |
| "entropy": 0.012856585904955864, |
| "epoch": 0.05489478499542543, |
| "frac_reward_zero_std": 0.828125, |
| "grad_norm": 0.09204206328450969, |
| "kl": 0.036135767586529255, |
| "learning_rate": 4.9747410834584134e-06, |
| "loss": 0.0008, |
| "num_tokens": 8915947.0, |
| "reward": 2.779680848121643, |
| "reward_std": 0.37843772768974304, |
| "rewards/reward_fn/mean": 2.779680848121643, |
| "rewards/reward_fn/std": 0.37843775749206543, |
| "step": 60, |
| "step_time": 308.8453072188422 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1042.75, |
| "completions/max_terminated_length": 1042.75, |
| "completions/mean_length": 212.9931640625, |
| "completions/mean_terminated_length": 212.9931640625, |
| "completions/min_length": 79.25, |
| "completions/min_terminated_length": 79.25, |
| "entropy": 0.012349803757388145, |
| "epoch": 0.0585544373284538, |
| "frac_reward_zero_std": 0.80859375, |
| "grad_norm": 0.10624366951375605, |
| "kl": 0.037413959740661085, |
| "learning_rate": 4.970457328254707e-06, |
| "loss": -0.0226, |
| "num_tokens": 9536872.0, |
| "reward": 2.7904839515686035, |
| "reward_std": 0.3760991021990776, |
| "rewards/reward_fn/mean": 2.7904839515686035, |
| "rewards/reward_fn/std": 0.3760991171002388, |
| "step": 64, |
| "step_time": 250.47715630708262 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1020.25, |
| "completions/max_terminated_length": 1020.25, |
| "completions/mean_length": 220.515625, |
| "completions/mean_terminated_length": 220.515625, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.01191511406796053, |
| "epoch": 0.06221408966148216, |
| "frac_reward_zero_std": 0.8125, |
| "grad_norm": 0.062386160477754214, |
| "kl": 0.035652039339765906, |
| "learning_rate": 4.965840347970436e-06, |
| "loss": -0.0145, |
| "num_tokens": 10165112.0, |
| "reward": 2.7982013821601868, |
| "reward_std": 0.34512612223625183, |
| "rewards/reward_fn/mean": 2.7982013821601868, |
| "rewards/reward_fn/std": 0.34512611478567123, |
| "step": 68, |
| "step_time": 243.45551919797435 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1169.5, |
| "completions/max_terminated_length": 1169.5, |
| "completions/mean_length": 216.1025390625, |
| "completions/mean_terminated_length": 216.1025390625, |
| "completions/min_length": 66.5, |
| "completions/min_terminated_length": 66.5, |
| "entropy": 0.012621318746823817, |
| "epoch": 0.06587374199451052, |
| "frac_reward_zero_std": 0.8046875, |
| "grad_norm": 0.06076991641355838, |
| "kl": 0.036882418557070196, |
| "learning_rate": 4.96089076536221e-06, |
| "loss": -0.0137, |
| "num_tokens": 10794093.0, |
| "reward": 2.8089590668678284, |
| "reward_std": 0.35318899899721146, |
| "rewards/reward_fn/mean": 2.8089590668678284, |
| "rewards/reward_fn/std": 0.35318901389837265, |
| "step": 72, |
| "step_time": 288.5881523680873 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1078.25, |
| "completions/max_terminated_length": 1078.25, |
| "completions/mean_length": 219.3916015625, |
| "completions/mean_terminated_length": 219.3916015625, |
| "completions/min_length": 77.75, |
| "completions/min_terminated_length": 77.75, |
| "entropy": 0.012647980998735875, |
| "epoch": 0.06953339432753888, |
| "frac_reward_zero_std": 0.77734375, |
| "grad_norm": 0.10469311451857824, |
| "kl": 0.04040143545717001, |
| "learning_rate": 4.9556092480493584e-06, |
| "loss": -0.021, |
| "num_tokens": 11425154.0, |
| "reward": 2.818051338195801, |
| "reward_std": 0.3466835618019104, |
| "rewards/reward_fn/mean": 2.818051338195801, |
| "rewards/reward_fn/std": 0.3466835543513298, |
| "step": 76, |
| "step_time": 260.33293359261006 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1126.0, |
| "completions/max_terminated_length": 1126.0, |
| "completions/mean_length": 227.54296875, |
| "completions/mean_terminated_length": 227.54296875, |
| "completions/min_length": 81.25, |
| "completions/min_terminated_length": 81.25, |
| "entropy": 0.014289415907114744, |
| "epoch": 0.07319304666056725, |
| "frac_reward_zero_std": 0.80859375, |
| "grad_norm": 0.07665055865966315, |
| "kl": 0.03189453040249646, |
| "learning_rate": 4.949996508423877e-06, |
| "loss": -0.0087, |
| "num_tokens": 12067202.0, |
| "reward": 2.7729015350341797, |
| "reward_std": 0.3783550187945366, |
| "rewards/reward_fn/mean": 2.7729015350341797, |
| "rewards/reward_fn/std": 0.3783550336956978, |
| "step": 80, |
| "step_time": 275.7156659611501 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1096.0, |
| "completions/max_terminated_length": 1096.0, |
| "completions/mean_length": 234.634765625, |
| "completions/mean_terminated_length": 234.634765625, |
| "completions/min_length": 81.25, |
| "completions/min_terminated_length": 81.25, |
| "entropy": 0.016412670316640288, |
| "epoch": 0.0768526989935956, |
| "frac_reward_zero_std": 0.79296875, |
| "grad_norm": 0.0802093480911299, |
| "kl": 0.028840772691182792, |
| "learning_rate": 4.944053303554343e-06, |
| "loss": -0.0118, |
| "num_tokens": 12721888.0, |
| "reward": 2.807867646217346, |
| "reward_std": 0.36286504566669464, |
| "rewards/reward_fn/mean": 2.807867646217346, |
| "rewards/reward_fn/std": 0.36286503076553345, |
| "step": 84, |
| "step_time": 266.78033225191757 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1047.25, |
| "completions/max_terminated_length": 1047.25, |
| "completions/mean_length": 241.9208984375, |
| "completions/mean_terminated_length": 241.9208984375, |
| "completions/min_length": 75.0, |
| "completions/min_terminated_length": 75.0, |
| "entropy": 0.016632919432595372, |
| "epoch": 0.08051235132662397, |
| "frac_reward_zero_std": 0.79296875, |
| "grad_norm": 0.08807158873251, |
| "kl": 0.02757147583179176, |
| "learning_rate": 4.937780435083797e-06, |
| "loss": -0.0035, |
| "num_tokens": 13387983.0, |
| "reward": 2.826618194580078, |
| "reward_std": 0.35499829053878784, |
| "rewards/reward_fn/mean": 2.826618194580078, |
| "rewards/reward_fn/std": 0.35499831289052963, |
| "step": 88, |
| "step_time": 253.21191494399682 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1000.0, |
| "completions/max_terminated_length": 1000.0, |
| "completions/mean_length": 234.90234375, |
| "completions/mean_terminated_length": 234.90234375, |
| "completions/min_length": 82.5, |
| "completions/min_terminated_length": 82.5, |
| "entropy": 0.015686047787312418, |
| "epoch": 0.08417200365965233, |
| "frac_reward_zero_std": 0.77734375, |
| "grad_norm": 0.0902657319512717, |
| "kl": 0.02856659993994981, |
| "learning_rate": 4.931178749121612e-06, |
| "loss": -0.0036, |
| "num_tokens": 14046663.0, |
| "reward": 2.7893744707107544, |
| "reward_std": 0.367780365049839, |
| "rewards/reward_fn/mean": 2.7893744707107544, |
| "rewards/reward_fn/std": 0.3677803725004196, |
| "step": 92, |
| "step_time": 241.46138817537576 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1010.25, |
| "completions/max_terminated_length": 1010.25, |
| "completions/mean_length": 233.6337890625, |
| "completions/mean_terminated_length": 233.6337890625, |
| "completions/min_length": 76.5, |
| "completions/min_terminated_length": 76.5, |
| "entropy": 0.016080917441286147, |
| "epoch": 0.0878316559926807, |
| "frac_reward_zero_std": 0.80859375, |
| "grad_norm": 0.10844162037915826, |
| "kl": 0.030640183249488473, |
| "learning_rate": 4.924249136129368e-06, |
| "loss": -0.0016, |
| "num_tokens": 14703728.0, |
| "reward": 2.821086883544922, |
| "reward_std": 0.34664493054151535, |
| "rewards/reward_fn/mean": 2.821086883544922, |
| "rewards/reward_fn/std": 0.34664495289325714, |
| "step": 96, |
| "step_time": 242.6293911980465 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.001953125, |
| "completions/max_length": 1350.5, |
| "completions/max_terminated_length": 1142.5, |
| "completions/mean_length": 241.9013671875, |
| "completions/mean_terminated_length": 239.37954330444336, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.01606400089804083, |
| "epoch": 0.09149130832570906, |
| "frac_reward_zero_std": 0.80078125, |
| "grad_norm": 0.10026115815134104, |
| "kl": 0.028314362629316747, |
| "learning_rate": 4.916992530800743e-06, |
| "loss": 0.0104, |
| "num_tokens": 15372987.0, |
| "reward": 2.8225176334381104, |
| "reward_std": 0.35643360018730164, |
| "rewards/reward_fn/mean": 2.8225176334381104, |
| "rewards/reward_fn/std": 0.35643360763788223, |
| "step": 100, |
| "step_time": 353.8205208936706 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1125.75, |
| "completions/max_terminated_length": 1125.75, |
| "completions/mean_length": 240.900390625, |
| "completions/mean_terminated_length": 240.900390625, |
| "completions/min_length": 78.5, |
| "completions/min_terminated_length": 78.5, |
| "entropy": 0.01435602194396779, |
| "epoch": 0.09515096065873742, |
| "frac_reward_zero_std": 0.78515625, |
| "grad_norm": 0.07084664638416241, |
| "kl": 0.0261101343203336, |
| "learning_rate": 4.909409911935439e-06, |
| "loss": 0.0157, |
| "num_tokens": 16040689.0, |
| "reward": 2.8153278827667236, |
| "reward_std": 0.3514478802680969, |
| "rewards/reward_fn/mean": 2.8153278827667236, |
| "rewards/reward_fn/std": 0.3514478877186775, |
| "step": 104, |
| "step_time": 281.9691889616661 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1062.0, |
| "completions/max_terminated_length": 1062.0, |
| "completions/mean_length": 220.57421875, |
| "completions/mean_terminated_length": 220.57421875, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.013337482989300042, |
| "epoch": 0.09881061299176579, |
| "frac_reward_zero_std": 0.8515625, |
| "grad_norm": 0.04353003522988331, |
| "kl": 0.02664439135696739, |
| "learning_rate": 4.901502302307155e-06, |
| "loss": 0.0034, |
| "num_tokens": 16687393.0, |
| "reward": 2.836458206176758, |
| "reward_std": 0.3151797950267792, |
| "rewards/reward_fn/mean": 2.836458206176758, |
| "rewards/reward_fn/std": 0.3151797950267792, |
| "step": 108, |
| "step_time": 262.62998094735667 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1111.5, |
| "completions/max_terminated_length": 1111.5, |
| "completions/mean_length": 232.6630859375, |
| "completions/mean_terminated_length": 232.6630859375, |
| "completions/min_length": 81.25, |
| "completions/min_terminated_length": 81.25, |
| "entropy": 0.013577717880252749, |
| "epoch": 0.10247026532479414, |
| "frac_reward_zero_std": 0.8359375, |
| "grad_norm": 0.06257959896127956, |
| "kl": 0.02790737326722592, |
| "learning_rate": 4.893270768525636e-06, |
| "loss": 0.005, |
| "num_tokens": 17352068.0, |
| "reward": 2.817973017692566, |
| "reward_std": 0.3653535544872284, |
| "rewards/reward_fn/mean": 2.817973017692566, |
| "rewards/reward_fn/std": 0.3653535917401314, |
| "step": 112, |
| "step_time": 279.3403219631873 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1052.0, |
| "completions/max_terminated_length": 1052.0, |
| "completions/mean_length": 226.76953125, |
| "completions/mean_terminated_length": 226.76953125, |
| "completions/min_length": 81.5, |
| "completions/min_terminated_length": 81.5, |
| "entropy": 0.014110724616330117, |
| "epoch": 0.10612991765782251, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.10101762024902548, |
| "kl": 0.03716003953013569, |
| "learning_rate": 4.884716420892796e-06, |
| "loss": 0.0149, |
| "num_tokens": 18014020.0, |
| "reward": 2.846973419189453, |
| "reward_std": 0.33806994557380676, |
| "rewards/reward_fn/mean": 2.846973419189453, |
| "rewards/reward_fn/std": 0.33806995302438736, |
| "step": 116, |
| "step_time": 262.1372203491628 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1113.25, |
| "completions/max_terminated_length": 1113.25, |
| "completions/mean_length": 232.7587890625, |
| "completions/mean_terminated_length": 232.7587890625, |
| "completions/min_length": 82.75, |
| "completions/min_terminated_length": 82.75, |
| "entropy": 0.01280687196413055, |
| "epoch": 0.10978956999085086, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.10160550077935328, |
| "kl": 0.04039534099865705, |
| "learning_rate": 4.875840413252968e-06, |
| "loss": 0.0196, |
| "num_tokens": 18683765.0, |
| "reward": 2.8628095388412476, |
| "reward_std": 0.3213796243071556, |
| "rewards/reward_fn/mean": 2.8628095388412476, |
| "rewards/reward_fn/std": 0.3213796466588974, |
| "step": 120, |
| "step_time": 283.8032908006571 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 950.75, |
| "completions/max_terminated_length": 950.75, |
| "completions/mean_length": 221.6962890625, |
| "completions/mean_terminated_length": 221.6962890625, |
| "completions/min_length": 80.5, |
| "completions/min_terminated_length": 80.5, |
| "entropy": 0.013175143743865192, |
| "epoch": 0.11344922232387923, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.06846652654639172, |
| "kl": 0.0442041486967355, |
| "learning_rate": 4.866643942837256e-06, |
| "loss": 0.0068, |
| "num_tokens": 19340998.0, |
| "reward": 2.863952100276947, |
| "reward_std": 0.33952105045318604, |
| "rewards/reward_fn/mean": 2.863952100276947, |
| "rewards/reward_fn/std": 0.3395210728049278, |
| "step": 124, |
| "step_time": 235.00603474816307 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1126.0, |
| "completions/max_terminated_length": 1126.0, |
| "completions/mean_length": 222.55078125, |
| "completions/mean_terminated_length": 222.55078125, |
| "completions/min_length": 79.5, |
| "completions/min_terminated_length": 79.5, |
| "entropy": 0.012510814412962645, |
| "epoch": 0.1171088746569076, |
| "frac_reward_zero_std": 0.89453125, |
| "grad_norm": 0.07639946031491095, |
| "kl": 0.04450422874651849, |
| "learning_rate": 4.857128250102057e-06, |
| "loss": 0.0065, |
| "num_tokens": 20002494.0, |
| "reward": 2.8543471097946167, |
| "reward_std": 0.3549715727567673, |
| "rewards/reward_fn/mean": 2.8543471097946167, |
| "rewards/reward_fn/std": 0.35497158020734787, |
| "step": 128, |
| "step_time": 287.90223784698173 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1197.25, |
| "completions/max_terminated_length": 1142.25, |
| "completions/mean_length": 230.3544921875, |
| "completions/mean_terminated_length": 229.08981704711914, |
| "completions/min_length": 81.5, |
| "completions/min_terminated_length": 81.5, |
| "entropy": 0.0119518170831725, |
| "epoch": 0.12076852698993595, |
| "frac_reward_zero_std": 0.8984375, |
| "grad_norm": 0.06323385395208185, |
| "kl": 0.04492187639698386, |
| "learning_rate": 4.8472946185617395e-06, |
| "loss": 0.0015, |
| "num_tokens": 20671653.0, |
| "reward": 2.8591416478157043, |
| "reward_std": 0.33486203104257584, |
| "rewards/reward_fn/mean": 2.8591416478157043, |
| "rewards/reward_fn/std": 0.33486203849315643, |
| "step": 132, |
| "step_time": 315.46063051139936 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1066.75, |
| "completions/max_terminated_length": 1066.75, |
| "completions/mean_length": 223.9208984375, |
| "completions/mean_terminated_length": 223.9208984375, |
| "completions/min_length": 76.75, |
| "completions/min_terminated_length": 76.75, |
| "entropy": 0.01208365970524028, |
| "epoch": 0.12442817932296432, |
| "frac_reward_zero_std": 0.921875, |
| "grad_norm": 0.08297054131204595, |
| "kl": 0.049826212925836444, |
| "learning_rate": 4.83714437461552e-06, |
| "loss": 0.0009, |
| "num_tokens": 21335048.0, |
| "reward": 2.857163429260254, |
| "reward_std": 0.3398366719484329, |
| "rewards/reward_fn/mean": 2.857163429260254, |
| "rewards/reward_fn/std": 0.3398367017507553, |
| "step": 136, |
| "step_time": 271.9807884255424 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1044.75, |
| "completions/max_terminated_length": 1044.75, |
| "completions/mean_length": 225.634765625, |
| "completions/mean_terminated_length": 225.634765625, |
| "completions/min_length": 80.0, |
| "completions/min_terminated_length": 80.0, |
| "entropy": 0.011399393086321652, |
| "epoch": 0.1280878316559927, |
| "frac_reward_zero_std": 0.9453125, |
| "grad_norm": 0.05599980204371085, |
| "kl": 0.049895928939804435, |
| "learning_rate": 4.82667888736855e-06, |
| "loss": 0.0063, |
| "num_tokens": 22007114.0, |
| "reward": 2.8718236088752747, |
| "reward_std": 0.31254760548472404, |
| "rewards/reward_fn/mean": 2.8718236088752747, |
| "rewards/reward_fn/std": 0.31254761293530464, |
| "step": 140, |
| "step_time": 270.8902273066342 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1078.25, |
| "completions/max_terminated_length": 1078.25, |
| "completions/mean_length": 228.3505859375, |
| "completions/mean_terminated_length": 228.3505859375, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.012115511810407043, |
| "epoch": 0.13174748398902103, |
| "frac_reward_zero_std": 0.93359375, |
| "grad_norm": 0.06313071464948433, |
| "kl": 0.04882367001846433, |
| "learning_rate": 4.815899568447249e-06, |
| "loss": 0.002, |
| "num_tokens": 22683225.0, |
| "reward": 2.8702903985977173, |
| "reward_std": 0.3183404803276062, |
| "rewards/reward_fn/mean": 2.8702903985977173, |
| "rewards/reward_fn/std": 0.3183404952287674, |
| "step": 144, |
| "step_time": 279.12108469195664 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1074.0, |
| "completions/max_terminated_length": 1074.0, |
| "completions/mean_length": 232.51953125, |
| "completions/mean_terminated_length": 232.51953125, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.012113740318454802, |
| "epoch": 0.1354071363220494, |
| "frac_reward_zero_std": 0.9296875, |
| "grad_norm": 0.03658778754859278, |
| "kl": 0.04669306240975857, |
| "learning_rate": 4.804807871808896e-06, |
| "loss": 0.0021, |
| "num_tokens": 23365881.0, |
| "reward": 2.8644264340400696, |
| "reward_std": 0.33891405910253525, |
| "rewards/reward_fn/mean": 2.8644264340400696, |
| "rewards/reward_fn/std": 0.33891407400369644, |
| "step": 148, |
| "step_time": 280.8253190875985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1042.75, |
| "completions/max_terminated_length": 1042.75, |
| "completions/mean_length": 229.6142578125, |
| "completions/mean_terminated_length": 229.6142578125, |
| "completions/min_length": 66.5, |
| "completions/min_terminated_length": 66.5, |
| "entropy": 0.012918102380353957, |
| "epoch": 0.13906678865507777, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.02909046433282949, |
| "kl": 0.05268579628318548, |
| "learning_rate": 4.793405293545515e-06, |
| "loss": 0.005, |
| "num_tokens": 24046030.0, |
| "reward": 2.840142250061035, |
| "reward_std": 0.3678673878312111, |
| "rewards/reward_fn/mean": 2.840142250061035, |
| "rewards/reward_fn/std": 0.3678673878312111, |
| "step": 152, |
| "step_time": 270.6633171187714 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1053.25, |
| "completions/max_terminated_length": 1053.25, |
| "completions/mean_length": 228.44140625, |
| "completions/mean_terminated_length": 228.44140625, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.012696940742898732, |
| "epoch": 0.14272644098810613, |
| "frac_reward_zero_std": 0.91796875, |
| "grad_norm": 0.048905338186676375, |
| "kl": 0.048741934821009636, |
| "learning_rate": 4.781693371682078e-06, |
| "loss": 0.0075, |
| "num_tokens": 24728038.0, |
| "reward": 2.8578240871429443, |
| "reward_std": 0.34266316145658493, |
| "rewards/reward_fn/mean": 2.8578240871429443, |
| "rewards/reward_fn/std": 0.3426631763577461, |
| "step": 156, |
| "step_time": 278.5923395641148 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1050.0, |
| "completions/max_terminated_length": 1050.0, |
| "completions/mean_length": 241.5390625, |
| "completions/mean_terminated_length": 241.5390625, |
| "completions/min_length": 79.25, |
| "completions/min_terminated_length": 79.25, |
| "entropy": 0.014673987170681357, |
| "epoch": 0.1463860933211345, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.09602525853563185, |
| "kl": 0.050844733603298664, |
| "learning_rate": 4.769673685969047e-06, |
| "loss": 0.002, |
| "num_tokens": 25425806.0, |
| "reward": 2.8598700165748596, |
| "reward_std": 0.31299157440662384, |
| "rewards/reward_fn/mean": 2.8598700165748596, |
| "rewards/reward_fn/std": 0.31299156695604324, |
| "step": 160, |
| "step_time": 275.66814939584583 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1011.25, |
| "completions/max_terminated_length": 1011.25, |
| "completions/mean_length": 216.0478515625, |
| "completions/mean_terminated_length": 216.0478515625, |
| "completions/min_length": 77.25, |
| "completions/min_terminated_length": 77.25, |
| "entropy": 0.01529375184327364, |
| "epoch": 0.15004574565416284, |
| "frac_reward_zero_std": 0.859375, |
| "grad_norm": 0.06508558042156037, |
| "kl": 0.05107123241759837, |
| "learning_rate": 4.757347857669292e-06, |
| "loss": -0.0049, |
| "num_tokens": 26097111.0, |
| "reward": 2.8294225335121155, |
| "reward_std": 0.3605535253882408, |
| "rewards/reward_fn/mean": 2.8294225335121155, |
| "rewards/reward_fn/std": 0.3605535477399826, |
| "step": 164, |
| "step_time": 265.74749380582944 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1110.5, |
| "completions/max_terminated_length": 1110.5, |
| "completions/mean_length": 250.458984375, |
| "completions/mean_terminated_length": 250.458984375, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.01382788352202624, |
| "epoch": 0.1537053979871912, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.06539406663732159, |
| "kl": 0.050654102582484484, |
| "learning_rate": 4.744717549339412e-06, |
| "loss": -0.0012, |
| "num_tokens": 26813365.0, |
| "reward": 2.8500934839248657, |
| "reward_std": 0.3334427922964096, |
| "rewards/reward_fn/mean": 2.8500934839248657, |
| "rewards/reward_fn/std": 0.333442784845829, |
| "step": 168, |
| "step_time": 298.08681116625667 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1034.5, |
| "completions/max_terminated_length": 1034.5, |
| "completions/mean_length": 230.283203125, |
| "completions/mean_terminated_length": 230.283203125, |
| "completions/min_length": 78.75, |
| "completions/min_terminated_length": 78.75, |
| "entropy": 0.013774615188594908, |
| "epoch": 0.15736505032021958, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.22897135637733165, |
| "kl": 0.05037340638227761, |
| "learning_rate": 4.731784464605474e-06, |
| "loss": 0.0021, |
| "num_tokens": 27510167.0, |
| "reward": 2.8380470871925354, |
| "reward_std": 0.34361911565065384, |
| "rewards/reward_fn/mean": 2.8380470871925354, |
| "rewards/reward_fn/std": 0.34361913055181503, |
| "step": 172, |
| "step_time": 277.122442452237 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1013.0, |
| "completions/max_terminated_length": 1013.0, |
| "completions/mean_length": 241.236328125, |
| "completions/mean_terminated_length": 241.236328125, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.01372558658476919, |
| "epoch": 0.16102470265324795, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.05383569167271158, |
| "kl": 0.052041274728253484, |
| "learning_rate": 4.718550347933234e-06, |
| "loss": -0.0009, |
| "num_tokens": 28219421.0, |
| "reward": 2.8658467531204224, |
| "reward_std": 0.3258620947599411, |
| "rewards/reward_fn/mean": 2.8658467531204224, |
| "rewards/reward_fn/std": 0.3258620873093605, |
| "step": 176, |
| "step_time": 270.5161182801239 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1019.5, |
| "completions/max_terminated_length": 1019.5, |
| "completions/mean_length": 238.8955078125, |
| "completions/mean_terminated_length": 238.8955078125, |
| "completions/min_length": 80.75, |
| "completions/min_terminated_length": 80.75, |
| "entropy": 0.014220306125935167, |
| "epoch": 0.16468435498627632, |
| "frac_reward_zero_std": 0.84765625, |
| "grad_norm": 0.0919566209583686, |
| "kl": 0.06164557579904795, |
| "learning_rate": 4.705016984392825e-06, |
| "loss": 0.0033, |
| "num_tokens": 28927074.0, |
| "reward": 2.81077641248703, |
| "reward_std": 0.3736064210534096, |
| "rewards/reward_fn/mean": 2.81077641248703, |
| "rewards/reward_fn/std": 0.3736064285039902, |
| "step": 180, |
| "step_time": 278.1804432417266 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1054.5, |
| "completions/max_terminated_length": 1054.5, |
| "completions/mean_length": 245.607421875, |
| "completions/mean_terminated_length": 245.607421875, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.014155362150631845, |
| "epoch": 0.16834400731930466, |
| "frac_reward_zero_std": 0.90625, |
| "grad_norm": 0.0717895655009332, |
| "kl": 0.06093443278223276, |
| "learning_rate": 4.691186199417985e-06, |
| "loss": 0.0029, |
| "num_tokens": 29646528.0, |
| "reward": 2.8060117959976196, |
| "reward_std": 0.4102167412638664, |
| "rewards/reward_fn/mean": 2.8060117959976196, |
| "rewards/reward_fn/std": 0.4102167412638664, |
| "step": 184, |
| "step_time": 287.0040536648594 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1221.25, |
| "completions/max_terminated_length": 1221.25, |
| "completions/mean_length": 256.5146484375, |
| "completions/mean_terminated_length": 256.5146484375, |
| "completions/min_length": 77.5, |
| "completions/min_terminated_length": 77.5, |
| "entropy": 0.014777194592170417, |
| "epoch": 0.17200365965233302, |
| "frac_reward_zero_std": 0.82421875, |
| "grad_norm": 0.10308365762658511, |
| "kl": 0.06632018205709755, |
| "learning_rate": 4.677059858559835e-06, |
| "loss": 0.0031, |
| "num_tokens": 30378431.0, |
| "reward": 2.7752703428268433, |
| "reward_std": 0.46188151836395264, |
| "rewards/reward_fn/mean": 2.7752703428268433, |
| "rewards/reward_fn/std": 0.461881548166275, |
| "step": 188, |
| "step_time": 340.33803110197186 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1087.0, |
| "completions/max_terminated_length": 1087.0, |
| "completions/mean_length": 234.787109375, |
| "completions/mean_terminated_length": 234.787109375, |
| "completions/min_length": 80.75, |
| "completions/min_terminated_length": 80.75, |
| "entropy": 0.013827059709001333, |
| "epoch": 0.1756633119853614, |
| "frac_reward_zero_std": 0.84375, |
| "grad_norm": 0.07237213346872136, |
| "kl": 0.0638300587888807, |
| "learning_rate": 4.662639867235244e-06, |
| "loss": 0.0031, |
| "num_tokens": 31089909.0, |
| "reward": 2.8027281761169434, |
| "reward_std": 0.3978399187326431, |
| "rewards/reward_fn/mean": 2.8027281761169434, |
| "rewards/reward_fn/std": 0.3978399336338043, |
| "step": 192, |
| "step_time": 300.94366448652 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1064.0, |
| "completions/max_terminated_length": 1064.0, |
| "completions/mean_length": 255.17578125, |
| "completions/mean_terminated_length": 255.17578125, |
| "completions/min_length": 79.5, |
| "completions/min_terminated_length": 79.5, |
| "entropy": 0.01407104026293382, |
| "epoch": 0.17932296431838976, |
| "frac_reward_zero_std": 0.875, |
| "grad_norm": 0.06382882251270793, |
| "kl": 0.06381136830896139, |
| "learning_rate": 4.647928170469824e-06, |
| "loss": 0.0043, |
| "num_tokens": 31830217.0, |
| "reward": 2.815064787864685, |
| "reward_std": 0.3995492458343506, |
| "rewards/reward_fn/mean": 2.815064787864685, |
| "rewards/reward_fn/std": 0.3995492681860924, |
| "step": 196, |
| "step_time": 295.87483162106946 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1163.25, |
| "completions/max_terminated_length": 1163.25, |
| "completions/mean_length": 243.5771484375, |
| "completions/mean_terminated_length": 243.5771484375, |
| "completions/min_length": 81.5, |
| "completions/min_terminated_length": 81.5, |
| "entropy": 0.014106316433753818, |
| "epoch": 0.18298261665141813, |
| "frac_reward_zero_std": 0.8515625, |
| "grad_norm": 0.06695057253435484, |
| "kl": 0.0665284029673785, |
| "learning_rate": 4.632926752635569e-06, |
| "loss": 0.0076, |
| "num_tokens": 32558312.0, |
| "reward": 2.7023602724075317, |
| "reward_std": 0.5262805372476578, |
| "rewards/reward_fn/mean": 2.7023602724075317, |
| "rewards/reward_fn/std": 0.5262805446982384, |
| "step": 200, |
| "step_time": 328.4555319393985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1141.25, |
| "completions/max_terminated_length": 1141.25, |
| "completions/mean_length": 248.427734375, |
| "completions/mean_terminated_length": 248.427734375, |
| "completions/min_length": 81.25, |
| "completions/min_terminated_length": 81.25, |
| "entropy": 0.013146597717422992, |
| "epoch": 0.18664226898444647, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.07539641236005702, |
| "kl": 0.06348171643912792, |
| "learning_rate": 4.617637637183204e-06, |
| "loss": 0.0034, |
| "num_tokens": 33291082.0, |
| "reward": 2.761439323425293, |
| "reward_std": 0.4976746663451195, |
| "rewards/reward_fn/mean": 2.761439323425293, |
| "rewards/reward_fn/std": 0.4976746737957001, |
| "step": 204, |
| "step_time": 319.8018886325881 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1132.5, |
| "completions/max_terminated_length": 1132.5, |
| "completions/mean_length": 240.9658203125, |
| "completions/mean_terminated_length": 240.9658203125, |
| "completions/min_length": 78.25, |
| "completions/min_terminated_length": 78.25, |
| "entropy": 0.015186345437541604, |
| "epoch": 0.19030192131747484, |
| "frac_reward_zero_std": 0.88671875, |
| "grad_norm": 0.03993634279516113, |
| "kl": 0.06433808198198676, |
| "learning_rate": 4.602062886369244e-06, |
| "loss": 0.0099, |
| "num_tokens": 34020671.0, |
| "reward": 2.793409049510956, |
| "reward_std": 0.4567374736070633, |
| "rewards/reward_fn/mean": 2.793409049510956, |
| "rewards/reward_fn/std": 0.4567374810576439, |
| "step": 208, |
| "step_time": 323.2913993848488 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1083.0, |
| "completions/max_terminated_length": 1083.0, |
| "completions/mean_length": 244.0146484375, |
| "completions/mean_terminated_length": 244.0146484375, |
| "completions/min_length": 77.75, |
| "completions/min_terminated_length": 77.75, |
| "entropy": 0.014897112385369837, |
| "epoch": 0.1939615736505032, |
| "frac_reward_zero_std": 0.93359375, |
| "grad_norm": 0.03717160672790563, |
| "kl": 0.06326445587910712, |
| "learning_rate": 4.58620460097784e-06, |
| "loss": 0.0018, |
| "num_tokens": 34753822.0, |
| "reward": 2.8207574486732483, |
| "reward_std": 0.3912634551525116, |
| "rewards/reward_fn/mean": 2.8207574486732483, |
| "rewards/reward_fn/std": 0.391263484954834, |
| "step": 212, |
| "step_time": 309.24909047270194 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1077.0, |
| "completions/max_terminated_length": 1077.0, |
| "completions/mean_length": 258.7001953125, |
| "completions/mean_terminated_length": 258.7001953125, |
| "completions/min_length": 77.25, |
| "completions/min_terminated_length": 77.25, |
| "entropy": 0.014759315876290202, |
| "epoch": 0.19762122598353157, |
| "frac_reward_zero_std": 0.890625, |
| "grad_norm": 0.07081269541976444, |
| "kl": 0.059495773166418076, |
| "learning_rate": 4.5700649200374095e-06, |
| "loss": 0.0029, |
| "num_tokens": 35501763.0, |
| "reward": 2.809674561023712, |
| "reward_std": 0.3931515961885452, |
| "rewards/reward_fn/mean": 2.809674561023712, |
| "rewards/reward_fn/std": 0.3931516334414482, |
| "step": 216, |
| "step_time": 301.3579012993723 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1171.0, |
| "completions/max_terminated_length": 1040.0, |
| "completions/mean_length": 244.5625, |
| "completions/mean_terminated_length": 243.29501342773438, |
| "completions/min_length": 81.0, |
| "completions/min_terminated_length": 81.0, |
| "entropy": 0.015227734867949039, |
| "epoch": 0.2012808783165599, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.0591513019608395, |
| "kl": 0.06103493459522724, |
| "learning_rate": 4.553646020532114e-06, |
| "loss": 0.0096, |
| "num_tokens": 36239575.0, |
| "reward": 2.8282867074012756, |
| "reward_std": 0.36736876517534256, |
| "rewards/reward_fn/mean": 2.8282867074012756, |
| "rewards/reward_fn/std": 0.36736876517534256, |
| "step": 220, |
| "step_time": 342.628715894185 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1115.25, |
| "completions/max_terminated_length": 1115.25, |
| "completions/mean_length": 250.5625, |
| "completions/mean_terminated_length": 250.5625, |
| "completions/min_length": 77.25, |
| "completions/min_terminated_length": 77.25, |
| "entropy": 0.014696637750603259, |
| "epoch": 0.20494053064958828, |
| "frac_reward_zero_std": 0.8984375, |
| "grad_norm": 0.056132077039738944, |
| "kl": 0.05931313335895538, |
| "learning_rate": 4.536950117108226e-06, |
| "loss": 0.0078, |
| "num_tokens": 36990911.0, |
| "reward": 2.8466833233833313, |
| "reward_std": 0.3461478427052498, |
| "rewards/reward_fn/mean": 2.8466833233833313, |
| "rewards/reward_fn/std": 0.3461478427052498, |
| "step": 224, |
| "step_time": 327.5807393779978 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1158.25, |
| "completions/max_terminated_length": 1158.25, |
| "completions/mean_length": 252.7353515625, |
| "completions/mean_terminated_length": 252.7353515625, |
| "completions/min_length": 77.75, |
| "completions/min_terminated_length": 77.75, |
| "entropy": 0.014584530203137547, |
| "epoch": 0.20860018298261665, |
| "frac_reward_zero_std": 0.828125, |
| "grad_norm": 0.0836067486087614, |
| "kl": 0.05628983583301306, |
| "learning_rate": 4.519979461775404e-06, |
| "loss": 0.0138, |
| "num_tokens": 37745484.0, |
| "reward": 2.784464180469513, |
| "reward_std": 0.42813974618911743, |
| "rewards/reward_fn/mean": 2.784464180469513, |
| "rewards/reward_fn/std": 0.4281397685408592, |
| "step": 228, |
| "step_time": 338.8130832114257 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1122.0, |
| "completions/max_terminated_length": 1122.0, |
| "completions/mean_length": 246.970703125, |
| "completions/mean_terminated_length": 246.970703125, |
| "completions/min_length": 81.0, |
| "completions/min_terminated_length": 81.0, |
| "entropy": 0.013940769073087722, |
| "epoch": 0.21225983531564502, |
| "frac_reward_zero_std": 0.84765625, |
| "grad_norm": 0.06107697088317581, |
| "kl": 0.06036040303297341, |
| "learning_rate": 4.502736343602934e-06, |
| "loss": 0.015, |
| "num_tokens": 38494170.0, |
| "reward": 2.8251479864120483, |
| "reward_std": 0.37282292544841766, |
| "rewards/reward_fn/mean": 2.8251479864120483, |
| "rewards/reward_fn/std": 0.37282292544841766, |
| "step": 232, |
| "step_time": 328.3553645936772 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1226.0, |
| "completions/max_terminated_length": 1116.0, |
| "completions/mean_length": 263.27734375, |
| "completions/mean_terminated_length": 262.0295715332031, |
| "completions/min_length": 78.25, |
| "completions/min_terminated_length": 78.25, |
| "entropy": 0.01664691447513178, |
| "epoch": 0.21591948764867339, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.07936710594166071, |
| "kl": 0.057622806169092655, |
| "learning_rate": 4.485223088410967e-06, |
| "loss": 0.0085, |
| "num_tokens": 39264842.0, |
| "reward": 2.824228823184967, |
| "reward_std": 0.37058399617671967, |
| "rewards/reward_fn/mean": 2.824228823184967, |
| "rewards/reward_fn/std": 0.37058401107788086, |
| "step": 236, |
| "step_time": 371.6977700777352 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1304.0, |
| "completions/max_terminated_length": 1304.0, |
| "completions/mean_length": 265.8642578125, |
| "completions/mean_terminated_length": 265.8642578125, |
| "completions/min_length": 78.5, |
| "completions/min_terminated_length": 78.5, |
| "entropy": 0.02004530024714768, |
| "epoch": 0.21957913998170173, |
| "frac_reward_zero_std": 0.828125, |
| "grad_norm": 0.05728495119096045, |
| "kl": 0.05499382968991995, |
| "learning_rate": 4.4674420584568105e-06, |
| "loss": 0.0167, |
| "num_tokens": 40041699.0, |
| "reward": 2.815194547176361, |
| "reward_std": 0.3625386580824852, |
| "rewards/reward_fn/mean": 2.815194547176361, |
| "rewards/reward_fn/std": 0.3625386580824852, |
| "step": 240, |
| "step_time": 398.28707833588123 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1201.5, |
| "completions/max_terminated_length": 1065.5, |
| "completions/mean_length": 245.466796875, |
| "completions/mean_terminated_length": 244.22003173828125, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.018354313855525106, |
| "epoch": 0.2232387923147301, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.06205789770558449, |
| "kl": 0.0634293460752815, |
| "learning_rate": 4.449395652116291e-06, |
| "loss": 0.0253, |
| "num_tokens": 40798509.0, |
| "reward": 2.81777161359787, |
| "reward_std": 0.3663923665881157, |
| "rewards/reward_fn/mean": 2.81777161359787, |
| "rewards/reward_fn/std": 0.3663923814892769, |
| "step": 244, |
| "step_time": 360.5024783038534 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 920.5, |
| "completions/max_terminated_length": 920.5, |
| "completions/mean_length": 231.9951171875, |
| "completions/mean_terminated_length": 231.9951171875, |
| "completions/min_length": 82.75, |
| "completions/min_terminated_length": 82.75, |
| "entropy": 0.017063919513020664, |
| "epoch": 0.22689844464775846, |
| "frac_reward_zero_std": 0.91015625, |
| "grad_norm": 0.07078082856668842, |
| "kl": 0.07647754997015, |
| "learning_rate": 4.431086303560256e-06, |
| "loss": 0.0167, |
| "num_tokens": 41546504.0, |
| "reward": 2.7994924187660217, |
| "reward_std": 0.40477121621370316, |
| "rewards/reward_fn/mean": 2.7994924187660217, |
| "rewards/reward_fn/std": 0.40477124601602554, |
| "step": 248, |
| "step_time": 268.8032052577473 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1013.25, |
| "completions/max_terminated_length": 1013.25, |
| "completions/mean_length": 233.9052734375, |
| "completions/mean_terminated_length": 233.9052734375, |
| "completions/min_length": 82.75, |
| "completions/min_terminated_length": 82.75, |
| "entropy": 0.017987663624808192, |
| "epoch": 0.23055809698078683, |
| "frac_reward_zero_std": 0.8671875, |
| "grad_norm": 0.1563436187383896, |
| "kl": 0.08293303567916155, |
| "learning_rate": 4.412516482426241e-06, |
| "loss": 0.0093, |
| "num_tokens": 42303207.0, |
| "reward": 2.8115214109420776, |
| "reward_std": 0.3961891457438469, |
| "rewards/reward_fn/mean": 2.8115214109420776, |
| "rewards/reward_fn/std": 0.3961891755461693, |
| "step": 252, |
| "step_time": 305.94378135027364 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 976.75, |
| "completions/max_terminated_length": 976.75, |
| "completions/mean_length": 241.396484375, |
| "completions/mean_terminated_length": 241.396484375, |
| "completions/min_length": 71.25, |
| "completions/min_terminated_length": 71.25, |
| "entropy": 0.01701750111533329, |
| "epoch": 0.2342177493138152, |
| "frac_reward_zero_std": 0.890625, |
| "grad_norm": 0.07228795401138102, |
| "kl": 0.07505286997184157, |
| "learning_rate": 4.393688693485355e-06, |
| "loss": 0.0043, |
| "num_tokens": 43068741.0, |
| "reward": 2.8061317801475525, |
| "reward_std": 0.39476943761110306, |
| "rewards/reward_fn/mean": 2.8061317801475525, |
| "rewards/reward_fn/std": 0.39476945251226425, |
| "step": 256, |
| "step_time": 290.7374283745885 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1056.75, |
| "completions/max_terminated_length": 1056.75, |
| "completions/mean_length": 239.0703125, |
| "completions/mean_terminated_length": 239.0703125, |
| "completions/min_length": 81.5, |
| "completions/min_terminated_length": 81.5, |
| "entropy": 0.016208724409807473, |
| "epoch": 0.23787740164684354, |
| "frac_reward_zero_std": 0.88671875, |
| "grad_norm": 0.06165748716249506, |
| "kl": 0.07173356646671891, |
| "learning_rate": 4.374605476304431e-06, |
| "loss": 0.0088, |
| "num_tokens": 43830085.0, |
| "reward": 2.839516520500183, |
| "reward_std": 0.34048842638731003, |
| "rewards/reward_fn/mean": 2.839516520500183, |
| "rewards/reward_fn/std": 0.3404884412884712, |
| "step": 260, |
| "step_time": 314.90353200305253 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1060.5, |
| "completions/max_terminated_length": 1060.5, |
| "completions/mean_length": 238.794921875, |
| "completions/mean_terminated_length": 238.794921875, |
| "completions/min_length": 80.75, |
| "completions/min_terminated_length": 80.75, |
| "entropy": 0.015883819665759802, |
| "epoch": 0.2415370539798719, |
| "frac_reward_zero_std": 0.95703125, |
| "grad_norm": 0.015264229481008371, |
| "kl": 0.06516523798927665, |
| "learning_rate": 4.355269404903469e-06, |
| "loss": 0.0026, |
| "num_tokens": 44596127.0, |
| "reward": 2.820503354072571, |
| "reward_std": 0.3636850267648697, |
| "rewards/reward_fn/mean": 2.820503354072571, |
| "rewards/reward_fn/std": 0.3636850491166115, |
| "step": 264, |
| "step_time": 324.4773340257816 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1088.5, |
| "completions/max_terminated_length": 1088.5, |
| "completions/mean_length": 235.8466796875, |
| "completions/mean_terminated_length": 235.8466796875, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.015809061413165182, |
| "epoch": 0.24519670631290028, |
| "frac_reward_zero_std": 0.9453125, |
| "grad_norm": 0.02256531674816499, |
| "kl": 0.06406322331167758, |
| "learning_rate": 4.335683087408456e-06, |
| "loss": 0.0065, |
| "num_tokens": 45358850.0, |
| "reward": 2.824212431907654, |
| "reward_std": 0.38692566752433777, |
| "rewards/reward_fn/mean": 2.824212431907654, |
| "rewards/reward_fn/std": 0.38692569732666016, |
| "step": 268, |
| "step_time": 333.7911497359164 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1086.75, |
| "completions/max_terminated_length": 1086.75, |
| "completions/mean_length": 246.55078125, |
| "completions/mean_terminated_length": 246.55078125, |
| "completions/min_length": 78.0, |
| "completions/min_terminated_length": 78.0, |
| "entropy": 0.015495246276259422, |
| "epoch": 0.24885635864592864, |
| "frac_reward_zero_std": 0.9375, |
| "grad_norm": 0.04421854218025354, |
| "kl": 0.06460057967342436, |
| "learning_rate": 4.315849165699562e-06, |
| "loss": 0.0077, |
| "num_tokens": 46132358.0, |
| "reward": 2.8535637259483337, |
| "reward_std": 0.34523719549179077, |
| "rewards/reward_fn/mean": 2.8535637259483337, |
| "rewards/reward_fn/std": 0.34523719549179077, |
| "step": 272, |
| "step_time": 333.43558596400544 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1054.0, |
| "completions/max_terminated_length": 1054.0, |
| "completions/mean_length": 241.197265625, |
| "completions/mean_terminated_length": 241.197265625, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.015473014034796506, |
| "epoch": 0.252516010978957, |
| "frac_reward_zero_std": 0.96484375, |
| "grad_norm": 0.013480684856473399, |
| "kl": 0.06405930104665458, |
| "learning_rate": 4.295770315054792e-06, |
| "loss": 0.0035, |
| "num_tokens": 46907528.0, |
| "reward": 2.851532220840454, |
| "reward_std": 0.33368974179029465, |
| "rewards/reward_fn/mean": 2.851532220840454, |
| "rewards/reward_fn/std": 0.33368974924087524, |
| "step": 276, |
| "step_time": 325.99443143745884 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1059.5, |
| "completions/max_terminated_length": 1059.5, |
| "completions/mean_length": 240.9228515625, |
| "completions/mean_terminated_length": 240.9228515625, |
| "completions/min_length": 80.75, |
| "completions/min_terminated_length": 80.75, |
| "entropy": 0.01620938949054107, |
| "epoch": 0.2561756633119854, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.0384191494606653, |
| "kl": 0.06601658696308732, |
| "learning_rate": 4.275449243789141e-06, |
| "loss": 0.0087, |
| "num_tokens": 47687061.0, |
| "reward": 2.8018744587898254, |
| "reward_std": 0.42315705120563507, |
| "rewards/reward_fn/mean": 2.8018744587898254, |
| "rewards/reward_fn/std": 0.42315705865621567, |
| "step": 280, |
| "step_time": 334.94401907455176 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1138.75, |
| "completions/max_terminated_length": 1138.75, |
| "completions/mean_length": 245.0146484375, |
| "completions/mean_terminated_length": 245.0146484375, |
| "completions/min_length": 77.75, |
| "completions/min_terminated_length": 77.75, |
| "entropy": 0.01545735343825072, |
| "epoch": 0.2598353156450137, |
| "frac_reward_zero_std": 0.91015625, |
| "grad_norm": 0.043103819404576645, |
| "kl": 0.06110293813981116, |
| "learning_rate": 4.254888692889283e-06, |
| "loss": 0.0063, |
| "num_tokens": 48471708.0, |
| "reward": 2.8103264570236206, |
| "reward_std": 0.40437929332256317, |
| "rewards/reward_fn/mean": 2.8103264570236206, |
| "rewards/reward_fn/std": 0.40437931567430496, |
| "step": 284, |
| "step_time": 357.6546063958667 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1228.25, |
| "completions/max_terminated_length": 1228.25, |
| "completions/mean_length": 256.615234375, |
| "completions/mean_terminated_length": 256.615234375, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.01589210593374446, |
| "epoch": 0.26349496797804206, |
| "frac_reward_zero_std": 0.9453125, |
| "grad_norm": 0.03047478394577322, |
| "kl": 0.0623739012517035, |
| "learning_rate": 4.234091435643848e-06, |
| "loss": 0.0038, |
| "num_tokens": 49267282.0, |
| "reward": 2.813932716846466, |
| "reward_std": 0.37685296684503555, |
| "rewards/reward_fn/mean": 2.813932716846466, |
| "rewards/reward_fn/std": 0.37685298919677734, |
| "step": 288, |
| "step_time": 393.72843163786456 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1007.0, |
| "completions/max_terminated_length": 1007.0, |
| "completions/mean_length": 247.5576171875, |
| "completions/mean_terminated_length": 247.5576171875, |
| "completions/min_length": 82.75, |
| "completions/min_terminated_length": 82.75, |
| "entropy": 0.01576699677389115, |
| "epoch": 0.26715462031107046, |
| "frac_reward_zero_std": 0.9296875, |
| "grad_norm": 0.03088838040322047, |
| "kl": 0.06363737187348306, |
| "learning_rate": 4.213060277269364e-06, |
| "loss": 0.0055, |
| "num_tokens": 50061077.0, |
| "reward": 2.8067115545272827, |
| "reward_std": 0.4056583568453789, |
| "rewards/reward_fn/mean": 2.8067115545272827, |
| "rewards/reward_fn/std": 0.4056583642959595, |
| "step": 292, |
| "step_time": 317.5303096845746 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1094.75, |
| "completions/max_terminated_length": 1094.75, |
| "completions/mean_length": 250.1669921875, |
| "completions/mean_terminated_length": 250.1669921875, |
| "completions/min_length": 78.0, |
| "completions/min_terminated_length": 78.0, |
| "entropy": 0.01639867457561195, |
| "epoch": 0.2708142726440988, |
| "frac_reward_zero_std": 0.921875, |
| "grad_norm": 0.041079727814531536, |
| "kl": 0.05844363057985902, |
| "learning_rate": 4.191798054531868e-06, |
| "loss": 0.0086, |
| "num_tokens": 50857108.0, |
| "reward": 2.836637794971466, |
| "reward_std": 0.3569115251302719, |
| "rewards/reward_fn/mean": 2.836637794971466, |
| "rewards/reward_fn/std": 0.3569115325808525, |
| "step": 296, |
| "step_time": 351.88712799642235 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1052.25, |
| "completions/max_terminated_length": 1052.25, |
| "completions/mean_length": 244.1083984375, |
| "completions/mean_terminated_length": 244.1083984375, |
| "completions/min_length": 80.75, |
| "completions/min_terminated_length": 80.75, |
| "entropy": 0.01703868998447433, |
| "epoch": 0.2744739249771272, |
| "frac_reward_zero_std": 0.9296875, |
| "grad_norm": 0.024799921261213508, |
| "kl": 0.055512375896796584, |
| "learning_rate": 4.170307635364272e-06, |
| "loss": 0.0052, |
| "num_tokens": 51646531.0, |
| "reward": 2.802725374698639, |
| "reward_std": 0.39312127232551575, |
| "rewards/reward_fn/mean": 2.802725374698639, |
| "rewards/reward_fn/std": 0.39312130957841873, |
| "step": 300, |
| "step_time": 332.61469624424353 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1048.75, |
| "completions/max_terminated_length": 1048.75, |
| "completions/mean_length": 243.427734375, |
| "completions/mean_terminated_length": 243.427734375, |
| "completions/min_length": 80.5, |
| "completions/min_terminated_length": 80.5, |
| "entropy": 0.01662707404466346, |
| "epoch": 0.27813357731015553, |
| "frac_reward_zero_std": 0.9375, |
| "grad_norm": 0.026354343662826192, |
| "kl": 0.05272407038137317, |
| "learning_rate": 4.148591918479531e-06, |
| "loss": 0.0038, |
| "num_tokens": 52444925.0, |
| "reward": 2.820052146911621, |
| "reward_std": 0.3816464841365814, |
| "rewards/reward_fn/mean": 2.820052146911621, |
| "rewards/reward_fn/std": 0.3816464990377426, |
| "step": 304, |
| "step_time": 338.50240997457877 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1215.75, |
| "completions/max_terminated_length": 1215.75, |
| "completions/mean_length": 257.8525390625, |
| "completions/mean_terminated_length": 257.8525390625, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.016469117137603462, |
| "epoch": 0.2817932296431839, |
| "frac_reward_zero_std": 0.8984375, |
| "grad_norm": 0.0575717215341458, |
| "kl": 0.05287824175320566, |
| "learning_rate": 4.126653832979647e-06, |
| "loss": 0.013, |
| "num_tokens": 53260738.0, |
| "reward": 2.821549654006958, |
| "reward_std": 0.38622572273015976, |
| "rewards/reward_fn/mean": 2.821549654006958, |
| "rewards/reward_fn/std": 0.38622572273015976, |
| "step": 308, |
| "step_time": 403.6054828152992 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1117.25, |
| "completions/max_terminated_length": 1117.25, |
| "completions/mean_length": 255.22265625, |
| "completions/mean_terminated_length": 255.22265625, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.016801700461655855, |
| "epoch": 0.28545288197621227, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.05338408530793631, |
| "kl": 0.052849456667900085, |
| "learning_rate": 4.104496337960583e-06, |
| "loss": 0.0091, |
| "num_tokens": 54075526.0, |
| "reward": 2.7818004488945007, |
| "reward_std": 0.4278130531311035, |
| "rewards/reward_fn/mean": 2.7818004488945007, |
| "rewards/reward_fn/std": 0.4278130829334259, |
| "step": 312, |
| "step_time": 366.00835085008293 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1110.75, |
| "completions/max_terminated_length": 1110.75, |
| "completions/mean_length": 248.7255859375, |
| "completions/mean_terminated_length": 248.7255859375, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.018975069804582745, |
| "epoch": 0.2891125343092406, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.1664088784287396, |
| "kl": 0.05820453749038279, |
| "learning_rate": 4.08212242211313e-06, |
| "loss": 0.0133, |
| "num_tokens": 54889625.0, |
| "reward": 2.7843167185783386, |
| "reward_std": 0.41212283819913864, |
| "rewards/reward_fn/mean": 2.7843167185783386, |
| "rewards/reward_fn/std": 0.41212283819913864, |
| "step": 316, |
| "step_time": 365.95128867262974 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1123.75, |
| "completions/max_terminated_length": 1123.75, |
| "completions/mean_length": 254.044921875, |
| "completions/mean_terminated_length": 254.044921875, |
| "completions/min_length": 80.0, |
| "completions/min_terminated_length": 80.0, |
| "entropy": 0.019178961927536875, |
| "epoch": 0.292772186642269, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.04633976278879389, |
| "kl": 0.05333703476935625, |
| "learning_rate": 4.059535103319777e-06, |
| "loss": 0.0052, |
| "num_tokens": 55711011.0, |
| "reward": 2.7893587946891785, |
| "reward_std": 0.4591749906539917, |
| "rewards/reward_fn/mean": 2.7893587946891785, |
| "rewards/reward_fn/std": 0.4591749981045723, |
| "step": 320, |
| "step_time": 375.5183221241459 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1257.0, |
| "completions/max_terminated_length": 1257.0, |
| "completions/mean_length": 255.3662109375, |
| "completions/mean_terminated_length": 255.3662109375, |
| "completions/min_length": 76.5, |
| "completions/min_terminated_length": 76.5, |
| "entropy": 0.01964702148688957, |
| "epoch": 0.29643183897529735, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.07176312284666939, |
| "kl": 0.0537005455698818, |
| "learning_rate": 4.036737428247648e-06, |
| "loss": 0.0124, |
| "num_tokens": 56535790.0, |
| "reward": 2.75516414642334, |
| "reward_std": 0.47900988161563873, |
| "rewards/reward_fn/mean": 2.75516414642334, |
| "rewards/reward_fn/std": 0.4790099114179611, |
| "step": 324, |
| "step_time": 421.30054724914953 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1111.25, |
| "completions/max_terminated_length": 1111.25, |
| "completions/mean_length": 259.1708984375, |
| "completions/mean_terminated_length": 259.1708984375, |
| "completions/min_length": 84.25, |
| "completions/min_terminated_length": 84.25, |
| "entropy": 0.01968019304331392, |
| "epoch": 0.3000914913083257, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.04273386196110684, |
| "kl": 0.05436060787178576, |
| "learning_rate": 4.013732471937557e-06, |
| "loss": 0.0113, |
| "num_tokens": 57362565.0, |
| "reward": 2.7828410267829895, |
| "reward_std": 0.4619806557893753, |
| "rewards/reward_fn/mean": 2.7828410267829895, |
| "rewards/reward_fn/std": 0.4619806706905365, |
| "step": 328, |
| "step_time": 370.9271548213437 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1077.75, |
| "completions/max_terminated_length": 1077.75, |
| "completions/mean_length": 248.6767578125, |
| "completions/mean_terminated_length": 248.6767578125, |
| "completions/min_length": 81.5, |
| "completions/min_terminated_length": 81.5, |
| "entropy": 0.020743513945490122, |
| "epoch": 0.3037511436413541, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.18401126487271094, |
| "kl": 0.05533292470499873, |
| "learning_rate": 3.9905233373892316e-06, |
| "loss": 0.0109, |
| "num_tokens": 58194178.0, |
| "reward": 2.8100368976593018, |
| "reward_std": 0.4278334304690361, |
| "rewards/reward_fn/mean": 2.8100368976593018, |
| "rewards/reward_fn/std": 0.4278334453701973, |
| "step": 332, |
| "step_time": 368.58857105113566 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1095.0, |
| "completions/max_terminated_length": 1095.0, |
| "completions/mean_length": 255.9384765625, |
| "completions/mean_terminated_length": 255.9384765625, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.02218158240430057, |
| "epoch": 0.3074107959743824, |
| "frac_reward_zero_std": 0.8359375, |
| "grad_norm": 0.08064705887787386, |
| "kl": 0.057683263905346394, |
| "learning_rate": 3.9671131551427726e-06, |
| "loss": 0.0171, |
| "num_tokens": 59029263.0, |
| "reward": 2.756513297557831, |
| "reward_std": 0.4825749918818474, |
| "rewards/reward_fn/mean": 2.756513297557831, |
| "rewards/reward_fn/std": 0.4825750067830086, |
| "step": 336, |
| "step_time": 372.0321779474616 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1227.75, |
| "completions/max_terminated_length": 1127.25, |
| "completions/mean_length": 259.9306640625, |
| "completions/mean_terminated_length": 258.67418670654297, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.020468115573748946, |
| "epoch": 0.3110704483074108, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.06172113304563455, |
| "kl": 0.05510277720168233, |
| "learning_rate": 3.943505082856389e-06, |
| "loss": 0.014, |
| "num_tokens": 59871908.0, |
| "reward": 2.770124852657318, |
| "reward_std": 0.47307033091783524, |
| "rewards/reward_fn/mean": 2.770124852657318, |
| "rewards/reward_fn/std": 0.473070353269577, |
| "step": 340, |
| "step_time": 425.1579443984665 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.001953125, |
| "completions/max_length": 1364.5, |
| "completions/max_terminated_length": 1336.25, |
| "completions/mean_length": 258.8916015625, |
| "completions/mean_terminated_length": 256.39228439331055, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.01920273172436282, |
| "epoch": 0.31473010064043916, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.05523480707673635, |
| "kl": 0.054923647781834006, |
| "learning_rate": 3.919702304880485e-06, |
| "loss": 0.0101, |
| "num_tokens": 60718497.0, |
| "reward": 2.79230135679245, |
| "reward_std": 0.4282824918627739, |
| "rewards/reward_fn/mean": 2.79230135679245, |
| "rewards/reward_fn/std": 0.4282825067639351, |
| "step": 344, |
| "step_time": 487.7186696450226 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1177.5, |
| "completions/max_terminated_length": 1177.5, |
| "completions/mean_length": 258.7548828125, |
| "completions/mean_terminated_length": 258.7548828125, |
| "completions/min_length": 75.0, |
| "completions/min_terminated_length": 75.0, |
| "entropy": 0.01964573678560555, |
| "epoch": 0.3183897529734675, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.052307460272456544, |
| "kl": 0.05672413087449968, |
| "learning_rate": 3.895708031828138e-06, |
| "loss": 0.0061, |
| "num_tokens": 61563858.0, |
| "reward": 2.815351724624634, |
| "reward_std": 0.3556395545601845, |
| "rewards/reward_fn/mean": 2.815351724624634, |
| "rewards/reward_fn/std": 0.35563959181308746, |
| "step": 348, |
| "step_time": 412.0264103813097 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.001953125, |
| "completions/max_length": 1217.75, |
| "completions/max_terminated_length": 1126.5, |
| "completions/mean_length": 256.095703125, |
| "completions/mean_terminated_length": 253.58877563476562, |
| "completions/min_length": 82.25, |
| "completions/min_terminated_length": 82.25, |
| "entropy": 0.02070168766658753, |
| "epoch": 0.3220494053064959, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.04768525752073658, |
| "kl": 0.055517966859042645, |
| "learning_rate": 3.871525500142044e-06, |
| "loss": 0.0141, |
| "num_tokens": 62409304.0, |
| "reward": 2.8017632961273193, |
| "reward_std": 0.39498939365148544, |
| "rewards/reward_fn/mean": 2.8017632961273193, |
| "rewards/reward_fn/std": 0.39498940855264664, |
| "step": 352, |
| "step_time": 427.9275800297037 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1246.25, |
| "completions/max_terminated_length": 1246.25, |
| "completions/mean_length": 266.845703125, |
| "completions/mean_terminated_length": 266.845703125, |
| "completions/min_length": 76.0, |
| "completions/min_terminated_length": 76.0, |
| "entropy": 0.02018115541432053, |
| "epoch": 0.32570905763952424, |
| "frac_reward_zero_std": 0.90625, |
| "grad_norm": 0.05805445549946706, |
| "kl": 0.05352379777468741, |
| "learning_rate": 3.847157971657966e-06, |
| "loss": 0.0165, |
| "num_tokens": 63266074.0, |
| "reward": 2.7965877056121826, |
| "reward_std": 0.3831140995025635, |
| "rewards/reward_fn/mean": 2.7965877056121826, |
| "rewards/reward_fn/std": 0.38311412930488586, |
| "step": 356, |
| "step_time": 443.1046460606158 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1031.0, |
| "completions/max_terminated_length": 1031.0, |
| "completions/mean_length": 248.220703125, |
| "completions/mean_terminated_length": 248.220703125, |
| "completions/min_length": 76.5, |
| "completions/min_terminated_length": 76.5, |
| "entropy": 0.02002508647274226, |
| "epoch": 0.32936870997255263, |
| "frac_reward_zero_std": 0.921875, |
| "grad_norm": 0.03867933820199063, |
| "kl": 0.05510533833876252, |
| "learning_rate": 3.822608733164773e-06, |
| "loss": 0.0059, |
| "num_tokens": 64114416.0, |
| "reward": 2.8273446559906006, |
| "reward_std": 0.3517664670944214, |
| "rewards/reward_fn/mean": 2.8273446559906006, |
| "rewards/reward_fn/std": 0.3517664894461632, |
| "step": 360, |
| "step_time": 365.8222716213204 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1103.25, |
| "completions/max_terminated_length": 1103.25, |
| "completions/mean_length": 252.4033203125, |
| "completions/mean_terminated_length": 252.4033203125, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.021197067166212946, |
| "epoch": 0.33302836230558097, |
| "frac_reward_zero_std": 0.8984375, |
| "grad_norm": 0.08004850682857483, |
| "kl": 0.06018217862583697, |
| "learning_rate": 3.7978810959610963e-06, |
| "loss": 0.0113, |
| "num_tokens": 64967709.0, |
| "reward": 2.8053765892982483, |
| "reward_std": 0.4107788950204849, |
| "rewards/reward_fn/mean": 2.8053765892982483, |
| "rewards/reward_fn/std": 0.4107789024710655, |
| "step": 364, |
| "step_time": 390.9309317241423 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1106.5, |
| "completions/max_terminated_length": 1106.5, |
| "completions/mean_length": 249.0361328125, |
| "completions/mean_terminated_length": 249.0361328125, |
| "completions/min_length": 81.0, |
| "completions/min_terminated_length": 81.0, |
| "entropy": 0.022055889829061925, |
| "epoch": 0.3366880146386093, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.052541789630115916, |
| "kl": 0.05786171229556203, |
| "learning_rate": 3.7729783954086957e-06, |
| "loss": 0.0108, |
| "num_tokens": 65820186.0, |
| "reward": 2.7696725726127625, |
| "reward_std": 0.4212775230407715, |
| "rewards/reward_fn/mean": 2.7696725726127625, |
| "rewards/reward_fn/std": 0.4212775304913521, |
| "step": 368, |
| "step_time": 390.50815150514245 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1112.75, |
| "completions/max_terminated_length": 1112.75, |
| "completions/mean_length": 248.923828125, |
| "completions/mean_terminated_length": 248.923828125, |
| "completions/min_length": 80.5, |
| "completions/min_terminated_length": 80.5, |
| "entropy": 0.02093490824336186, |
| "epoch": 0.3403476669716377, |
| "frac_reward_zero_std": 0.875, |
| "grad_norm": 0.0593891232691319, |
| "kl": 0.05624948791228235, |
| "learning_rate": 3.7479039904825687e-06, |
| "loss": 0.0139, |
| "num_tokens": 66679968.0, |
| "reward": 2.749756693840027, |
| "reward_std": 0.4831894710659981, |
| "rewards/reward_fn/mean": 2.749756693840027, |
| "rewards/reward_fn/std": 0.4831894785165787, |
| "step": 372, |
| "step_time": 401.47150969458744 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1084.75, |
| "completions/max_terminated_length": 1084.75, |
| "completions/mean_length": 260.5693359375, |
| "completions/mean_terminated_length": 260.5693359375, |
| "completions/min_length": 82.0, |
| "completions/min_terminated_length": 82.0, |
| "entropy": 0.020609806873835623, |
| "epoch": 0.34400731930466605, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.05883873174274732, |
| "kl": 0.055998800322413445, |
| "learning_rate": 3.722661263317878e-06, |
| "loss": 0.009, |
| "num_tokens": 67547627.0, |
| "reward": 2.733439564704895, |
| "reward_std": 0.464918315410614, |
| "rewards/reward_fn/mean": 2.733439564704895, |
| "rewards/reward_fn/std": 0.464918352663517, |
| "step": 376, |
| "step_time": 388.22158924490213 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1015.0, |
| "completions/max_terminated_length": 1015.0, |
| "completions/mean_length": 256.7197265625, |
| "completions/mean_terminated_length": 256.7197265625, |
| "completions/min_length": 76.25, |
| "completions/min_terminated_length": 76.25, |
| "entropy": 0.020553262438625097, |
| "epoch": 0.34766697163769444, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.04900971293316462, |
| "kl": 0.053110834909603, |
| "learning_rate": 3.697253618753761e-06, |
| "loss": 0.0032, |
| "num_tokens": 68414508.0, |
| "reward": 2.7533047795295715, |
| "reward_std": 0.46282485127449036, |
| "rewards/reward_fn/mean": 2.7533047795295715, |
| "rewards/reward_fn/std": 0.46282486617565155, |
| "step": 380, |
| "step_time": 366.4996922570281 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1063.0, |
| "completions/max_terminated_length": 1063.0, |
| "completions/mean_length": 250.87109375, |
| "completions/mean_terminated_length": 250.87109375, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.02080925728660077, |
| "epoch": 0.3513266239707228, |
| "frac_reward_zero_std": 0.86328125, |
| "grad_norm": 0.04947422229421151, |
| "kl": 0.05815490847453475, |
| "learning_rate": 3.6716844838740644e-06, |
| "loss": 0.011, |
| "num_tokens": 69279260.0, |
| "reward": 2.745911419391632, |
| "reward_std": 0.45054207742214203, |
| "rewards/reward_fn/mean": 2.745911419391632, |
| "rewards/reward_fn/std": 0.4505421072244644, |
| "step": 384, |
| "step_time": 383.8784329756163 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1050.0, |
| "completions/max_terminated_length": 1050.0, |
| "completions/mean_length": 255.9560546875, |
| "completions/mean_terminated_length": 255.9560546875, |
| "completions/min_length": 79.5, |
| "completions/min_terminated_length": 79.5, |
| "entropy": 0.020128908683545887, |
| "epoch": 0.3549862763037511, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.033508899160515984, |
| "kl": 0.05457201926037669, |
| "learning_rate": 3.6459573075450904e-06, |
| "loss": 0.005, |
| "num_tokens": 70162583.0, |
| "reward": 2.758453071117401, |
| "reward_std": 0.47200874239206314, |
| "rewards/reward_fn/mean": 2.758453071117401, |
| "rewards/reward_fn/std": 0.47200877219438553, |
| "step": 388, |
| "step_time": 387.83291961671785 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1117.5, |
| "completions/max_terminated_length": 1117.5, |
| "completions/mean_length": 266.169921875, |
| "completions/mean_terminated_length": 266.169921875, |
| "completions/min_length": 81.75, |
| "completions/min_terminated_length": 81.75, |
| "entropy": 0.022365455282852054, |
| "epoch": 0.3586459286367795, |
| "frac_reward_zero_std": 0.87109375, |
| "grad_norm": 0.08663973531428495, |
| "kl": 0.055080414516851306, |
| "learning_rate": 3.6200755599503987e-06, |
| "loss": 0.0246, |
| "num_tokens": 71052317.0, |
| "reward": 2.767097592353821, |
| "reward_std": 0.4057338833808899, |
| "rewards/reward_fn/mean": 2.767097592353821, |
| "rewards/reward_fn/std": 0.4057339131832123, |
| "step": 392, |
| "step_time": 412.0218286192976 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1055.0, |
| "completions/max_terminated_length": 1055.0, |
| "completions/mean_length": 262.7802734375, |
| "completions/mean_terminated_length": 262.7802734375, |
| "completions/min_length": 74.25, |
| "completions/min_terminated_length": 74.25, |
| "entropy": 0.023916912963613868, |
| "epoch": 0.36230558096980786, |
| "frac_reward_zero_std": 0.8359375, |
| "grad_norm": 0.10456738086214372, |
| "kl": 0.055561956483870745, |
| "learning_rate": 3.594042732122735e-06, |
| "loss": 0.02, |
| "num_tokens": 71947252.0, |
| "reward": 2.746975362300873, |
| "reward_std": 0.44752104580402374, |
| "rewards/reward_fn/mean": 2.746975362300873, |
| "rewards/reward_fn/std": 0.44752106815576553, |
| "step": 396, |
| "step_time": 388.2638799441047 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1113.0, |
| "completions/max_terminated_length": 1113.0, |
| "completions/mean_length": 255.6689453125, |
| "completions/mean_terminated_length": 255.6689453125, |
| "completions/min_length": 84.0, |
| "completions/min_terminated_length": 84.0, |
| "entropy": 0.023536766064353287, |
| "epoch": 0.36596523330283626, |
| "frac_reward_zero_std": 0.81640625, |
| "grad_norm": 0.07529446863056388, |
| "kl": 0.059099239530041814, |
| "learning_rate": 3.567862335473144e-06, |
| "loss": 0.0172, |
| "num_tokens": 72839781.0, |
| "reward": 2.743981719017029, |
| "reward_std": 0.4595012813806534, |
| "rewards/reward_fn/mean": 2.743981719017029, |
| "rewards/reward_fn/std": 0.45950130373239517, |
| "step": 400, |
| "step_time": 425.3582200766541 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1169.5, |
| "completions/max_terminated_length": 1169.5, |
| "completions/mean_length": 264.8642578125, |
| "completions/mean_terminated_length": 264.8642578125, |
| "completions/min_length": 76.25, |
| "completions/min_terminated_length": 76.25, |
| "entropy": 0.021931169438175857, |
| "epoch": 0.3696248856358646, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.06680599370876592, |
| "kl": 0.06040530768223107, |
| "learning_rate": 3.541537901317338e-06, |
| "loss": 0.0248, |
| "num_tokens": 73741594.0, |
| "reward": 2.7557645440101624, |
| "reward_std": 0.44654303044080734, |
| "rewards/reward_fn/mean": 2.7557645440101624, |
| "rewards/reward_fn/std": 0.44654303789138794, |
| "step": 404, |
| "step_time": 441.0422842912376 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1128.0, |
| "completions/max_terminated_length": 1128.0, |
| "completions/mean_length": 252.75, |
| "completions/mean_terminated_length": 252.75, |
| "completions/min_length": 79.25, |
| "completions/min_terminated_length": 79.25, |
| "entropy": 0.020903860335238278, |
| "epoch": 0.37328453796889294, |
| "frac_reward_zero_std": 0.88671875, |
| "grad_norm": 0.04185240259764596, |
| "kl": 0.057490175822749734, |
| "learning_rate": 3.5150729803993822e-06, |
| "loss": 0.0072, |
| "num_tokens": 74630742.0, |
| "reward": 2.7497737407684326, |
| "reward_std": 0.440328411757946, |
| "rewards/reward_fn/mean": 2.7497737407684326, |
| "rewards/reward_fn/std": 0.4403284415602684, |
| "step": 408, |
| "step_time": 428.5058938455768 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1261.5, |
| "completions/max_terminated_length": 1242.0, |
| "completions/mean_length": 256.89453125, |
| "completions/mean_terminated_length": 255.64898681640625, |
| "completions/min_length": 82.0, |
| "completions/min_terminated_length": 82.0, |
| "entropy": 0.020710848970338702, |
| "epoch": 0.37694419030192133, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.06088040163935032, |
| "kl": 0.058197865495458245, |
| "learning_rate": 3.4884711424127505e-06, |
| "loss": 0.0096, |
| "num_tokens": 75530950.0, |
| "reward": 2.7645320892333984, |
| "reward_std": 0.43489429354667664, |
| "rewards/reward_fn/mean": 2.7645320892333984, |
| "rewards/reward_fn/std": 0.43489427864551544, |
| "step": 412, |
| "step_time": 489.43961844732985 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1102.5, |
| "completions/max_terminated_length": 1102.5, |
| "completions/mean_length": 246.728515625, |
| "completions/mean_terminated_length": 246.728515625, |
| "completions/min_length": 80.25, |
| "completions/min_terminated_length": 80.25, |
| "entropy": 0.020525632542558014, |
| "epoch": 0.3806038426349497, |
| "frac_reward_zero_std": 0.88671875, |
| "grad_norm": 0.05255012893742152, |
| "kl": 0.05784119013696909, |
| "learning_rate": 3.461735975518836e-06, |
| "loss": 0.0074, |
| "num_tokens": 76428300.0, |
| "reward": 2.7354435324668884, |
| "reward_std": 0.5114951506257057, |
| "rewards/reward_fn/mean": 2.7354435324668884, |
| "rewards/reward_fn/std": 0.5114951729774475, |
| "step": 416, |
| "step_time": 426.42004963662475 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1093.5, |
| "completions/max_terminated_length": 1093.5, |
| "completions/mean_length": 249.1689453125, |
| "completions/mean_terminated_length": 249.1689453125, |
| "completions/min_length": 73.0, |
| "completions/min_terminated_length": 73.0, |
| "entropy": 0.02230722492095083, |
| "epoch": 0.38426349496797807, |
| "frac_reward_zero_std": 0.87109375, |
| "grad_norm": 0.042090748623265346, |
| "kl": 0.06311949389055371, |
| "learning_rate": 3.4348710858629626e-06, |
| "loss": 0.0061, |
| "num_tokens": 77327597.0, |
| "reward": 2.7153448462486267, |
| "reward_std": 0.5665776580572128, |
| "rewards/reward_fn/mean": 2.7153448462486267, |
| "rewards/reward_fn/std": 0.5665776580572128, |
| "step": 420, |
| "step_time": 423.4696046109311 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1085.0, |
| "completions/max_terminated_length": 1085.0, |
| "completions/mean_length": 261.4853515625, |
| "completions/mean_terminated_length": 261.4853515625, |
| "completions/min_length": 78.25, |
| "completions/min_terminated_length": 78.25, |
| "entropy": 0.02214888739399612, |
| "epoch": 0.3879231473010064, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.06477089541532775, |
| "kl": 0.05813350295647979, |
| "learning_rate": 3.4078800970879742e-06, |
| "loss": 0.0032, |
| "num_tokens": 78240558.0, |
| "reward": 2.7490416765213013, |
| "reward_std": 0.4679771661758423, |
| "rewards/reward_fn/mean": 2.7490416765213013, |
| "rewards/reward_fn/std": 0.4679771810770035, |
| "step": 424, |
| "step_time": 418.0459654238075 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1153.0, |
| "completions/max_terminated_length": 1153.0, |
| "completions/mean_length": 256.8740234375, |
| "completions/mean_terminated_length": 256.8740234375, |
| "completions/min_length": 65.75, |
| "completions/min_terminated_length": 65.75, |
| "entropy": 0.022280778735876083, |
| "epoch": 0.39158279963403475, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.05071083161625681, |
| "kl": 0.05995195545256138, |
| "learning_rate": 3.3807666498454637e-06, |
| "loss": 0.0113, |
| "num_tokens": 79158477.0, |
| "reward": 2.763080358505249, |
| "reward_std": 0.4363250806927681, |
| "rewards/reward_fn/mean": 2.763080358505249, |
| "rewards/reward_fn/std": 0.4363251179456711, |
| "step": 428, |
| "step_time": 458.79944900562987 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1053.75, |
| "completions/max_terminated_length": 1053.75, |
| "completions/mean_length": 258.5673828125, |
| "completions/mean_terminated_length": 258.5673828125, |
| "completions/min_length": 73.25, |
| "completions/min_terminated_length": 73.25, |
| "entropy": 0.022120147943496704, |
| "epoch": 0.39524245196706315, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.041363944327058255, |
| "kl": 0.059779631439596415, |
| "learning_rate": 3.3535344013047107e-06, |
| "loss": 0.0103, |
| "num_tokens": 80074798.0, |
| "reward": 2.7749913930892944, |
| "reward_std": 0.4496825709939003, |
| "rewards/reward_fn/mean": 2.7749913930892944, |
| "rewards/reward_fn/std": 0.4496825933456421, |
| "step": 432, |
| "step_time": 413.0009705722332 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1176.0, |
| "completions/max_terminated_length": 1176.0, |
| "completions/mean_length": 254.3369140625, |
| "completions/mean_terminated_length": 254.3369140625, |
| "completions/min_length": 79.75, |
| "completions/min_terminated_length": 79.75, |
| "entropy": 0.021958659519441426, |
| "epoch": 0.3989021043000915, |
| "frac_reward_zero_std": 0.91796875, |
| "grad_norm": 0.029236661292790365, |
| "kl": 0.05681233340874314, |
| "learning_rate": 3.3261870246593804e-06, |
| "loss": 0.0054, |
| "num_tokens": 80987639.0, |
| "reward": 2.7603381276130676, |
| "reward_std": 0.4442455396056175, |
| "rewards/reward_fn/mean": 2.7603381276130676, |
| "rewards/reward_fn/std": 0.4442455470561981, |
| "step": 436, |
| "step_time": 465.0974967526272 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1091.0, |
| "completions/max_terminated_length": 1091.0, |
| "completions/mean_length": 256.734375, |
| "completions/mean_terminated_length": 256.734375, |
| "completions/min_length": 78.5, |
| "completions/min_terminated_length": 78.5, |
| "entropy": 0.021211280778516084, |
| "epoch": 0.4025617566331198, |
| "frac_reward_zero_std": 0.8984375, |
| "grad_norm": 0.02865251038786027, |
| "kl": 0.05977889499627054, |
| "learning_rate": 3.2987282086320756e-06, |
| "loss": 0.0012, |
| "num_tokens": 81908035.0, |
| "reward": 2.803665339946747, |
| "reward_std": 0.4023703336715698, |
| "rewards/reward_fn/mean": 2.803665339946747, |
| "rewards/reward_fn/std": 0.4023703634738922, |
| "step": 440, |
| "step_time": 431.6096727340482 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1156.5, |
| "completions/max_terminated_length": 1156.5, |
| "completions/mean_length": 255.9501953125, |
| "completions/mean_terminated_length": 255.9501953125, |
| "completions/min_length": 73.25, |
| "completions/min_terminated_length": 73.25, |
| "entropy": 0.021191293955780566, |
| "epoch": 0.4062214089661482, |
| "frac_reward_zero_std": 0.91796875, |
| "grad_norm": 0.031018340336869805, |
| "kl": 0.05657385312952101, |
| "learning_rate": 3.271161656976787e-06, |
| "loss": 0.0062, |
| "num_tokens": 82839944.0, |
| "reward": 2.771248936653137, |
| "reward_std": 0.4050714895129204, |
| "rewards/reward_fn/mean": 2.771248936653137, |
| "rewards/reward_fn/std": 0.40507151931524277, |
| "step": 444, |
| "step_time": 467.8179361205548 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1117.25, |
| "completions/max_terminated_length": 1117.25, |
| "completions/mean_length": 257.2421875, |
| "completions/mean_terminated_length": 257.2421875, |
| "completions/min_length": 77.0, |
| "completions/min_terminated_length": 77.0, |
| "entropy": 0.021266737254336476, |
| "epoch": 0.40988106129917656, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.045221824384451687, |
| "kl": 0.05658956873230636, |
| "learning_rate": 3.243491087979309e-06, |
| "loss": 0.0101, |
| "num_tokens": 83773492.0, |
| "reward": 2.7226498126983643, |
| "reward_std": 0.5032680928707123, |
| "rewards/reward_fn/mean": 2.7226498126983643, |
| "rewards/reward_fn/std": 0.5032681003212929, |
| "step": 448, |
| "step_time": 454.49583586258814 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1088.75, |
| "completions/max_terminated_length": 1088.75, |
| "completions/mean_length": 260.2900390625, |
| "completions/mean_terminated_length": 260.2900390625, |
| "completions/min_length": 70.0, |
| "completions/min_terminated_length": 70.0, |
| "entropy": 0.021140800090506673, |
| "epoch": 0.41354071363220496, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.06434430504394675, |
| "kl": 0.05833716504275799, |
| "learning_rate": 3.2157202339557124e-06, |
| "loss": 0.0051, |
| "num_tokens": 84713729.0, |
| "reward": 2.70987868309021, |
| "reward_std": 0.5102557465434074, |
| "rewards/reward_fn/mean": 2.70987868309021, |
| "rewards/reward_fn/std": 0.5102557688951492, |
| "step": 452, |
| "step_time": 441.5249572279863 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1206.25, |
| "completions/max_terminated_length": 1069.25, |
| "completions/mean_length": 255.240234375, |
| "completions/mean_terminated_length": 253.99888610839844, |
| "completions/min_length": 73.75, |
| "completions/min_terminated_length": 73.75, |
| "entropy": 0.02270206529647112, |
| "epoch": 0.4172003659652333, |
| "frac_reward_zero_std": 0.89453125, |
| "grad_norm": 0.06982801322672433, |
| "kl": 0.05912393843755126, |
| "learning_rate": 3.1878528407489068e-06, |
| "loss": 0.0153, |
| "num_tokens": 85651831.0, |
| "reward": 2.766897201538086, |
| "reward_std": 0.453893318772316, |
| "rewards/reward_fn/mean": 2.766897201538086, |
| "rewards/reward_fn/std": 0.45389336347579956, |
| "step": 456, |
| "step_time": 500.6877592424862 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1148.0, |
| "completions/max_terminated_length": 1148.0, |
| "completions/mean_length": 265.1240234375, |
| "completions/mean_terminated_length": 265.1240234375, |
| "completions/min_length": 74.75, |
| "completions/min_terminated_length": 74.75, |
| "entropy": 0.022504316177219152, |
| "epoch": 0.42086001829826164, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.2329375834800146, |
| "kl": 0.05883700540289283, |
| "learning_rate": 3.1598926672233916e-06, |
| "loss": 0.0034, |
| "num_tokens": 86598682.0, |
| "reward": 2.756505012512207, |
| "reward_std": 0.440291665494442, |
| "rewards/reward_fn/mean": 2.756505012512207, |
| "rewards/reward_fn/std": 0.4402916952967644, |
| "step": 460, |
| "step_time": 474.26262577204034 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1123.5, |
| "completions/max_terminated_length": 1123.5, |
| "completions/mean_length": 254.2958984375, |
| "completions/mean_terminated_length": 254.2958984375, |
| "completions/min_length": 73.75, |
| "completions/min_terminated_length": 73.75, |
| "entropy": 0.023365118307992816, |
| "epoch": 0.42451967063129004, |
| "frac_reward_zero_std": 0.90234375, |
| "grad_norm": 0.053993897375030235, |
| "kl": 0.05943758552893996, |
| "learning_rate": 3.131843484758242e-06, |
| "loss": 0.0152, |
| "num_tokens": 87535253.0, |
| "reward": 2.7400495409965515, |
| "reward_std": 0.48594386130571365, |
| "rewards/reward_fn/mean": 2.7400495409965515, |
| "rewards/reward_fn/std": 0.48594388365745544, |
| "step": 464, |
| "step_time": 459.86614966299385 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1115.25, |
| "completions/max_terminated_length": 1115.25, |
| "completions/mean_length": 264.953125, |
| "completions/mean_terminated_length": 264.953125, |
| "completions/min_length": 79.0, |
| "completions/min_terminated_length": 79.0, |
| "entropy": 0.02322944754268974, |
| "epoch": 0.4281793229643184, |
| "frac_reward_zero_std": 0.87109375, |
| "grad_norm": 0.05386210913195399, |
| "kl": 0.05839638109318912, |
| "learning_rate": 3.1037090767384128e-06, |
| "loss": 0.0176, |
| "num_tokens": 88492045.0, |
| "reward": 2.757098913192749, |
| "reward_std": 0.44403837621212006, |
| "rewards/reward_fn/mean": 2.757098913192749, |
| "rewards/reward_fn/std": 0.44403839111328125, |
| "step": 468, |
| "step_time": 463.87284979457036 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1138.25, |
| "completions/max_terminated_length": 1138.25, |
| "completions/mean_length": 265.919921875, |
| "completions/mean_terminated_length": 265.919921875, |
| "completions/min_length": 77.25, |
| "completions/min_terminated_length": 77.25, |
| "entropy": 0.02893318561837077, |
| "epoch": 0.43183897529734677, |
| "frac_reward_zero_std": 0.77734375, |
| "grad_norm": 0.10988372126851806, |
| "kl": 0.06366393179632723, |
| "learning_rate": 3.0754932380444204e-06, |
| "loss": 0.0647, |
| "num_tokens": 89458687.0, |
| "reward": 2.7596094608306885, |
| "reward_std": 0.4247737377882004, |
| "rewards/reward_fn/mean": 2.7596094608306885, |
| "rewards/reward_fn/std": 0.42477378249168396, |
| "step": 472, |
| "step_time": 481.56263000750914 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1195.5, |
| "completions/max_terminated_length": 1058.5, |
| "completions/mean_length": 260.591796875, |
| "completions/mean_terminated_length": 259.3289260864258, |
| "completions/min_length": 68.75, |
| "completions/min_terminated_length": 68.75, |
| "entropy": 0.030687018181197345, |
| "epoch": 0.4354986276303751, |
| "frac_reward_zero_std": 0.82421875, |
| "grad_norm": 0.0672494129113056, |
| "kl": 0.06749603967182338, |
| "learning_rate": 3.04719977454047e-06, |
| "loss": 0.0401, |
| "num_tokens": 90417957.0, |
| "reward": 2.7592405676841736, |
| "reward_std": 0.44465500861406326, |
| "rewards/reward_fn/mean": 2.7592405676841736, |
| "rewards/reward_fn/std": 0.44465503841638565, |
| "step": 476, |
| "step_time": 508.5822904477827 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1168.0, |
| "completions/max_terminated_length": 1168.0, |
| "completions/mean_length": 259.3037109375, |
| "completions/mean_terminated_length": 259.3037109375, |
| "completions/min_length": 69.75, |
| "completions/min_terminated_length": 69.75, |
| "entropy": 0.03189253679011017, |
| "epoch": 0.43915827996340345, |
| "frac_reward_zero_std": 0.8203125, |
| "grad_norm": 0.1133846195195051, |
| "kl": 0.07560356380417943, |
| "learning_rate": 3.018832502561114e-06, |
| "loss": 0.0483, |
| "num_tokens": 91386696.0, |
| "reward": 2.7331442832946777, |
| "reward_std": 0.48754725605249405, |
| "rewards/reward_fn/mean": 2.7331442832946777, |
| "rewards/reward_fn/std": 0.48754723370075226, |
| "step": 480, |
| "step_time": 499.99215250462294 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1011.5, |
| "completions/max_terminated_length": 1011.5, |
| "completions/mean_length": 253.7373046875, |
| "completions/mean_terminated_length": 253.7373046875, |
| "completions/min_length": 75.0, |
| "completions/min_terminated_length": 75.0, |
| "entropy": 0.027319170185364783, |
| "epoch": 0.44281793229643185, |
| "frac_reward_zero_std": 0.8671875, |
| "grad_norm": 0.03537841408588395, |
| "kl": 0.06349759339354932, |
| "learning_rate": 2.9903952483964804e-06, |
| "loss": 0.0063, |
| "num_tokens": 92351395.0, |
| "reward": 2.747387111186981, |
| "reward_std": 0.44578108191490173, |
| "rewards/reward_fn/mean": 2.747387111186981, |
| "rewards/reward_fn/std": 0.4457810968160629, |
| "step": 484, |
| "step_time": 436.93430981226265 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1156.25, |
| "completions/max_terminated_length": 1156.25, |
| "completions/mean_length": 257.44140625, |
| "completions/mean_terminated_length": 257.44140625, |
| "completions/min_length": 81.0, |
| "completions/min_terminated_length": 81.0, |
| "entropy": 0.026745406910777092, |
| "epoch": 0.4464775846294602, |
| "frac_reward_zero_std": 0.85546875, |
| "grad_norm": 0.05053599725656754, |
| "kl": 0.06292624375782907, |
| "learning_rate": 2.961891847776174e-06, |
| "loss": 0.0139, |
| "num_tokens": 93317339.0, |
| "reward": 2.7652419805526733, |
| "reward_std": 0.4404691755771637, |
| "rewards/reward_fn/mean": 2.7652419805526733, |
| "rewards/reward_fn/std": 0.4404691904783249, |
| "step": 488, |
| "step_time": 498.2464709286578 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1044.5, |
| "completions/max_terminated_length": 1044.5, |
| "completions/mean_length": 245.5771484375, |
| "completions/mean_terminated_length": 245.5771484375, |
| "completions/min_length": 74.0, |
| "completions/min_terminated_length": 74.0, |
| "entropy": 0.02598623267840594, |
| "epoch": 0.4501372369624886, |
| "frac_reward_zero_std": 0.8828125, |
| "grad_norm": 0.054287951744274386, |
| "kl": 0.06363053061068058, |
| "learning_rate": 2.933326145351895e-06, |
| "loss": 0.003, |
| "num_tokens": 94273746.0, |
| "reward": 2.755536675453186, |
| "reward_std": 0.43755053728818893, |
| "rewards/reward_fn/mean": 2.755536675453186, |
| "rewards/reward_fn/std": 0.43755055218935013, |
| "step": 492, |
| "step_time": 447.63349654572085 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0009765625, |
| "completions/max_length": 1222.75, |
| "completions/max_terminated_length": 1096.0, |
| "completions/mean_length": 258.2802734375, |
| "completions/mean_terminated_length": 257.03343200683594, |
| "completions/min_length": 80.5, |
| "completions/min_terminated_length": 80.5, |
| "entropy": 0.026308491243980825, |
| "epoch": 0.4537968892955169, |
| "frac_reward_zero_std": 0.859375, |
| "grad_norm": 0.06635558280599736, |
| "kl": 0.06186160701327026, |
| "learning_rate": 2.9047019941788584e-06, |
| "loss": 0.0209, |
| "num_tokens": 95258201.0, |
| "reward": 2.753948450088501, |
| "reward_std": 0.4302341192960739, |
| "rewards/reward_fn/mean": 2.753948450088501, |
| "rewards/reward_fn/std": 0.4302341416478157, |
| "step": 496, |
| "step_time": 541.1232072496787 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 1168.75, |
| "completions/max_terminated_length": 1168.75, |
| "completions/mean_length": 250.376953125, |
| "completions/mean_terminated_length": 250.376953125, |
| "completions/min_length": 83.75, |
| "completions/min_terminated_length": 83.75, |
| "entropy": 0.026318517164327204, |
| "epoch": 0.45745654162854527, |
| "frac_reward_zero_std": 0.87890625, |
| "grad_norm": 0.05395106254679997, |
| "kl": 0.06209555873647332, |
| "learning_rate": 2.8760232551960782e-06, |
| "loss": 0.0063, |
| "num_tokens": 96236875.0, |
| "reward": 2.7645170092582703, |
| "reward_std": 0.4234401807188988, |
| "rewards/reward_fn/mean": 2.7645170092582703, |
| "rewards/reward_fn/std": 0.42344019562005997, |
| "step": 500, |
| "step_time": 512.8094203663059 |
| } |
| ], |
| "logging_steps": 4, |
| "max_steps": 1093, |
| "num_input_tokens_seen": 96236875, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
