{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.45745654162854527, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1080.75, "completions/max_terminated_length": 1080.75, "completions/mean_length": 206.859375, "completions/mean_terminated_length": 206.859375, "completions/min_length": 72.75, "completions/min_terminated_length": 72.75, "entropy": 0.01571042463183403, "epoch": 0.0036596523330283625, "frac_reward_zero_std": 0.6484375, "grad_norm": 0.1166667682172164, "kl": 6.741156056477848e-06, "learning_rate": 1.8181818181818183e-06, "loss": 0.019, "num_tokens": 595592.0, "reward": 2.5668949484825134, "reward_std": 0.6298200786113739, "rewards/reward_fn/mean": 2.5668949484825134, "rewards/reward_fn/std": 0.6298201084136963, "step": 4, "step_time": 260.7198786311783 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1112.5, "completions/max_terminated_length": 1112.5, "completions/mean_length": 207.083984375, "completions/mean_terminated_length": 207.083984375, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.014931437617633492, "epoch": 0.007319304666056725, "frac_reward_zero_std": 0.67578125, "grad_norm": 0.07972757343469568, "kl": 7.538596197065317e-05, "learning_rate": 3.6363636363636366e-06, "loss": 0.0116, "num_tokens": 1191286.0, "reward": 2.6111356019973755, "reward_std": 0.5753327757120132, "rewards/reward_fn/mean": 2.6111356019973755, "rewards/reward_fn/std": 0.5753328055143356, "step": 8, "step_time": 270.5837429687381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1084.5, "completions/max_terminated_length": 1084.5, "completions/mean_length": 200.9501953125, "completions/mean_terminated_length": 200.9501953125, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.015009274939075112, "epoch": 0.010978956999085087, "frac_reward_zero_std": 0.75390625, "grad_norm": 0.08571915975811008, "kl": 0.0005235518705433151, "learning_rate": 4.999989462079163e-06, "loss": 0.0121, "num_tokens": 1780099.0, "reward": 2.6977345943450928, "reward_std": 0.44414132833480835, "rewards/reward_fn/mean": 2.6977345943450928, "rewards/reward_fn/std": 0.44414133578538895, "step": 12, "step_time": 261.6133712823503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.75, "completions/max_terminated_length": 919.75, "completions/mean_length": 201.7529296875, "completions/mean_terminated_length": 201.7529296875, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.015052395581733435, "epoch": 0.01463860933211345, "frac_reward_zero_std": 0.765625, "grad_norm": 0.06477325550449937, "kl": 0.0006825914342698525, "learning_rate": 4.999736556420954e-06, "loss": 0.018, "num_tokens": 2371954.0, "reward": 2.7144583463668823, "reward_std": 0.42256640642881393, "rewards/reward_fn/mean": 2.7144583463668823, "rewards/reward_fn/std": 0.42256640642881393, "step": 16, "step_time": 214.89796239417046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1129.75, "completions/max_terminated_length": 1129.75, "completions/mean_length": 207.7060546875, "completions/mean_terminated_length": 207.7060546875, "completions/min_length": 79.25, "completions/min_terminated_length": 79.25, "entropy": 0.013794319529552013, "epoch": 0.018298261665141813, "frac_reward_zero_std": 0.8046875, "grad_norm": 0.07172710130306945, "kl": 0.001074870842785458, "learning_rate": 4.9991464763837844e-06, "loss": 0.0119, "num_tokens": 2971833.0, "reward": 2.7160686254501343, "reward_std": 0.4207172691822052, "rewards/reward_fn/mean": 2.7160686254501343, "rewards/reward_fn/std": 0.4207172840833664, "step": 20, "step_time": 275.1132181570865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 974.5, "completions/max_terminated_length": 974.5, "completions/mean_length": 207.3681640625, "completions/mean_terminated_length": 207.3681640625, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.014441891689784825, "epoch": 0.021957913998170174, "frac_reward_zero_std": 0.734375, "grad_norm": 0.08257356486413421, "kl": 0.0034124660814995877, "learning_rate": 4.99821930155999e-06, "loss": -0.0014, "num_tokens": 3569890.0, "reward": 2.6976758241653442, "reward_std": 0.4469035342335701, "rewards/reward_fn/mean": 2.6976758241653442, "rewards/reward_fn/std": 0.4469035491347313, "step": 24, "step_time": 229.607499552425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 985.5, "completions/max_terminated_length": 985.5, "completions/mean_length": 187.828125, "completions/mean_terminated_length": 187.828125, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 0.01451652345713228, "epoch": 0.025617566331198535, "frac_reward_zero_std": 0.7734375, "grad_norm": 0.1178542444151902, "kl": 0.011817638587672263, "learning_rate": 4.996955157010586e-06, "loss": 0.0146, "num_tokens": 4150210.0, "reward": 2.7241913080215454, "reward_std": 0.42349882423877716, "rewards/reward_fn/mean": 2.7241913080215454, "rewards/reward_fn/std": 0.42349884659051895, "step": 28, "step_time": 236.96254792809486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 964.0, "completions/max_terminated_length": 964.0, "completions/mean_length": 180.63671875, "completions/mean_terminated_length": 180.63671875, "completions/min_length": 78.25, "completions/min_terminated_length": 78.25, "entropy": 0.014597299275919795, "epoch": 0.0292772186642269, "frac_reward_zero_std": 0.80078125, "grad_norm": 0.191243062362765, "kl": 0.03357400168897584, "learning_rate": 4.995354213248404e-06, "loss": 0.002, "num_tokens": 4728162.0, "reward": 2.7361614108085632, "reward_std": 0.40531516820192337, "rewards/reward_fn/mean": 2.7361614108085632, "rewards/reward_fn/std": 0.40531518310308456, "step": 32, "step_time": 227.9491332215257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1252.5, "completions/max_terminated_length": 1252.5, "completions/mean_length": 198.6748046875, "completions/mean_terminated_length": 198.6748046875, "completions/min_length": 77.25, "completions/min_terminated_length": 77.25, "entropy": 0.014511767774820328, "epoch": 0.03293687099725526, "frac_reward_zero_std": 0.765625, "grad_norm": 1.2528338790788236, "kl": 0.02709985862020403, "learning_rate": 4.9934166862150855e-06, "loss": -0.0043, "num_tokens": 5325577.0, "reward": 2.7323471903800964, "reward_std": 0.4196270480751991, "rewards/reward_fn/mean": 2.7323471903800964, "rewards/reward_fn/std": 0.4196270480751991, "step": 36, "step_time": 311.6690187435597 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1034.5, "completions/max_terminated_length": 1034.5, "completions/mean_length": 190.1337890625, "completions/mean_terminated_length": 190.1337890625, "completions/min_length": 73.5, "completions/min_terminated_length": 73.5, "entropy": 0.013505439972504973, "epoch": 0.036596523330283626, "frac_reward_zero_std": 0.80078125, "grad_norm": 0.08402394326328486, "kl": 0.020394871709868312, "learning_rate": 4.991142837251961e-06, "loss": -0.0138, "num_tokens": 5913470.0, "reward": 2.760911762714386, "reward_std": 0.3911248818039894, "rewards/reward_fn/mean": 2.760911762714386, "rewards/reward_fn/std": 0.3911249041557312, "step": 40, "step_time": 245.01106920232996 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 986.5, "completions/max_terminated_length": 986.5, "completions/mean_length": 197.076171875, "completions/mean_terminated_length": 197.076171875, "completions/min_length": 73.25, "completions/min_terminated_length": 73.25, "entropy": 0.013148889294825494, "epoch": 0.04025617566331199, "frac_reward_zero_std": 0.78515625, "grad_norm": 0.07640337001807694, "kl": 0.02207769750384614, "learning_rate": 4.988532973064793e-06, "loss": -0.016, "num_tokens": 6509836.0, "reward": 2.7576756477355957, "reward_std": 0.40184224396944046, "rewards/reward_fn/mean": 2.7576756477355957, "rewards/reward_fn/std": 0.40184228122234344, "step": 44, "step_time": 232.39970854343846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1041.0, "completions/max_terminated_length": 1041.0, "completions/mean_length": 201.873046875, "completions/mean_terminated_length": 201.873046875, "completions/min_length": 83.75, "completions/min_terminated_length": 83.75, "entropy": 0.012962576001882553, "epoch": 0.04391582799634035, "frac_reward_zero_std": 0.79296875, "grad_norm": 0.09402744049682173, "kl": 0.015623664075974375, "learning_rate": 4.985587445682414e-06, "loss": -0.0103, "num_tokens": 7112210.0, "reward": 2.761009693145752, "reward_std": 0.3792940303683281, "rewards/reward_fn/mean": 2.761009693145752, "rewards/reward_fn/std": 0.3792940229177475, "step": 48, "step_time": 248.20540295541286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 850.0, "completions/max_terminated_length": 850.0, "completions/mean_length": 180.35546875, "completions/mean_terminated_length": 180.35546875, "completions/min_length": 70.25, "completions/min_terminated_length": 70.25, "entropy": 0.012066802359186113, "epoch": 0.04757548032936871, "frac_reward_zero_std": 0.80078125, "grad_norm": 0.05676690796949128, "kl": 0.02138438867405057, "learning_rate": 4.982306652409236e-06, "loss": -0.0238, "num_tokens": 7691458.0, "reward": 2.7505001425743103, "reward_std": 0.4136660769581795, "rewards/reward_fn/mean": 2.7505001425743103, "rewards/reward_fn/std": 0.4136660695075989, "step": 52, "step_time": 195.69965509325266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1014.5, "completions/max_terminated_length": 1014.5, "completions/mean_length": 203.6640625, "completions/mean_terminated_length": 203.6640625, "completions/min_length": 81.25, "completions/min_terminated_length": 81.25, "entropy": 0.012359246786218137, "epoch": 0.05123513266239707, "frac_reward_zero_std": 0.81640625, "grad_norm": 0.1820463265067798, "kl": 0.018120157648809254, "learning_rate": 4.978691035771666e-06, "loss": -0.0187, "num_tokens": 8298814.0, "reward": 2.7694973945617676, "reward_std": 0.40114688873291016, "rewards/reward_fn/mean": 2.7694973945617676, "rewards/reward_fn/std": 0.40114689618349075, "step": 56, "step_time": 240.87798455543816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1240.75, "completions/max_terminated_length": 1240.75, "completions/mean_length": 209.6689453125, "completions/mean_terminated_length": 209.6689453125, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.012856585904955864, "epoch": 0.05489478499542543, "frac_reward_zero_std": 0.828125, "grad_norm": 0.09204206328450969, "kl": 0.036135767586529255, "learning_rate": 4.9747410834584134e-06, "loss": 0.0008, "num_tokens": 8915947.0, "reward": 2.779680848121643, "reward_std": 0.37843772768974304, "rewards/reward_fn/mean": 2.779680848121643, "rewards/reward_fn/std": 0.37843775749206543, "step": 60, "step_time": 308.8453072188422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1042.75, "completions/max_terminated_length": 1042.75, "completions/mean_length": 212.9931640625, "completions/mean_terminated_length": 212.9931640625, "completions/min_length": 79.25, "completions/min_terminated_length": 79.25, "entropy": 0.012349803757388145, "epoch": 0.0585544373284538, "frac_reward_zero_std": 0.80859375, "grad_norm": 0.10624366951375605, "kl": 0.037413959740661085, "learning_rate": 4.970457328254707e-06, "loss": -0.0226, "num_tokens": 9536872.0, "reward": 2.7904839515686035, "reward_std": 0.3760991021990776, "rewards/reward_fn/mean": 2.7904839515686035, "rewards/reward_fn/std": 0.3760991171002388, "step": 64, "step_time": 250.47715630708262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1020.25, "completions/max_terminated_length": 1020.25, "completions/mean_length": 220.515625, "completions/mean_terminated_length": 220.515625, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.01191511406796053, "epoch": 0.06221408966148216, "frac_reward_zero_std": 0.8125, "grad_norm": 0.062386160477754214, "kl": 0.035652039339765906, "learning_rate": 4.965840347970436e-06, "loss": -0.0145, "num_tokens": 10165112.0, "reward": 2.7982013821601868, "reward_std": 0.34512612223625183, "rewards/reward_fn/mean": 2.7982013821601868, "rewards/reward_fn/std": 0.34512611478567123, "step": 68, "step_time": 243.45551919797435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1169.5, "completions/max_terminated_length": 1169.5, "completions/mean_length": 216.1025390625, "completions/mean_terminated_length": 216.1025390625, "completions/min_length": 66.5, "completions/min_terminated_length": 66.5, "entropy": 0.012621318746823817, "epoch": 0.06587374199451052, "frac_reward_zero_std": 0.8046875, "grad_norm": 0.06076991641355838, "kl": 0.036882418557070196, "learning_rate": 4.96089076536221e-06, "loss": -0.0137, "num_tokens": 10794093.0, "reward": 2.8089590668678284, "reward_std": 0.35318899899721146, "rewards/reward_fn/mean": 2.8089590668678284, "rewards/reward_fn/std": 0.35318901389837265, "step": 72, "step_time": 288.5881523680873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.25, "completions/max_terminated_length": 1078.25, "completions/mean_length": 219.3916015625, "completions/mean_terminated_length": 219.3916015625, "completions/min_length": 77.75, "completions/min_terminated_length": 77.75, "entropy": 0.012647980998735875, "epoch": 0.06953339432753888, "frac_reward_zero_std": 0.77734375, "grad_norm": 0.10469311451857824, "kl": 0.04040143545717001, "learning_rate": 4.9556092480493584e-06, "loss": -0.021, "num_tokens": 11425154.0, "reward": 2.818051338195801, "reward_std": 0.3466835618019104, "rewards/reward_fn/mean": 2.818051338195801, "rewards/reward_fn/std": 0.3466835543513298, "step": 76, "step_time": 260.33293359261006 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 227.54296875, "completions/mean_terminated_length": 227.54296875, "completions/min_length": 81.25, "completions/min_terminated_length": 81.25, "entropy": 0.014289415907114744, "epoch": 0.07319304666056725, "frac_reward_zero_std": 0.80859375, "grad_norm": 0.07665055865966315, "kl": 0.03189453040249646, "learning_rate": 4.949996508423877e-06, "loss": -0.0087, "num_tokens": 12067202.0, "reward": 2.7729015350341797, "reward_std": 0.3783550187945366, "rewards/reward_fn/mean": 2.7729015350341797, "rewards/reward_fn/std": 0.3783550336956978, "step": 80, "step_time": 275.7156659611501 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1096.0, "completions/max_terminated_length": 1096.0, "completions/mean_length": 234.634765625, "completions/mean_terminated_length": 234.634765625, "completions/min_length": 81.25, "completions/min_terminated_length": 81.25, "entropy": 0.016412670316640288, "epoch": 0.0768526989935956, "frac_reward_zero_std": 0.79296875, "grad_norm": 0.0802093480911299, "kl": 0.028840772691182792, "learning_rate": 4.944053303554343e-06, "loss": -0.0118, "num_tokens": 12721888.0, "reward": 2.807867646217346, "reward_std": 0.36286504566669464, "rewards/reward_fn/mean": 2.807867646217346, "rewards/reward_fn/std": 0.36286503076553345, "step": 84, "step_time": 266.78033225191757 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1047.25, "completions/max_terminated_length": 1047.25, "completions/mean_length": 241.9208984375, "completions/mean_terminated_length": 241.9208984375, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 0.016632919432595372, "epoch": 0.08051235132662397, "frac_reward_zero_std": 0.79296875, "grad_norm": 0.08807158873251, "kl": 0.02757147583179176, "learning_rate": 4.937780435083797e-06, "loss": -0.0035, "num_tokens": 13387983.0, "reward": 2.826618194580078, "reward_std": 0.35499829053878784, "rewards/reward_fn/mean": 2.826618194580078, "rewards/reward_fn/std": 0.35499831289052963, "step": 88, "step_time": 253.21191494399682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1000.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 234.90234375, "completions/mean_terminated_length": 234.90234375, "completions/min_length": 82.5, "completions/min_terminated_length": 82.5, "entropy": 0.015686047787312418, "epoch": 0.08417200365965233, "frac_reward_zero_std": 0.77734375, "grad_norm": 0.0902657319512717, "kl": 0.02856659993994981, "learning_rate": 4.931178749121612e-06, "loss": -0.0036, "num_tokens": 14046663.0, "reward": 2.7893744707107544, "reward_std": 0.367780365049839, "rewards/reward_fn/mean": 2.7893744707107544, "rewards/reward_fn/std": 0.3677803725004196, "step": 92, "step_time": 241.46138817537576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1010.25, "completions/max_terminated_length": 1010.25, "completions/mean_length": 233.6337890625, "completions/mean_terminated_length": 233.6337890625, "completions/min_length": 76.5, "completions/min_terminated_length": 76.5, "entropy": 0.016080917441286147, "epoch": 0.0878316559926807, "frac_reward_zero_std": 0.80859375, "grad_norm": 0.10844162037915826, "kl": 0.030640183249488473, "learning_rate": 4.924249136129368e-06, "loss": -0.0016, "num_tokens": 14703728.0, "reward": 2.821086883544922, "reward_std": 0.34664493054151535, "rewards/reward_fn/mean": 2.821086883544922, "rewards/reward_fn/std": 0.34664495289325714, "step": 96, "step_time": 242.6293911980465 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 1350.5, "completions/max_terminated_length": 1142.5, "completions/mean_length": 241.9013671875, "completions/mean_terminated_length": 239.37954330444336, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.01606400089804083, "epoch": 0.09149130832570906, "frac_reward_zero_std": 0.80078125, "grad_norm": 0.10026115815134104, "kl": 0.028314362629316747, "learning_rate": 4.916992530800743e-06, "loss": 0.0104, "num_tokens": 15372987.0, "reward": 2.8225176334381104, "reward_std": 0.35643360018730164, "rewards/reward_fn/mean": 2.8225176334381104, "rewards/reward_fn/std": 0.35643360763788223, "step": 100, "step_time": 353.8205208936706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1125.75, "completions/max_terminated_length": 1125.75, "completions/mean_length": 240.900390625, "completions/mean_terminated_length": 240.900390625, "completions/min_length": 78.5, "completions/min_terminated_length": 78.5, "entropy": 0.01435602194396779, "epoch": 0.09515096065873742, "frac_reward_zero_std": 0.78515625, "grad_norm": 0.07084664638416241, "kl": 0.0261101343203336, "learning_rate": 4.909409911935439e-06, "loss": 0.0157, "num_tokens": 16040689.0, "reward": 2.8153278827667236, "reward_std": 0.3514478802680969, "rewards/reward_fn/mean": 2.8153278827667236, "rewards/reward_fn/std": 0.3514478877186775, "step": 104, "step_time": 281.9691889616661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1062.0, "completions/max_terminated_length": 1062.0, "completions/mean_length": 220.57421875, "completions/mean_terminated_length": 220.57421875, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.013337482989300042, "epoch": 0.09881061299176579, "frac_reward_zero_std": 0.8515625, "grad_norm": 0.04353003522988331, "kl": 0.02664439135696739, "learning_rate": 4.901502302307155e-06, "loss": 0.0034, "num_tokens": 16687393.0, "reward": 2.836458206176758, "reward_std": 0.3151797950267792, "rewards/reward_fn/mean": 2.836458206176758, "rewards/reward_fn/std": 0.3151797950267792, "step": 108, "step_time": 262.62998094735667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1111.5, "completions/max_terminated_length": 1111.5, "completions/mean_length": 232.6630859375, "completions/mean_terminated_length": 232.6630859375, "completions/min_length": 81.25, "completions/min_terminated_length": 81.25, "entropy": 0.013577717880252749, "epoch": 0.10247026532479414, "frac_reward_zero_std": 0.8359375, "grad_norm": 0.06257959896127956, "kl": 0.02790737326722592, "learning_rate": 4.893270768525636e-06, "loss": 0.005, "num_tokens": 17352068.0, "reward": 2.817973017692566, "reward_std": 0.3653535544872284, "rewards/reward_fn/mean": 2.817973017692566, "rewards/reward_fn/std": 0.3653535917401314, "step": 112, "step_time": 279.3403219631873 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1052.0, "completions/max_terminated_length": 1052.0, "completions/mean_length": 226.76953125, "completions/mean_terminated_length": 226.76953125, "completions/min_length": 81.5, "completions/min_terminated_length": 81.5, "entropy": 0.014110724616330117, "epoch": 0.10612991765782251, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.10101762024902548, "kl": 0.03716003953013569, "learning_rate": 4.884716420892796e-06, "loss": 0.0149, "num_tokens": 18014020.0, "reward": 2.846973419189453, "reward_std": 0.33806994557380676, "rewards/reward_fn/mean": 2.846973419189453, "rewards/reward_fn/std": 0.33806995302438736, "step": 116, "step_time": 262.1372203491628 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1113.25, "completions/max_terminated_length": 1113.25, "completions/mean_length": 232.7587890625, "completions/mean_terminated_length": 232.7587890625, "completions/min_length": 82.75, "completions/min_terminated_length": 82.75, "entropy": 0.01280687196413055, "epoch": 0.10978956999085086, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.10160550077935328, "kl": 0.04039534099865705, "learning_rate": 4.875840413252968e-06, "loss": 0.0196, "num_tokens": 18683765.0, "reward": 2.8628095388412476, "reward_std": 0.3213796243071556, "rewards/reward_fn/mean": 2.8628095388412476, "rewards/reward_fn/std": 0.3213796466588974, "step": 120, "step_time": 283.8032908006571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 950.75, "completions/max_terminated_length": 950.75, "completions/mean_length": 221.6962890625, "completions/mean_terminated_length": 221.6962890625, "completions/min_length": 80.5, "completions/min_terminated_length": 80.5, "entropy": 0.013175143743865192, "epoch": 0.11344922232387923, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.06846652654639172, "kl": 0.0442041486967355, "learning_rate": 4.866643942837256e-06, "loss": 0.0068, "num_tokens": 19340998.0, "reward": 2.863952100276947, "reward_std": 0.33952105045318604, "rewards/reward_fn/mean": 2.863952100276947, "rewards/reward_fn/std": 0.3395210728049278, "step": 124, "step_time": 235.00603474816307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1126.0, "completions/max_terminated_length": 1126.0, "completions/mean_length": 222.55078125, "completions/mean_terminated_length": 222.55078125, "completions/min_length": 79.5, "completions/min_terminated_length": 79.5, "entropy": 0.012510814412962645, "epoch": 0.1171088746569076, "frac_reward_zero_std": 0.89453125, "grad_norm": 0.07639946031491095, "kl": 0.04450422874651849, "learning_rate": 4.857128250102057e-06, "loss": 0.0065, "num_tokens": 20002494.0, "reward": 2.8543471097946167, "reward_std": 0.3549715727567673, "rewards/reward_fn/mean": 2.8543471097946167, "rewards/reward_fn/std": 0.35497158020734787, "step": 128, "step_time": 287.90223784698173 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1197.25, "completions/max_terminated_length": 1142.25, "completions/mean_length": 230.3544921875, "completions/mean_terminated_length": 229.08981704711914, "completions/min_length": 81.5, "completions/min_terminated_length": 81.5, "entropy": 0.0119518170831725, "epoch": 0.12076852698993595, "frac_reward_zero_std": 0.8984375, "grad_norm": 0.06323385395208185, "kl": 0.04492187639698386, "learning_rate": 4.8472946185617395e-06, "loss": 0.0015, "num_tokens": 20671653.0, "reward": 2.8591416478157043, "reward_std": 0.33486203104257584, "rewards/reward_fn/mean": 2.8591416478157043, "rewards/reward_fn/std": 0.33486203849315643, "step": 132, "step_time": 315.46063051139936 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1066.75, "completions/max_terminated_length": 1066.75, "completions/mean_length": 223.9208984375, "completions/mean_terminated_length": 223.9208984375, "completions/min_length": 76.75, "completions/min_terminated_length": 76.75, "entropy": 0.01208365970524028, "epoch": 0.12442817932296432, "frac_reward_zero_std": 0.921875, "grad_norm": 0.08297054131204595, "kl": 0.049826212925836444, "learning_rate": 4.83714437461552e-06, "loss": 0.0009, "num_tokens": 21335048.0, "reward": 2.857163429260254, "reward_std": 0.3398366719484329, "rewards/reward_fn/mean": 2.857163429260254, "rewards/reward_fn/std": 0.3398367017507553, "step": 136, "step_time": 271.9807884255424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1044.75, "completions/max_terminated_length": 1044.75, "completions/mean_length": 225.634765625, "completions/mean_terminated_length": 225.634765625, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "entropy": 0.011399393086321652, "epoch": 0.1280878316559927, "frac_reward_zero_std": 0.9453125, "grad_norm": 0.05599980204371085, "kl": 0.049895928939804435, "learning_rate": 4.82667888736855e-06, "loss": 0.0063, "num_tokens": 22007114.0, "reward": 2.8718236088752747, "reward_std": 0.31254760548472404, "rewards/reward_fn/mean": 2.8718236088752747, "rewards/reward_fn/std": 0.31254761293530464, "step": 140, "step_time": 270.8902273066342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1078.25, "completions/max_terminated_length": 1078.25, "completions/mean_length": 228.3505859375, "completions/mean_terminated_length": 228.3505859375, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.012115511810407043, "epoch": 0.13174748398902103, "frac_reward_zero_std": 0.93359375, "grad_norm": 0.06313071464948433, "kl": 0.04882367001846433, "learning_rate": 4.815899568447249e-06, "loss": 0.002, "num_tokens": 22683225.0, "reward": 2.8702903985977173, "reward_std": 0.3183404803276062, "rewards/reward_fn/mean": 2.8702903985977173, "rewards/reward_fn/std": 0.3183404952287674, "step": 144, "step_time": 279.12108469195664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1074.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 232.51953125, "completions/mean_terminated_length": 232.51953125, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.012113740318454802, "epoch": 0.1354071363220494, "frac_reward_zero_std": 0.9296875, "grad_norm": 0.03658778754859278, "kl": 0.04669306240975857, "learning_rate": 4.804807871808896e-06, "loss": 0.0021, "num_tokens": 23365881.0, "reward": 2.8644264340400696, "reward_std": 0.33891405910253525, "rewards/reward_fn/mean": 2.8644264340400696, "rewards/reward_fn/std": 0.33891407400369644, "step": 148, "step_time": 280.8253190875985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1042.75, "completions/max_terminated_length": 1042.75, "completions/mean_length": 229.6142578125, "completions/mean_terminated_length": 229.6142578125, "completions/min_length": 66.5, "completions/min_terminated_length": 66.5, "entropy": 0.012918102380353957, "epoch": 0.13906678865507777, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.02909046433282949, "kl": 0.05268579628318548, "learning_rate": 4.793405293545515e-06, "loss": 0.005, "num_tokens": 24046030.0, "reward": 2.840142250061035, "reward_std": 0.3678673878312111, "rewards/reward_fn/mean": 2.840142250061035, "rewards/reward_fn/std": 0.3678673878312111, "step": 152, "step_time": 270.6633171187714 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1053.25, "completions/max_terminated_length": 1053.25, "completions/mean_length": 228.44140625, "completions/mean_terminated_length": 228.44140625, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.012696940742898732, "epoch": 0.14272644098810613, "frac_reward_zero_std": 0.91796875, "grad_norm": 0.048905338186676375, "kl": 0.048741934821009636, "learning_rate": 4.781693371682078e-06, "loss": 0.0075, "num_tokens": 24728038.0, "reward": 2.8578240871429443, "reward_std": 0.34266316145658493, "rewards/reward_fn/mean": 2.8578240871429443, "rewards/reward_fn/std": 0.3426631763577461, "step": 156, "step_time": 278.5923395641148 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1050.0, "completions/max_terminated_length": 1050.0, "completions/mean_length": 241.5390625, "completions/mean_terminated_length": 241.5390625, "completions/min_length": 79.25, "completions/min_terminated_length": 79.25, "entropy": 0.014673987170681357, "epoch": 0.1463860933211345, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.09602525853563185, "kl": 0.050844733603298664, "learning_rate": 4.769673685969047e-06, "loss": 0.002, "num_tokens": 25425806.0, "reward": 2.8598700165748596, "reward_std": 0.31299157440662384, "rewards/reward_fn/mean": 2.8598700165748596, "rewards/reward_fn/std": 0.31299156695604324, "step": 160, "step_time": 275.66814939584583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1011.25, "completions/max_terminated_length": 1011.25, "completions/mean_length": 216.0478515625, "completions/mean_terminated_length": 216.0478515625, "completions/min_length": 77.25, "completions/min_terminated_length": 77.25, "entropy": 0.01529375184327364, "epoch": 0.15004574565416284, "frac_reward_zero_std": 0.859375, "grad_norm": 0.06508558042156037, "kl": 0.05107123241759837, "learning_rate": 4.757347857669292e-06, "loss": -0.0049, "num_tokens": 26097111.0, "reward": 2.8294225335121155, "reward_std": 0.3605535253882408, "rewards/reward_fn/mean": 2.8294225335121155, "rewards/reward_fn/std": 0.3605535477399826, "step": 164, "step_time": 265.74749380582944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1110.5, "completions/max_terminated_length": 1110.5, "completions/mean_length": 250.458984375, "completions/mean_terminated_length": 250.458984375, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.01382788352202624, "epoch": 0.1537053979871912, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.06539406663732159, "kl": 0.050654102582484484, "learning_rate": 4.744717549339412e-06, "loss": -0.0012, "num_tokens": 26813365.0, "reward": 2.8500934839248657, "reward_std": 0.3334427922964096, "rewards/reward_fn/mean": 2.8500934839248657, "rewards/reward_fn/std": 0.333442784845829, "step": 168, "step_time": 298.08681116625667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1034.5, "completions/max_terminated_length": 1034.5, "completions/mean_length": 230.283203125, "completions/mean_terminated_length": 230.283203125, "completions/min_length": 78.75, "completions/min_terminated_length": 78.75, "entropy": 0.013774615188594908, "epoch": 0.15736505032021958, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.22897135637733165, "kl": 0.05037340638227761, "learning_rate": 4.731784464605474e-06, "loss": 0.0021, "num_tokens": 27510167.0, "reward": 2.8380470871925354, "reward_std": 0.34361911565065384, "rewards/reward_fn/mean": 2.8380470871925354, "rewards/reward_fn/std": 0.34361913055181503, "step": 172, "step_time": 277.122442452237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1013.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 241.236328125, "completions/mean_terminated_length": 241.236328125, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.01372558658476919, "epoch": 0.16102470265324795, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.05383569167271158, "kl": 0.052041274728253484, "learning_rate": 4.718550347933234e-06, "loss": -0.0009, "num_tokens": 28219421.0, "reward": 2.8658467531204224, "reward_std": 0.3258620947599411, "rewards/reward_fn/mean": 2.8658467531204224, "rewards/reward_fn/std": 0.3258620873093605, "step": 176, "step_time": 270.5161182801239 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1019.5, "completions/max_terminated_length": 1019.5, "completions/mean_length": 238.8955078125, "completions/mean_terminated_length": 238.8955078125, "completions/min_length": 80.75, "completions/min_terminated_length": 80.75, "entropy": 0.014220306125935167, "epoch": 0.16468435498627632, "frac_reward_zero_std": 0.84765625, "grad_norm": 0.0919566209583686, "kl": 0.06164557579904795, "learning_rate": 4.705016984392825e-06, "loss": 0.0033, "num_tokens": 28927074.0, "reward": 2.81077641248703, "reward_std": 0.3736064210534096, "rewards/reward_fn/mean": 2.81077641248703, "rewards/reward_fn/std": 0.3736064285039902, "step": 180, "step_time": 278.1804432417266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.5, "completions/max_terminated_length": 1054.5, "completions/mean_length": 245.607421875, "completions/mean_terminated_length": 245.607421875, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.014155362150631845, "epoch": 0.16834400731930466, "frac_reward_zero_std": 0.90625, "grad_norm": 0.0717895655009332, "kl": 0.06093443278223276, "learning_rate": 4.691186199417985e-06, "loss": 0.0029, "num_tokens": 29646528.0, "reward": 2.8060117959976196, "reward_std": 0.4102167412638664, "rewards/reward_fn/mean": 2.8060117959976196, "rewards/reward_fn/std": 0.4102167412638664, "step": 184, "step_time": 287.0040536648594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1221.25, "completions/max_terminated_length": 1221.25, "completions/mean_length": 256.5146484375, "completions/mean_terminated_length": 256.5146484375, "completions/min_length": 77.5, "completions/min_terminated_length": 77.5, "entropy": 0.014777194592170417, "epoch": 0.17200365965233302, "frac_reward_zero_std": 0.82421875, "grad_norm": 0.10308365762658511, "kl": 0.06632018205709755, "learning_rate": 4.677059858559835e-06, "loss": 0.0031, "num_tokens": 30378431.0, "reward": 2.7752703428268433, "reward_std": 0.46188151836395264, "rewards/reward_fn/mean": 2.7752703428268433, "rewards/reward_fn/std": 0.461881548166275, "step": 188, "step_time": 340.33803110197186 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1087.0, "completions/max_terminated_length": 1087.0, "completions/mean_length": 234.787109375, "completions/mean_terminated_length": 234.787109375, "completions/min_length": 80.75, "completions/min_terminated_length": 80.75, "entropy": 0.013827059709001333, "epoch": 0.1756633119853614, "frac_reward_zero_std": 0.84375, "grad_norm": 0.07237213346872136, "kl": 0.0638300587888807, "learning_rate": 4.662639867235244e-06, "loss": 0.0031, "num_tokens": 31089909.0, "reward": 2.8027281761169434, "reward_std": 0.3978399187326431, "rewards/reward_fn/mean": 2.8027281761169434, "rewards/reward_fn/std": 0.3978399336338043, "step": 192, "step_time": 300.94366448652 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1064.0, "completions/max_terminated_length": 1064.0, "completions/mean_length": 255.17578125, "completions/mean_terminated_length": 255.17578125, "completions/min_length": 79.5, "completions/min_terminated_length": 79.5, "entropy": 0.01407104026293382, "epoch": 0.17932296431838976, "frac_reward_zero_std": 0.875, "grad_norm": 0.06382882251270793, "kl": 0.06381136830896139, "learning_rate": 4.647928170469824e-06, "loss": 0.0043, "num_tokens": 31830217.0, "reward": 2.815064787864685, "reward_std": 0.3995492458343506, "rewards/reward_fn/mean": 2.815064787864685, "rewards/reward_fn/std": 0.3995492681860924, "step": 196, "step_time": 295.87483162106946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1163.25, "completions/max_terminated_length": 1163.25, "completions/mean_length": 243.5771484375, "completions/mean_terminated_length": 243.5771484375, "completions/min_length": 81.5, "completions/min_terminated_length": 81.5, "entropy": 0.014106316433753818, "epoch": 0.18298261665141813, "frac_reward_zero_std": 0.8515625, "grad_norm": 0.06695057253435484, "kl": 0.0665284029673785, "learning_rate": 4.632926752635569e-06, "loss": 0.0076, "num_tokens": 32558312.0, "reward": 2.7023602724075317, "reward_std": 0.5262805372476578, "rewards/reward_fn/mean": 2.7023602724075317, "rewards/reward_fn/std": 0.5262805446982384, "step": 200, "step_time": 328.4555319393985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1141.25, "completions/max_terminated_length": 1141.25, "completions/mean_length": 248.427734375, "completions/mean_terminated_length": 248.427734375, "completions/min_length": 81.25, "completions/min_terminated_length": 81.25, "entropy": 0.013146597717422992, "epoch": 0.18664226898444647, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.07539641236005702, "kl": 0.06348171643912792, "learning_rate": 4.617637637183204e-06, "loss": 0.0034, "num_tokens": 33291082.0, "reward": 2.761439323425293, "reward_std": 0.4976746663451195, "rewards/reward_fn/mean": 2.761439323425293, "rewards/reward_fn/std": 0.4976746737957001, "step": 204, "step_time": 319.8018886325881 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1132.5, "completions/max_terminated_length": 1132.5, "completions/mean_length": 240.9658203125, "completions/mean_terminated_length": 240.9658203125, "completions/min_length": 78.25, "completions/min_terminated_length": 78.25, "entropy": 0.015186345437541604, "epoch": 0.19030192131747484, "frac_reward_zero_std": 0.88671875, "grad_norm": 0.03993634279516113, "kl": 0.06433808198198676, "learning_rate": 4.602062886369244e-06, "loss": 0.0099, "num_tokens": 34020671.0, "reward": 2.793409049510956, "reward_std": 0.4567374736070633, "rewards/reward_fn/mean": 2.793409049510956, "rewards/reward_fn/std": 0.4567374810576439, "step": 208, "step_time": 323.2913993848488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1083.0, "completions/max_terminated_length": 1083.0, "completions/mean_length": 244.0146484375, "completions/mean_terminated_length": 244.0146484375, "completions/min_length": 77.75, "completions/min_terminated_length": 77.75, "entropy": 0.014897112385369837, "epoch": 0.1939615736505032, "frac_reward_zero_std": 0.93359375, "grad_norm": 0.03717160672790563, "kl": 0.06326445587910712, "learning_rate": 4.58620460097784e-06, "loss": 0.0018, "num_tokens": 34753822.0, "reward": 2.8207574486732483, "reward_std": 0.3912634551525116, "rewards/reward_fn/mean": 2.8207574486732483, "rewards/reward_fn/std": 0.391263484954834, "step": 212, "step_time": 309.24909047270194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.0, "completions/max_terminated_length": 1077.0, "completions/mean_length": 258.7001953125, "completions/mean_terminated_length": 258.7001953125, "completions/min_length": 77.25, "completions/min_terminated_length": 77.25, "entropy": 0.014759315876290202, "epoch": 0.19762122598353157, "frac_reward_zero_std": 0.890625, "grad_norm": 0.07081269541976444, "kl": 0.059495773166418076, "learning_rate": 4.5700649200374095e-06, "loss": 0.0029, "num_tokens": 35501763.0, "reward": 2.809674561023712, "reward_std": 0.3931515961885452, "rewards/reward_fn/mean": 2.809674561023712, "rewards/reward_fn/std": 0.3931516334414482, "step": 216, "step_time": 301.3579012993723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1171.0, "completions/max_terminated_length": 1040.0, "completions/mean_length": 244.5625, "completions/mean_terminated_length": 243.29501342773438, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.015227734867949039, "epoch": 0.2012808783165599, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.0591513019608395, "kl": 0.06103493459522724, "learning_rate": 4.553646020532114e-06, "loss": 0.0096, "num_tokens": 36239575.0, "reward": 2.8282867074012756, "reward_std": 0.36736876517534256, "rewards/reward_fn/mean": 2.8282867074012756, "rewards/reward_fn/std": 0.36736876517534256, "step": 220, "step_time": 342.628715894185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1115.25, "completions/max_terminated_length": 1115.25, "completions/mean_length": 250.5625, "completions/mean_terminated_length": 250.5625, "completions/min_length": 77.25, "completions/min_terminated_length": 77.25, "entropy": 0.014696637750603259, "epoch": 0.20494053064958828, "frac_reward_zero_std": 0.8984375, "grad_norm": 0.056132077039738944, "kl": 0.05931313335895538, "learning_rate": 4.536950117108226e-06, "loss": 0.0078, "num_tokens": 36990911.0, "reward": 2.8466833233833313, "reward_std": 0.3461478427052498, "rewards/reward_fn/mean": 2.8466833233833313, "rewards/reward_fn/std": 0.3461478427052498, "step": 224, "step_time": 327.5807393779978 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1158.25, "completions/max_terminated_length": 1158.25, "completions/mean_length": 252.7353515625, "completions/mean_terminated_length": 252.7353515625, "completions/min_length": 77.75, "completions/min_terminated_length": 77.75, "entropy": 0.014584530203137547, "epoch": 0.20860018298261665, "frac_reward_zero_std": 0.828125, "grad_norm": 0.0836067486087614, "kl": 0.05628983583301306, "learning_rate": 4.519979461775404e-06, "loss": 0.0138, "num_tokens": 37745484.0, "reward": 2.784464180469513, "reward_std": 0.42813974618911743, "rewards/reward_fn/mean": 2.784464180469513, "rewards/reward_fn/std": 0.4281397685408592, "step": 228, "step_time": 338.8130832114257 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1122.0, "completions/max_terminated_length": 1122.0, "completions/mean_length": 246.970703125, "completions/mean_terminated_length": 246.970703125, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.013940769073087722, "epoch": 0.21225983531564502, "frac_reward_zero_std": 0.84765625, "grad_norm": 0.06107697088317581, "kl": 0.06036040303297341, "learning_rate": 4.502736343602934e-06, "loss": 0.015, "num_tokens": 38494170.0, "reward": 2.8251479864120483, "reward_std": 0.37282292544841766, "rewards/reward_fn/mean": 2.8251479864120483, "rewards/reward_fn/std": 0.37282292544841766, "step": 232, "step_time": 328.3553645936772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1226.0, "completions/max_terminated_length": 1116.0, "completions/mean_length": 263.27734375, "completions/mean_terminated_length": 262.0295715332031, "completions/min_length": 78.25, "completions/min_terminated_length": 78.25, "entropy": 0.01664691447513178, "epoch": 0.21591948764867339, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.07936710594166071, "kl": 0.057622806169092655, "learning_rate": 4.485223088410967e-06, "loss": 0.0085, "num_tokens": 39264842.0, "reward": 2.824228823184967, "reward_std": 0.37058399617671967, "rewards/reward_fn/mean": 2.824228823184967, "rewards/reward_fn/std": 0.37058401107788086, "step": 236, "step_time": 371.6977700777352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1304.0, "completions/max_terminated_length": 1304.0, "completions/mean_length": 265.8642578125, "completions/mean_terminated_length": 265.8642578125, "completions/min_length": 78.5, "completions/min_terminated_length": 78.5, "entropy": 0.02004530024714768, "epoch": 0.21957913998170173, "frac_reward_zero_std": 0.828125, "grad_norm": 0.05728495119096045, "kl": 0.05499382968991995, "learning_rate": 4.4674420584568105e-06, "loss": 0.0167, "num_tokens": 40041699.0, "reward": 2.815194547176361, "reward_std": 0.3625386580824852, "rewards/reward_fn/mean": 2.815194547176361, "rewards/reward_fn/std": 0.3625386580824852, "step": 240, "step_time": 398.28707833588123 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1201.5, "completions/max_terminated_length": 1065.5, "completions/mean_length": 245.466796875, "completions/mean_terminated_length": 244.22003173828125, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.018354313855525106, "epoch": 0.2232387923147301, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.06205789770558449, "kl": 0.0634293460752815, "learning_rate": 4.449395652116291e-06, "loss": 0.0253, "num_tokens": 40798509.0, "reward": 2.81777161359787, "reward_std": 0.3663923665881157, "rewards/reward_fn/mean": 2.81777161359787, "rewards/reward_fn/std": 0.3663923814892769, "step": 244, "step_time": 360.5024783038534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 920.5, "completions/max_terminated_length": 920.5, "completions/mean_length": 231.9951171875, "completions/mean_terminated_length": 231.9951171875, "completions/min_length": 82.75, "completions/min_terminated_length": 82.75, "entropy": 0.017063919513020664, "epoch": 0.22689844464775846, "frac_reward_zero_std": 0.91015625, "grad_norm": 0.07078082856668842, "kl": 0.07647754997015, "learning_rate": 4.431086303560256e-06, "loss": 0.0167, "num_tokens": 41546504.0, "reward": 2.7994924187660217, "reward_std": 0.40477121621370316, "rewards/reward_fn/mean": 2.7994924187660217, "rewards/reward_fn/std": 0.40477124601602554, "step": 248, "step_time": 268.8032052577473 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1013.25, "completions/max_terminated_length": 1013.25, "completions/mean_length": 233.9052734375, "completions/mean_terminated_length": 233.9052734375, "completions/min_length": 82.75, "completions/min_terminated_length": 82.75, "entropy": 0.017987663624808192, "epoch": 0.23055809698078683, "frac_reward_zero_std": 0.8671875, "grad_norm": 0.1563436187383896, "kl": 0.08293303567916155, "learning_rate": 4.412516482426241e-06, "loss": 0.0093, "num_tokens": 42303207.0, "reward": 2.8115214109420776, "reward_std": 0.3961891457438469, "rewards/reward_fn/mean": 2.8115214109420776, "rewards/reward_fn/std": 0.3961891755461693, "step": 252, "step_time": 305.94378135027364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 976.75, "completions/max_terminated_length": 976.75, "completions/mean_length": 241.396484375, "completions/mean_terminated_length": 241.396484375, "completions/min_length": 71.25, "completions/min_terminated_length": 71.25, "entropy": 0.01701750111533329, "epoch": 0.2342177493138152, "frac_reward_zero_std": 0.890625, "grad_norm": 0.07228795401138102, "kl": 0.07505286997184157, "learning_rate": 4.393688693485355e-06, "loss": 0.0043, "num_tokens": 43068741.0, "reward": 2.8061317801475525, "reward_std": 0.39476943761110306, "rewards/reward_fn/mean": 2.8061317801475525, "rewards/reward_fn/std": 0.39476945251226425, "step": 256, "step_time": 290.7374283745885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1056.75, "completions/max_terminated_length": 1056.75, "completions/mean_length": 239.0703125, "completions/mean_terminated_length": 239.0703125, "completions/min_length": 81.5, "completions/min_terminated_length": 81.5, "entropy": 0.016208724409807473, "epoch": 0.23787740164684354, "frac_reward_zero_std": 0.88671875, "grad_norm": 0.06165748716249506, "kl": 0.07173356646671891, "learning_rate": 4.374605476304431e-06, "loss": 0.0088, "num_tokens": 43830085.0, "reward": 2.839516520500183, "reward_std": 0.34048842638731003, "rewards/reward_fn/mean": 2.839516520500183, "rewards/reward_fn/std": 0.3404884412884712, "step": 260, "step_time": 314.90353200305253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1060.5, "completions/max_terminated_length": 1060.5, "completions/mean_length": 238.794921875, "completions/mean_terminated_length": 238.794921875, "completions/min_length": 80.75, "completions/min_terminated_length": 80.75, "entropy": 0.015883819665759802, "epoch": 0.2415370539798719, "frac_reward_zero_std": 0.95703125, "grad_norm": 0.015264229481008371, "kl": 0.06516523798927665, "learning_rate": 4.355269404903469e-06, "loss": 0.0026, "num_tokens": 44596127.0, "reward": 2.820503354072571, "reward_std": 0.3636850267648697, "rewards/reward_fn/mean": 2.820503354072571, "rewards/reward_fn/std": 0.3636850491166115, "step": 264, "step_time": 324.4773340257816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1088.5, "completions/max_terminated_length": 1088.5, "completions/mean_length": 235.8466796875, "completions/mean_terminated_length": 235.8466796875, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.015809061413165182, "epoch": 0.24519670631290028, "frac_reward_zero_std": 0.9453125, "grad_norm": 0.02256531674816499, "kl": 0.06406322331167758, "learning_rate": 4.335683087408456e-06, "loss": 0.0065, "num_tokens": 45358850.0, "reward": 2.824212431907654, "reward_std": 0.38692566752433777, "rewards/reward_fn/mean": 2.824212431907654, "rewards/reward_fn/std": 0.38692569732666016, "step": 268, "step_time": 333.7911497359164 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1086.75, "completions/max_terminated_length": 1086.75, "completions/mean_length": 246.55078125, "completions/mean_terminated_length": 246.55078125, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.015495246276259422, "epoch": 0.24885635864592864, "frac_reward_zero_std": 0.9375, "grad_norm": 0.04421854218025354, "kl": 0.06460057967342436, "learning_rate": 4.315849165699562e-06, "loss": 0.0077, "num_tokens": 46132358.0, "reward": 2.8535637259483337, "reward_std": 0.34523719549179077, "rewards/reward_fn/mean": 2.8535637259483337, "rewards/reward_fn/std": 0.34523719549179077, "step": 272, "step_time": 333.43558596400544 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1054.0, "completions/max_terminated_length": 1054.0, "completions/mean_length": 241.197265625, "completions/mean_terminated_length": 241.197265625, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.015473014034796506, "epoch": 0.252516010978957, "frac_reward_zero_std": 0.96484375, "grad_norm": 0.013480684856473399, "kl": 0.06405930104665458, "learning_rate": 4.295770315054792e-06, "loss": 0.0035, "num_tokens": 46907528.0, "reward": 2.851532220840454, "reward_std": 0.33368974179029465, "rewards/reward_fn/mean": 2.851532220840454, "rewards/reward_fn/std": 0.33368974924087524, "step": 276, "step_time": 325.99443143745884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1059.5, "completions/max_terminated_length": 1059.5, "completions/mean_length": 240.9228515625, "completions/mean_terminated_length": 240.9228515625, "completions/min_length": 80.75, "completions/min_terminated_length": 80.75, "entropy": 0.01620938949054107, "epoch": 0.2561756633119854, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.0384191494606653, "kl": 0.06601658696308732, "learning_rate": 4.275449243789141e-06, "loss": 0.0087, "num_tokens": 47687061.0, "reward": 2.8018744587898254, "reward_std": 0.42315705120563507, "rewards/reward_fn/mean": 2.8018744587898254, "rewards/reward_fn/std": 0.42315705865621567, "step": 280, "step_time": 334.94401907455176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1138.75, "completions/max_terminated_length": 1138.75, "completions/mean_length": 245.0146484375, "completions/mean_terminated_length": 245.0146484375, "completions/min_length": 77.75, "completions/min_terminated_length": 77.75, "entropy": 0.01545735343825072, "epoch": 0.2598353156450137, "frac_reward_zero_std": 0.91015625, "grad_norm": 0.043103819404576645, "kl": 0.06110293813981116, "learning_rate": 4.254888692889283e-06, "loss": 0.0063, "num_tokens": 48471708.0, "reward": 2.8103264570236206, "reward_std": 0.40437929332256317, "rewards/reward_fn/mean": 2.8103264570236206, "rewards/reward_fn/std": 0.40437931567430496, "step": 284, "step_time": 357.6546063958667 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1228.25, "completions/max_terminated_length": 1228.25, "completions/mean_length": 256.615234375, "completions/mean_terminated_length": 256.615234375, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.01589210593374446, "epoch": 0.26349496797804206, "frac_reward_zero_std": 0.9453125, "grad_norm": 0.03047478394577322, "kl": 0.0623739012517035, "learning_rate": 4.234091435643848e-06, "loss": 0.0038, "num_tokens": 49267282.0, "reward": 2.813932716846466, "reward_std": 0.37685296684503555, "rewards/reward_fn/mean": 2.813932716846466, "rewards/reward_fn/std": 0.37685298919677734, "step": 288, "step_time": 393.72843163786456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1007.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 247.5576171875, "completions/mean_terminated_length": 247.5576171875, "completions/min_length": 82.75, "completions/min_terminated_length": 82.75, "entropy": 0.01576699677389115, "epoch": 0.26715462031107046, "frac_reward_zero_std": 0.9296875, "grad_norm": 0.03088838040322047, "kl": 0.06363737187348306, "learning_rate": 4.213060277269364e-06, "loss": 0.0055, "num_tokens": 50061077.0, "reward": 2.8067115545272827, "reward_std": 0.4056583568453789, "rewards/reward_fn/mean": 2.8067115545272827, "rewards/reward_fn/std": 0.4056583642959595, "step": 292, "step_time": 317.5303096845746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1094.75, "completions/max_terminated_length": 1094.75, "completions/mean_length": 250.1669921875, "completions/mean_terminated_length": 250.1669921875, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.01639867457561195, "epoch": 0.2708142726440988, "frac_reward_zero_std": 0.921875, "grad_norm": 0.041079727814531536, "kl": 0.05844363057985902, "learning_rate": 4.191798054531868e-06, "loss": 0.0086, "num_tokens": 50857108.0, "reward": 2.836637794971466, "reward_std": 0.3569115251302719, "rewards/reward_fn/mean": 2.836637794971466, "rewards/reward_fn/std": 0.3569115325808525, "step": 296, "step_time": 351.88712799642235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1052.25, "completions/max_terminated_length": 1052.25, "completions/mean_length": 244.1083984375, "completions/mean_terminated_length": 244.1083984375, "completions/min_length": 80.75, "completions/min_terminated_length": 80.75, "entropy": 0.01703868998447433, "epoch": 0.2744739249771272, "frac_reward_zero_std": 0.9296875, "grad_norm": 0.024799921261213508, "kl": 0.055512375896796584, "learning_rate": 4.170307635364272e-06, "loss": 0.0052, "num_tokens": 51646531.0, "reward": 2.802725374698639, "reward_std": 0.39312127232551575, "rewards/reward_fn/mean": 2.802725374698639, "rewards/reward_fn/std": 0.39312130957841873, "step": 300, "step_time": 332.61469624424353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1048.75, "completions/max_terminated_length": 1048.75, "completions/mean_length": 243.427734375, "completions/mean_terminated_length": 243.427734375, "completions/min_length": 80.5, "completions/min_terminated_length": 80.5, "entropy": 0.01662707404466346, "epoch": 0.27813357731015553, "frac_reward_zero_std": 0.9375, "grad_norm": 0.026354343662826192, "kl": 0.05272407038137317, "learning_rate": 4.148591918479531e-06, "loss": 0.0038, "num_tokens": 52444925.0, "reward": 2.820052146911621, "reward_std": 0.3816464841365814, "rewards/reward_fn/mean": 2.820052146911621, "rewards/reward_fn/std": 0.3816464990377426, "step": 304, "step_time": 338.50240997457877 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1215.75, "completions/max_terminated_length": 1215.75, "completions/mean_length": 257.8525390625, "completions/mean_terminated_length": 257.8525390625, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.016469117137603462, "epoch": 0.2817932296431839, "frac_reward_zero_std": 0.8984375, "grad_norm": 0.0575717215341458, "kl": 0.05287824175320566, "learning_rate": 4.126653832979647e-06, "loss": 0.013, "num_tokens": 53260738.0, "reward": 2.821549654006958, "reward_std": 0.38622572273015976, "rewards/reward_fn/mean": 2.821549654006958, "rewards/reward_fn/std": 0.38622572273015976, "step": 308, "step_time": 403.6054828152992 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.25, "completions/max_terminated_length": 1117.25, "completions/mean_length": 255.22265625, "completions/mean_terminated_length": 255.22265625, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.016801700461655855, "epoch": 0.28545288197621227, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.05338408530793631, "kl": 0.052849456667900085, "learning_rate": 4.104496337960583e-06, "loss": 0.0091, "num_tokens": 54075526.0, "reward": 2.7818004488945007, "reward_std": 0.4278130531311035, "rewards/reward_fn/mean": 2.7818004488945007, "rewards/reward_fn/std": 0.4278130829334259, "step": 312, "step_time": 366.00835085008293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1110.75, "completions/max_terminated_length": 1110.75, "completions/mean_length": 248.7255859375, "completions/mean_terminated_length": 248.7255859375, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.018975069804582745, "epoch": 0.2891125343092406, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.1664088784287396, "kl": 0.05820453749038279, "learning_rate": 4.08212242211313e-06, "loss": 0.0133, "num_tokens": 54889625.0, "reward": 2.7843167185783386, "reward_std": 0.41212283819913864, "rewards/reward_fn/mean": 2.7843167185783386, "rewards/reward_fn/std": 0.41212283819913864, "step": 316, "step_time": 365.95128867262974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1123.75, "completions/max_terminated_length": 1123.75, "completions/mean_length": 254.044921875, "completions/mean_terminated_length": 254.044921875, "completions/min_length": 80.0, "completions/min_terminated_length": 80.0, "entropy": 0.019178961927536875, "epoch": 0.292772186642269, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.04633976278879389, "kl": 0.05333703476935625, "learning_rate": 4.059535103319777e-06, "loss": 0.0052, "num_tokens": 55711011.0, "reward": 2.7893587946891785, "reward_std": 0.4591749906539917, "rewards/reward_fn/mean": 2.7893587946891785, "rewards/reward_fn/std": 0.4591749981045723, "step": 320, "step_time": 375.5183221241459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1257.0, "completions/max_terminated_length": 1257.0, "completions/mean_length": 255.3662109375, "completions/mean_terminated_length": 255.3662109375, "completions/min_length": 76.5, "completions/min_terminated_length": 76.5, "entropy": 0.01964702148688957, "epoch": 0.29643183897529735, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.07176312284666939, "kl": 0.0537005455698818, "learning_rate": 4.036737428247648e-06, "loss": 0.0124, "num_tokens": 56535790.0, "reward": 2.75516414642334, "reward_std": 0.47900988161563873, "rewards/reward_fn/mean": 2.75516414642334, "rewards/reward_fn/std": 0.4790099114179611, "step": 324, "step_time": 421.30054724914953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1111.25, "completions/max_terminated_length": 1111.25, "completions/mean_length": 259.1708984375, "completions/mean_terminated_length": 259.1708984375, "completions/min_length": 84.25, "completions/min_terminated_length": 84.25, "entropy": 0.01968019304331392, "epoch": 0.3000914913083257, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.04273386196110684, "kl": 0.05436060787178576, "learning_rate": 4.013732471937557e-06, "loss": 0.0113, "num_tokens": 57362565.0, "reward": 2.7828410267829895, "reward_std": 0.4619806557893753, "rewards/reward_fn/mean": 2.7828410267829895, "rewards/reward_fn/std": 0.4619806706905365, "step": 328, "step_time": 370.9271548213437 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1077.75, "completions/max_terminated_length": 1077.75, "completions/mean_length": 248.6767578125, "completions/mean_terminated_length": 248.6767578125, "completions/min_length": 81.5, "completions/min_terminated_length": 81.5, "entropy": 0.020743513945490122, "epoch": 0.3037511436413541, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.18401126487271094, "kl": 0.05533292470499873, "learning_rate": 3.9905233373892316e-06, "loss": 0.0109, "num_tokens": 58194178.0, "reward": 2.8100368976593018, "reward_std": 0.4278334304690361, "rewards/reward_fn/mean": 2.8100368976593018, "rewards/reward_fn/std": 0.4278334453701973, "step": 332, "step_time": 368.58857105113566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1095.0, "completions/max_terminated_length": 1095.0, "completions/mean_length": 255.9384765625, "completions/mean_terminated_length": 255.9384765625, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.02218158240430057, "epoch": 0.3074107959743824, "frac_reward_zero_std": 0.8359375, "grad_norm": 0.08064705887787386, "kl": 0.057683263905346394, "learning_rate": 3.9671131551427726e-06, "loss": 0.0171, "num_tokens": 59029263.0, "reward": 2.756513297557831, "reward_std": 0.4825749918818474, "rewards/reward_fn/mean": 2.756513297557831, "rewards/reward_fn/std": 0.4825750067830086, "step": 336, "step_time": 372.0321779474616 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1227.75, "completions/max_terminated_length": 1127.25, "completions/mean_length": 259.9306640625, "completions/mean_terminated_length": 258.67418670654297, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.020468115573748946, "epoch": 0.3110704483074108, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.06172113304563455, "kl": 0.05510277720168233, "learning_rate": 3.943505082856389e-06, "loss": 0.014, "num_tokens": 59871908.0, "reward": 2.770124852657318, "reward_std": 0.47307033091783524, "rewards/reward_fn/mean": 2.770124852657318, "rewards/reward_fn/std": 0.473070353269577, "step": 340, "step_time": 425.1579443984665 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 1364.5, "completions/max_terminated_length": 1336.25, "completions/mean_length": 258.8916015625, "completions/mean_terminated_length": 256.39228439331055, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.01920273172436282, "epoch": 0.31473010064043916, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.05523480707673635, "kl": 0.054923647781834006, "learning_rate": 3.919702304880485e-06, "loss": 0.0101, "num_tokens": 60718497.0, "reward": 2.79230135679245, "reward_std": 0.4282824918627739, "rewards/reward_fn/mean": 2.79230135679245, "rewards/reward_fn/std": 0.4282825067639351, "step": 344, "step_time": 487.7186696450226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1177.5, "completions/max_terminated_length": 1177.5, "completions/mean_length": 258.7548828125, "completions/mean_terminated_length": 258.7548828125, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 0.01964573678560555, "epoch": 0.3183897529734675, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.052307460272456544, "kl": 0.05672413087449968, "learning_rate": 3.895708031828138e-06, "loss": 0.0061, "num_tokens": 61563858.0, "reward": 2.815351724624634, "reward_std": 0.3556395545601845, "rewards/reward_fn/mean": 2.815351724624634, "rewards/reward_fn/std": 0.35563959181308746, "step": 348, "step_time": 412.0264103813097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.001953125, "completions/max_length": 1217.75, "completions/max_terminated_length": 1126.5, "completions/mean_length": 256.095703125, "completions/mean_terminated_length": 253.58877563476562, "completions/min_length": 82.25, "completions/min_terminated_length": 82.25, "entropy": 0.02070168766658753, "epoch": 0.3220494053064959, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.04768525752073658, "kl": 0.055517966859042645, "learning_rate": 3.871525500142044e-06, "loss": 0.0141, "num_tokens": 62409304.0, "reward": 2.8017632961273193, "reward_std": 0.39498939365148544, "rewards/reward_fn/mean": 2.8017632961273193, "rewards/reward_fn/std": 0.39498940855264664, "step": 352, "step_time": 427.9275800297037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1246.25, "completions/max_terminated_length": 1246.25, "completions/mean_length": 266.845703125, "completions/mean_terminated_length": 266.845703125, "completions/min_length": 76.0, "completions/min_terminated_length": 76.0, "entropy": 0.02018115541432053, "epoch": 0.32570905763952424, "frac_reward_zero_std": 0.90625, "grad_norm": 0.05805445549946706, "kl": 0.05352379777468741, "learning_rate": 3.847157971657966e-06, "loss": 0.0165, "num_tokens": 63266074.0, "reward": 2.7965877056121826, "reward_std": 0.3831140995025635, "rewards/reward_fn/mean": 2.7965877056121826, "rewards/reward_fn/std": 0.38311412930488586, "step": 356, "step_time": 443.1046460606158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1031.0, "completions/max_terminated_length": 1031.0, "completions/mean_length": 248.220703125, "completions/mean_terminated_length": 248.220703125, "completions/min_length": 76.5, "completions/min_terminated_length": 76.5, "entropy": 0.02002508647274226, "epoch": 0.32936870997255263, "frac_reward_zero_std": 0.921875, "grad_norm": 0.03867933820199063, "kl": 0.05510533833876252, "learning_rate": 3.822608733164773e-06, "loss": 0.0059, "num_tokens": 64114416.0, "reward": 2.8273446559906006, "reward_std": 0.3517664670944214, "rewards/reward_fn/mean": 2.8273446559906006, "rewards/reward_fn/std": 0.3517664894461632, "step": 360, "step_time": 365.8222716213204 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1103.25, "completions/max_terminated_length": 1103.25, "completions/mean_length": 252.4033203125, "completions/mean_terminated_length": 252.4033203125, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.021197067166212946, "epoch": 0.33302836230558097, "frac_reward_zero_std": 0.8984375, "grad_norm": 0.08004850682857483, "kl": 0.06018217862583697, "learning_rate": 3.7978810959610963e-06, "loss": 0.0113, "num_tokens": 64967709.0, "reward": 2.8053765892982483, "reward_std": 0.4107788950204849, "rewards/reward_fn/mean": 2.8053765892982483, "rewards/reward_fn/std": 0.4107789024710655, "step": 364, "step_time": 390.9309317241423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1106.5, "completions/max_terminated_length": 1106.5, "completions/mean_length": 249.0361328125, "completions/mean_terminated_length": 249.0361328125, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.022055889829061925, "epoch": 0.3366880146386093, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.052541789630115916, "kl": 0.05786171229556203, "learning_rate": 3.7729783954086957e-06, "loss": 0.0108, "num_tokens": 65820186.0, "reward": 2.7696725726127625, "reward_std": 0.4212775230407715, "rewards/reward_fn/mean": 2.7696725726127625, "rewards/reward_fn/std": 0.4212775304913521, "step": 368, "step_time": 390.50815150514245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1112.75, "completions/max_terminated_length": 1112.75, "completions/mean_length": 248.923828125, "completions/mean_terminated_length": 248.923828125, "completions/min_length": 80.5, "completions/min_terminated_length": 80.5, "entropy": 0.02093490824336186, "epoch": 0.3403476669716377, "frac_reward_zero_std": 0.875, "grad_norm": 0.0593891232691319, "kl": 0.05624948791228235, "learning_rate": 3.7479039904825687e-06, "loss": 0.0139, "num_tokens": 66679968.0, "reward": 2.749756693840027, "reward_std": 0.4831894710659981, "rewards/reward_fn/mean": 2.749756693840027, "rewards/reward_fn/std": 0.4831894785165787, "step": 372, "step_time": 401.47150969458744 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1084.75, "completions/max_terminated_length": 1084.75, "completions/mean_length": 260.5693359375, "completions/mean_terminated_length": 260.5693359375, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "entropy": 0.020609806873835623, "epoch": 0.34400731930466605, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.05883873174274732, "kl": 0.055998800322413445, "learning_rate": 3.722661263317878e-06, "loss": 0.009, "num_tokens": 67547627.0, "reward": 2.733439564704895, "reward_std": 0.464918315410614, "rewards/reward_fn/mean": 2.733439564704895, "rewards/reward_fn/std": 0.464918352663517, "step": 376, "step_time": 388.22158924490213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1015.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 256.7197265625, "completions/mean_terminated_length": 256.7197265625, "completions/min_length": 76.25, "completions/min_terminated_length": 76.25, "entropy": 0.020553262438625097, "epoch": 0.34766697163769444, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.04900971293316462, "kl": 0.053110834909603, "learning_rate": 3.697253618753761e-06, "loss": 0.0032, "num_tokens": 68414508.0, "reward": 2.7533047795295715, "reward_std": 0.46282485127449036, "rewards/reward_fn/mean": 2.7533047795295715, "rewards/reward_fn/std": 0.46282486617565155, "step": 380, "step_time": 366.4996922570281 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1063.0, "completions/max_terminated_length": 1063.0, "completions/mean_length": 250.87109375, "completions/mean_terminated_length": 250.87109375, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.02080925728660077, "epoch": 0.3513266239707228, "frac_reward_zero_std": 0.86328125, "grad_norm": 0.04947422229421151, "kl": 0.05815490847453475, "learning_rate": 3.6716844838740644e-06, "loss": 0.011, "num_tokens": 69279260.0, "reward": 2.745911419391632, "reward_std": 0.45054207742214203, "rewards/reward_fn/mean": 2.745911419391632, "rewards/reward_fn/std": 0.4505421072244644, "step": 384, "step_time": 383.8784329756163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1050.0, "completions/max_terminated_length": 1050.0, "completions/mean_length": 255.9560546875, "completions/mean_terminated_length": 255.9560546875, "completions/min_length": 79.5, "completions/min_terminated_length": 79.5, "entropy": 0.020128908683545887, "epoch": 0.3549862763037511, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.033508899160515984, "kl": 0.05457201926037669, "learning_rate": 3.6459573075450904e-06, "loss": 0.005, "num_tokens": 70162583.0, "reward": 2.758453071117401, "reward_std": 0.47200874239206314, "rewards/reward_fn/mean": 2.758453071117401, "rewards/reward_fn/std": 0.47200877219438553, "step": 388, "step_time": 387.83291961671785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.5, "completions/max_terminated_length": 1117.5, "completions/mean_length": 266.169921875, "completions/mean_terminated_length": 266.169921875, "completions/min_length": 81.75, "completions/min_terminated_length": 81.75, "entropy": 0.022365455282852054, "epoch": 0.3586459286367795, "frac_reward_zero_std": 0.87109375, "grad_norm": 0.08663973531428495, "kl": 0.055080414516851306, "learning_rate": 3.6200755599503987e-06, "loss": 0.0246, "num_tokens": 71052317.0, "reward": 2.767097592353821, "reward_std": 0.4057338833808899, "rewards/reward_fn/mean": 2.767097592353821, "rewards/reward_fn/std": 0.4057339131832123, "step": 392, "step_time": 412.0218286192976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1055.0, "completions/max_terminated_length": 1055.0, "completions/mean_length": 262.7802734375, "completions/mean_terminated_length": 262.7802734375, "completions/min_length": 74.25, "completions/min_terminated_length": 74.25, "entropy": 0.023916912963613868, "epoch": 0.36230558096980786, "frac_reward_zero_std": 0.8359375, "grad_norm": 0.10456738086214372, "kl": 0.055561956483870745, "learning_rate": 3.594042732122735e-06, "loss": 0.02, "num_tokens": 71947252.0, "reward": 2.746975362300873, "reward_std": 0.44752104580402374, "rewards/reward_fn/mean": 2.746975362300873, "rewards/reward_fn/std": 0.44752106815576553, "step": 396, "step_time": 388.2638799441047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1113.0, "completions/max_terminated_length": 1113.0, "completions/mean_length": 255.6689453125, "completions/mean_terminated_length": 255.6689453125, "completions/min_length": 84.0, "completions/min_terminated_length": 84.0, "entropy": 0.023536766064353287, "epoch": 0.36596523330283626, "frac_reward_zero_std": 0.81640625, "grad_norm": 0.07529446863056388, "kl": 0.059099239530041814, "learning_rate": 3.567862335473144e-06, "loss": 0.0172, "num_tokens": 72839781.0, "reward": 2.743981719017029, "reward_std": 0.4595012813806534, "rewards/reward_fn/mean": 2.743981719017029, "rewards/reward_fn/std": 0.45950130373239517, "step": 400, "step_time": 425.3582200766541 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1169.5, "completions/max_terminated_length": 1169.5, "completions/mean_length": 264.8642578125, "completions/mean_terminated_length": 264.8642578125, "completions/min_length": 76.25, "completions/min_terminated_length": 76.25, "entropy": 0.021931169438175857, "epoch": 0.3696248856358646, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.06680599370876592, "kl": 0.06040530768223107, "learning_rate": 3.541537901317338e-06, "loss": 0.0248, "num_tokens": 73741594.0, "reward": 2.7557645440101624, "reward_std": 0.44654303044080734, "rewards/reward_fn/mean": 2.7557645440101624, "rewards/reward_fn/std": 0.44654303789138794, "step": 404, "step_time": 441.0422842912376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1128.0, "completions/max_terminated_length": 1128.0, "completions/mean_length": 252.75, "completions/mean_terminated_length": 252.75, "completions/min_length": 79.25, "completions/min_terminated_length": 79.25, "entropy": 0.020903860335238278, "epoch": 0.37328453796889294, "frac_reward_zero_std": 0.88671875, "grad_norm": 0.04185240259764596, "kl": 0.057490175822749734, "learning_rate": 3.5150729803993822e-06, "loss": 0.0072, "num_tokens": 74630742.0, "reward": 2.7497737407684326, "reward_std": 0.440328411757946, "rewards/reward_fn/mean": 2.7497737407684326, "rewards/reward_fn/std": 0.4403284415602684, "step": 408, "step_time": 428.5058938455768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1261.5, "completions/max_terminated_length": 1242.0, "completions/mean_length": 256.89453125, "completions/mean_terminated_length": 255.64898681640625, "completions/min_length": 82.0, "completions/min_terminated_length": 82.0, "entropy": 0.020710848970338702, "epoch": 0.37694419030192133, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.06088040163935032, "kl": 0.058197865495458245, "learning_rate": 3.4884711424127505e-06, "loss": 0.0096, "num_tokens": 75530950.0, "reward": 2.7645320892333984, "reward_std": 0.43489429354667664, "rewards/reward_fn/mean": 2.7645320892333984, "rewards/reward_fn/std": 0.43489427864551544, "step": 412, "step_time": 489.43961844732985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1102.5, "completions/max_terminated_length": 1102.5, "completions/mean_length": 246.728515625, "completions/mean_terminated_length": 246.728515625, "completions/min_length": 80.25, "completions/min_terminated_length": 80.25, "entropy": 0.020525632542558014, "epoch": 0.3806038426349497, "frac_reward_zero_std": 0.88671875, "grad_norm": 0.05255012893742152, "kl": 0.05784119013696909, "learning_rate": 3.461735975518836e-06, "loss": 0.0074, "num_tokens": 76428300.0, "reward": 2.7354435324668884, "reward_std": 0.5114951506257057, "rewards/reward_fn/mean": 2.7354435324668884, "rewards/reward_fn/std": 0.5114951729774475, "step": 416, "step_time": 426.42004963662475 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1093.5, "completions/max_terminated_length": 1093.5, "completions/mean_length": 249.1689453125, "completions/mean_terminated_length": 249.1689453125, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "entropy": 0.02230722492095083, "epoch": 0.38426349496797807, "frac_reward_zero_std": 0.87109375, "grad_norm": 0.042090748623265346, "kl": 0.06311949389055371, "learning_rate": 3.4348710858629626e-06, "loss": 0.0061, "num_tokens": 77327597.0, "reward": 2.7153448462486267, "reward_std": 0.5665776580572128, "rewards/reward_fn/mean": 2.7153448462486267, "rewards/reward_fn/std": 0.5665776580572128, "step": 420, "step_time": 423.4696046109311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1085.0, "completions/max_terminated_length": 1085.0, "completions/mean_length": 261.4853515625, "completions/mean_terminated_length": 261.4853515625, "completions/min_length": 78.25, "completions/min_terminated_length": 78.25, "entropy": 0.02214888739399612, "epoch": 0.3879231473010064, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.06477089541532775, "kl": 0.05813350295647979, "learning_rate": 3.4078800970879742e-06, "loss": 0.0032, "num_tokens": 78240558.0, "reward": 2.7490416765213013, "reward_std": 0.4679771661758423, "rewards/reward_fn/mean": 2.7490416765213013, "rewards/reward_fn/std": 0.4679771810770035, "step": 424, "step_time": 418.0459654238075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1153.0, "completions/max_terminated_length": 1153.0, "completions/mean_length": 256.8740234375, "completions/mean_terminated_length": 256.8740234375, "completions/min_length": 65.75, "completions/min_terminated_length": 65.75, "entropy": 0.022280778735876083, "epoch": 0.39158279963403475, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.05071083161625681, "kl": 0.05995195545256138, "learning_rate": 3.3807666498454637e-06, "loss": 0.0113, "num_tokens": 79158477.0, "reward": 2.763080358505249, "reward_std": 0.4363250806927681, "rewards/reward_fn/mean": 2.763080358505249, "rewards/reward_fn/std": 0.4363251179456711, "step": 428, "step_time": 458.79944900562987 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1053.75, "completions/max_terminated_length": 1053.75, "completions/mean_length": 258.5673828125, "completions/mean_terminated_length": 258.5673828125, "completions/min_length": 73.25, "completions/min_terminated_length": 73.25, "entropy": 0.022120147943496704, "epoch": 0.39524245196706315, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.041363944327058255, "kl": 0.059779631439596415, "learning_rate": 3.3535344013047107e-06, "loss": 0.0103, "num_tokens": 80074798.0, "reward": 2.7749913930892944, "reward_std": 0.4496825709939003, "rewards/reward_fn/mean": 2.7749913930892944, "rewards/reward_fn/std": 0.4496825933456421, "step": 432, "step_time": 413.0009705722332 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1176.0, "completions/max_terminated_length": 1176.0, "completions/mean_length": 254.3369140625, "completions/mean_terminated_length": 254.3369140625, "completions/min_length": 79.75, "completions/min_terminated_length": 79.75, "entropy": 0.021958659519441426, "epoch": 0.3989021043000915, "frac_reward_zero_std": 0.91796875, "grad_norm": 0.029236661292790365, "kl": 0.05681233340874314, "learning_rate": 3.3261870246593804e-06, "loss": 0.0054, "num_tokens": 80987639.0, "reward": 2.7603381276130676, "reward_std": 0.4442455396056175, "rewards/reward_fn/mean": 2.7603381276130676, "rewards/reward_fn/std": 0.4442455470561981, "step": 436, "step_time": 465.0974967526272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1091.0, "completions/max_terminated_length": 1091.0, "completions/mean_length": 256.734375, "completions/mean_terminated_length": 256.734375, "completions/min_length": 78.5, "completions/min_terminated_length": 78.5, "entropy": 0.021211280778516084, "epoch": 0.4025617566331198, "frac_reward_zero_std": 0.8984375, "grad_norm": 0.02865251038786027, "kl": 0.05977889499627054, "learning_rate": 3.2987282086320756e-06, "loss": 0.0012, "num_tokens": 81908035.0, "reward": 2.803665339946747, "reward_std": 0.4023703336715698, "rewards/reward_fn/mean": 2.803665339946747, "rewards/reward_fn/std": 0.4023703634738922, "step": 440, "step_time": 431.6096727340482 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1156.5, "completions/max_terminated_length": 1156.5, "completions/mean_length": 255.9501953125, "completions/mean_terminated_length": 255.9501953125, "completions/min_length": 73.25, "completions/min_terminated_length": 73.25, "entropy": 0.021191293955780566, "epoch": 0.4062214089661482, "frac_reward_zero_std": 0.91796875, "grad_norm": 0.031018340336869805, "kl": 0.05657385312952101, "learning_rate": 3.271161656976787e-06, "loss": 0.0062, "num_tokens": 82839944.0, "reward": 2.771248936653137, "reward_std": 0.4050714895129204, "rewards/reward_fn/mean": 2.771248936653137, "rewards/reward_fn/std": 0.40507151931524277, "step": 444, "step_time": 467.8179361205548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1117.25, "completions/max_terminated_length": 1117.25, "completions/mean_length": 257.2421875, "completions/mean_terminated_length": 257.2421875, "completions/min_length": 77.0, "completions/min_terminated_length": 77.0, "entropy": 0.021266737254336476, "epoch": 0.40988106129917656, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.045221824384451687, "kl": 0.05658956873230636, "learning_rate": 3.243491087979309e-06, "loss": 0.0101, "num_tokens": 83773492.0, "reward": 2.7226498126983643, "reward_std": 0.5032680928707123, "rewards/reward_fn/mean": 2.7226498126983643, "rewards/reward_fn/std": 0.5032681003212929, "step": 448, "step_time": 454.49583586258814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1088.75, "completions/max_terminated_length": 1088.75, "completions/mean_length": 260.2900390625, "completions/mean_terminated_length": 260.2900390625, "completions/min_length": 70.0, "completions/min_terminated_length": 70.0, "entropy": 0.021140800090506673, "epoch": 0.41354071363220496, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.06434430504394675, "kl": 0.05833716504275799, "learning_rate": 3.2157202339557124e-06, "loss": 0.0051, "num_tokens": 84713729.0, "reward": 2.70987868309021, "reward_std": 0.5102557465434074, "rewards/reward_fn/mean": 2.70987868309021, "rewards/reward_fn/std": 0.5102557688951492, "step": 452, "step_time": 441.5249572279863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1206.25, "completions/max_terminated_length": 1069.25, "completions/mean_length": 255.240234375, "completions/mean_terminated_length": 253.99888610839844, "completions/min_length": 73.75, "completions/min_terminated_length": 73.75, "entropy": 0.02270206529647112, "epoch": 0.4172003659652333, "frac_reward_zero_std": 0.89453125, "grad_norm": 0.06982801322672433, "kl": 0.05912393843755126, "learning_rate": 3.1878528407489068e-06, "loss": 0.0153, "num_tokens": 85651831.0, "reward": 2.766897201538086, "reward_std": 0.453893318772316, "rewards/reward_fn/mean": 2.766897201538086, "rewards/reward_fn/std": 0.45389336347579956, "step": 456, "step_time": 500.6877592424862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1148.0, "completions/max_terminated_length": 1148.0, "completions/mean_length": 265.1240234375, "completions/mean_terminated_length": 265.1240234375, "completions/min_length": 74.75, "completions/min_terminated_length": 74.75, "entropy": 0.022504316177219152, "epoch": 0.42086001829826164, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.2329375834800146, "kl": 0.05883700540289283, "learning_rate": 3.1598926672233916e-06, "loss": 0.0034, "num_tokens": 86598682.0, "reward": 2.756505012512207, "reward_std": 0.440291665494442, "rewards/reward_fn/mean": 2.756505012512207, "rewards/reward_fn/std": 0.4402916952967644, "step": 460, "step_time": 474.26262577204034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1123.5, "completions/max_terminated_length": 1123.5, "completions/mean_length": 254.2958984375, "completions/mean_terminated_length": 254.2958984375, "completions/min_length": 73.75, "completions/min_terminated_length": 73.75, "entropy": 0.023365118307992816, "epoch": 0.42451967063129004, "frac_reward_zero_std": 0.90234375, "grad_norm": 0.053993897375030235, "kl": 0.05943758552893996, "learning_rate": 3.131843484758242e-06, "loss": 0.0152, "num_tokens": 87535253.0, "reward": 2.7400495409965515, "reward_std": 0.48594386130571365, "rewards/reward_fn/mean": 2.7400495409965515, "rewards/reward_fn/std": 0.48594388365745544, "step": 464, "step_time": 459.86614966299385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1115.25, "completions/max_terminated_length": 1115.25, "completions/mean_length": 264.953125, "completions/mean_terminated_length": 264.953125, "completions/min_length": 79.0, "completions/min_terminated_length": 79.0, "entropy": 0.02322944754268974, "epoch": 0.4281793229643184, "frac_reward_zero_std": 0.87109375, "grad_norm": 0.05386210913195399, "kl": 0.05839638109318912, "learning_rate": 3.1037090767384128e-06, "loss": 0.0176, "num_tokens": 88492045.0, "reward": 2.757098913192749, "reward_std": 0.44403837621212006, "rewards/reward_fn/mean": 2.757098913192749, "rewards/reward_fn/std": 0.44403839111328125, "step": 468, "step_time": 463.87284979457036 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1138.25, "completions/max_terminated_length": 1138.25, "completions/mean_length": 265.919921875, "completions/mean_terminated_length": 265.919921875, "completions/min_length": 77.25, "completions/min_terminated_length": 77.25, "entropy": 0.02893318561837077, "epoch": 0.43183897529734677, "frac_reward_zero_std": 0.77734375, "grad_norm": 0.10988372126851806, "kl": 0.06366393179632723, "learning_rate": 3.0754932380444204e-06, "loss": 0.0647, "num_tokens": 89458687.0, "reward": 2.7596094608306885, "reward_std": 0.4247737377882004, "rewards/reward_fn/mean": 2.7596094608306885, "rewards/reward_fn/std": 0.42477378249168396, "step": 472, "step_time": 481.56263000750914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1195.5, "completions/max_terminated_length": 1058.5, "completions/mean_length": 260.591796875, "completions/mean_terminated_length": 259.3289260864258, "completions/min_length": 68.75, "completions/min_terminated_length": 68.75, "entropy": 0.030687018181197345, "epoch": 0.4354986276303751, "frac_reward_zero_std": 0.82421875, "grad_norm": 0.0672494129113056, "kl": 0.06749603967182338, "learning_rate": 3.04719977454047e-06, "loss": 0.0401, "num_tokens": 90417957.0, "reward": 2.7592405676841736, "reward_std": 0.44465500861406326, "rewards/reward_fn/mean": 2.7592405676841736, "rewards/reward_fn/std": 0.44465503841638565, "step": 476, "step_time": 508.5822904477827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1168.0, "completions/max_terminated_length": 1168.0, "completions/mean_length": 259.3037109375, "completions/mean_terminated_length": 259.3037109375, "completions/min_length": 69.75, "completions/min_terminated_length": 69.75, "entropy": 0.03189253679011017, "epoch": 0.43915827996340345, "frac_reward_zero_std": 0.8203125, "grad_norm": 0.1133846195195051, "kl": 0.07560356380417943, "learning_rate": 3.018832502561114e-06, "loss": 0.0483, "num_tokens": 91386696.0, "reward": 2.7331442832946777, "reward_std": 0.48754725605249405, "rewards/reward_fn/mean": 2.7331442832946777, "rewards/reward_fn/std": 0.48754723370075226, "step": 480, "step_time": 499.99215250462294 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1011.5, "completions/max_terminated_length": 1011.5, "completions/mean_length": 253.7373046875, "completions/mean_terminated_length": 253.7373046875, "completions/min_length": 75.0, "completions/min_terminated_length": 75.0, "entropy": 0.027319170185364783, "epoch": 0.44281793229643185, "frac_reward_zero_std": 0.8671875, "grad_norm": 0.03537841408588395, "kl": 0.06349759339354932, "learning_rate": 2.9903952483964804e-06, "loss": 0.0063, "num_tokens": 92351395.0, "reward": 2.747387111186981, "reward_std": 0.44578108191490173, "rewards/reward_fn/mean": 2.747387111186981, "rewards/reward_fn/std": 0.4457810968160629, "step": 484, "step_time": 436.93430981226265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1156.25, "completions/max_terminated_length": 1156.25, "completions/mean_length": 257.44140625, "completions/mean_terminated_length": 257.44140625, "completions/min_length": 81.0, "completions/min_terminated_length": 81.0, "entropy": 0.026745406910777092, "epoch": 0.4464775846294602, "frac_reward_zero_std": 0.85546875, "grad_norm": 0.05053599725656754, "kl": 0.06292624375782907, "learning_rate": 2.961891847776174e-06, "loss": 0.0139, "num_tokens": 93317339.0, "reward": 2.7652419805526733, "reward_std": 0.4404691755771637, "rewards/reward_fn/mean": 2.7652419805526733, "rewards/reward_fn/std": 0.4404691904783249, "step": 488, "step_time": 498.2464709286578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1044.5, "completions/max_terminated_length": 1044.5, "completions/mean_length": 245.5771484375, "completions/mean_terminated_length": 245.5771484375, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "entropy": 0.02598623267840594, "epoch": 0.4501372369624886, "frac_reward_zero_std": 0.8828125, "grad_norm": 0.054287951744274386, "kl": 0.06363053061068058, "learning_rate": 2.933326145351895e-06, "loss": 0.003, "num_tokens": 94273746.0, "reward": 2.755536675453186, "reward_std": 0.43755053728818893, "rewards/reward_fn/mean": 2.755536675453186, "rewards/reward_fn/std": 0.43755055218935013, "step": 492, "step_time": 447.63349654572085 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0009765625, "completions/max_length": 1222.75, "completions/max_terminated_length": 1096.0, "completions/mean_length": 258.2802734375, "completions/mean_terminated_length": 257.03343200683594, "completions/min_length": 80.5, "completions/min_terminated_length": 80.5, "entropy": 0.026308491243980825, "epoch": 0.4537968892955169, "frac_reward_zero_std": 0.859375, "grad_norm": 0.06635558280599736, "kl": 0.06186160701327026, "learning_rate": 2.9047019941788584e-06, "loss": 0.0209, "num_tokens": 95258201.0, "reward": 2.753948450088501, "reward_std": 0.4302341192960739, "rewards/reward_fn/mean": 2.753948450088501, "rewards/reward_fn/std": 0.4302341416478157, "step": 496, "step_time": 541.1232072496787 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1168.75, "completions/max_terminated_length": 1168.75, "completions/mean_length": 250.376953125, "completions/mean_terminated_length": 250.376953125, "completions/min_length": 83.75, "completions/min_terminated_length": 83.75, "entropy": 0.026318517164327204, "epoch": 0.45745654162854527, "frac_reward_zero_std": 0.87890625, "grad_norm": 0.05395106254679997, "kl": 0.06209555873647332, "learning_rate": 2.8760232551960782e-06, "loss": 0.0063, "num_tokens": 96236875.0, "reward": 2.7645170092582703, "reward_std": 0.4234401807188988, "rewards/reward_fn/mean": 2.7645170092582703, "rewards/reward_fn/std": 0.42344019562005997, "step": 500, "step_time": 512.8094203663059 } ], "logging_steps": 4, "max_steps": 1093, "num_input_tokens_seen": 96236875, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }