{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.20593080724876442, "eval_steps": 1000, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4658203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.3837890625, "completions/mean_terminated_length": 53.83729553222656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0787938493303955, "epoch": 0.00041186161449752884, "frac_reward_zero_std": 0.0, "grad_norm": 1.1026352643966675, "kl": 1.5408068257727336e-05, "learning_rate": 0.0, "loss": 0.0612, "num_tokens": 473618.0, "reward": -0.654300332069397, "reward_std": 1.2014957666397095, "rewards/reward_model/mean": -0.654300332069397, "rewards/reward_model/std": 1.4879947900772095, "step": 1, "step_time": 179.40438475832343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.521484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.0380859375, "completions/mean_terminated_length": 59.11632537841797, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0889650019817054, "epoch": 0.0008237232289950577, "frac_reward_zero_std": 0.0, "grad_norm": 0.9445520043373108, "kl": 1.5487904489575044e-05, "learning_rate": 1.2345679012345681e-08, "loss": 0.0685, "num_tokens": 944384.0, "reward": -0.6944406032562256, "reward_std": 1.1158981323242188, "rewards/reward_model/mean": -0.6944406032562256, "rewards/reward_model/std": 1.4779117107391357, "step": 2, "step_time": 168.28568758117035 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.1064453125, "completions/mean_terminated_length": 59.286537170410156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.1078118681907654, "epoch": 0.0012355848434925864, "frac_reward_zero_std": 0.0, "grad_norm": 0.9809994101524353, "kl": 0.0009136445219155576, "learning_rate": 2.4691358024691362e-08, "loss": 0.061, "num_tokens": 1417434.0, "reward": -0.8067716956138611, "reward_std": 1.1805193424224854, "rewards/reward_model/mean": -0.8067716956138611, "rewards/reward_model/std": 1.5296157598495483, "step": 3, "step_time": 168.7894278760068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4912109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.3203125, "completions/mean_terminated_length": 53.94241714477539, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0778324585407972, "epoch": 0.0016474464579901153, "frac_reward_zero_std": 0.0, "grad_norm": 0.9622601270675659, "kl": 0.0009502729969881329, "learning_rate": 3.7037037037037036e-08, "loss": 0.0854, "num_tokens": 1886250.0, "reward": -0.5533753037452698, "reward_std": 1.0693888664245605, "rewards/reward_model/mean": -0.5533753037452698, "rewards/reward_model/std": 1.3799840211868286, "step": 4, "step_time": 167.94514833204448 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.47802734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.8896484375, "completions/mean_terminated_length": 53.07202911376953, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.164030898362398, "epoch": 0.002059308072487644, "frac_reward_zero_std": 0.0, "grad_norm": 1.0810271501541138, "kl": 0.0010240612220968615, "learning_rate": 4.9382716049382724e-08, "loss": 0.0979, "num_tokens": 2372616.0, "reward": -0.8290466070175171, "reward_std": 1.1383775472640991, "rewards/reward_model/mean": -0.8290466070175171, "rewards/reward_model/std": 1.4821057319641113, "step": 5, "step_time": 168.5208105482161 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.44775390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.54541015625, "completions/mean_terminated_length": 52.934574127197266, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.121390470303595, "epoch": 0.002471169686985173, "frac_reward_zero_std": 0.0, "grad_norm": 1.026131272315979, "kl": 0.000956788239591333, "learning_rate": 6.17283950617284e-08, "loss": 0.102, "num_tokens": 2856453.0, "reward": -0.5948619842529297, "reward_std": 1.0859686136245728, "rewards/reward_model/mean": -0.5948619842529297, "rewards/reward_model/std": 1.4433753490447998, "step": 6, "step_time": 169.30755526619032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.47998046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.40185546875, "completions/mean_terminated_length": 51.85258483886719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.1293580746278167, "epoch": 0.002883031301482702, "frac_reward_zero_std": 0.0, "grad_norm": 0.9581882953643799, "kl": 0.0010509827170608332, "learning_rate": 7.407407407407407e-08, "loss": 0.0752, "num_tokens": 3349660.0, "reward": -0.8746315836906433, "reward_std": 1.1371493339538574, "rewards/reward_model/mean": -0.8746315836906433, "rewards/reward_model/std": 1.5432283878326416, "step": 7, "step_time": 170.4541406123899 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.462890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.59423828125, "completions/mean_terminated_length": 56.49545669555664, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.199626039713621, "epoch": 0.0032948929159802307, "frac_reward_zero_std": 0.0, "grad_norm": 1.1233900785446167, "kl": 0.0011659049921490805, "learning_rate": 8.641975308641976e-08, "loss": 0.0835, "num_tokens": 3869181.0, "reward": -0.9943232536315918, "reward_std": 1.099515438079834, "rewards/reward_model/mean": -0.9943232536315918, "rewards/reward_model/std": 1.4042030572891235, "step": 8, "step_time": 168.8292339304462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.50244140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.26611328125, "completions/mean_terminated_length": 56.181549072265625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.103476638905704, "epoch": 0.0037067545304777594, "frac_reward_zero_std": 0.0, "grad_norm": 1.0071220397949219, "kl": 0.0010218678287401417, "learning_rate": 9.876543209876545e-08, "loss": 0.0786, "num_tokens": 4330526.0, "reward": -0.7287623286247253, "reward_std": 1.2205724716186523, "rewards/reward_model/mean": -0.7287623286247253, "rewards/reward_model/std": 1.5410621166229248, "step": 9, "step_time": 168.4490856071934 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4384765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.068359375, "completions/mean_terminated_length": 55.10608673095703, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.068316952791065, "epoch": 0.004118616144975288, "frac_reward_zero_std": 0.0, "grad_norm": 1.0786375999450684, "kl": 0.0011272716699295415, "learning_rate": 1.1111111111111111e-07, "loss": 0.0808, "num_tokens": 4813482.0, "reward": -0.8588310480117798, "reward_std": 1.1204930543899536, "rewards/reward_model/mean": -0.8588310480117798, "rewards/reward_model/std": 1.4020955562591553, "step": 10, "step_time": 169.3498973324895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.486328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.27197265625, "completions/mean_terminated_length": 50.65874481201172, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.085946503095329, "epoch": 0.004530477759472817, "frac_reward_zero_std": 0.0, "grad_norm": 1.1249303817749023, "kl": 0.0010357791773003555, "learning_rate": 1.234567901234568e-07, "loss": 0.103, "num_tokens": 5276279.0, "reward": -0.7370425462722778, "reward_std": 1.1393404006958008, "rewards/reward_model/mean": -0.7370425462722778, "rewards/reward_model/std": 1.435203194618225, "step": 11, "step_time": 169.61693120608106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.435546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.62060546875, "completions/mean_terminated_length": 51.147926330566406, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.0578739237971604, "epoch": 0.004942339373970346, "frac_reward_zero_std": 0.0, "grad_norm": 1.1678587198257446, "kl": 0.0010205526389199804, "learning_rate": 1.3580246913580248e-07, "loss": 0.129, "num_tokens": 5750318.0, "reward": -0.6621623039245605, "reward_std": 1.1341545581817627, "rewards/reward_model/mean": -0.6621623039245605, "rewards/reward_model/std": 1.4956636428833008, "step": 12, "step_time": 170.6942683076486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.48876953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.955078125, "completions/mean_terminated_length": 53.581661224365234, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.1759862853214145, "epoch": 0.005354200988467875, "frac_reward_zero_std": 0.0, "grad_norm": 0.9353419542312622, "kl": 0.0009984489861381007, "learning_rate": 1.4814814814814815e-07, "loss": 0.0706, "num_tokens": 6237106.0, "reward": -0.71650230884552, "reward_std": 1.1081366539001465, "rewards/reward_model/mean": -0.71650230884552, "rewards/reward_model/std": 1.4882901906967163, "step": 13, "step_time": 168.60461562033743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.47265625, "completions/max_length": 128.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 90.6298828125, "completions/mean_terminated_length": 57.13518524169922, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.1225685542449355, "epoch": 0.005766062602965404, "frac_reward_zero_std": 0.0, "grad_norm": 0.9926177263259888, "kl": 0.0010822901392657513, "learning_rate": 1.6049382716049383e-07, "loss": 0.0705, "num_tokens": 6768988.0, "reward": -0.8033103346824646, "reward_std": 1.1658474206924438, "rewards/reward_model/mean": -0.8033103346824646, "rewards/reward_model/std": 1.5343424081802368, "step": 14, "step_time": 169.76986178942025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.43359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.02978515625, "completions/mean_terminated_length": 55.666378021240234, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.2208039346151054, "epoch": 0.006177924217462933, "frac_reward_zero_std": 0.0, "grad_norm": 1.110655426979065, "kl": 0.00102766218378747, "learning_rate": 1.7283950617283952e-07, "loss": 0.1137, "num_tokens": 7264761.0, "reward": -0.8211149573326111, "reward_std": 1.1067304611206055, "rewards/reward_model/mean": -0.8211149573326111, "rewards/reward_model/std": 1.4263983964920044, "step": 15, "step_time": 169.37415388552472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.8525390625, "completions/mean_terminated_length": 58.984100341796875, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.036483039613813, "epoch": 0.006589785831960461, "frac_reward_zero_std": 0.0, "grad_norm": 1.0956333875656128, "kl": 0.0011385848249574337, "learning_rate": 1.8518518518518518e-07, "loss": 0.0679, "num_tokens": 7745675.0, "reward": -0.5313577651977539, "reward_std": 1.1804759502410889, "rewards/reward_model/mean": -0.5313577651977539, "rewards/reward_model/std": 1.5051146745681763, "step": 16, "step_time": 168.84541190741584 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.46142578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.02197265625, "completions/mean_terminated_length": 51.91387176513672, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.1997494087554514, "epoch": 0.00700164744645799, "frac_reward_zero_std": 0.0, "grad_norm": 1.0221517086029053, "kl": 0.0010186115512169636, "learning_rate": 1.975308641975309e-07, "loss": 0.0962, "num_tokens": 8243704.0, "reward": -0.89983731508255, "reward_std": 1.135831356048584, "rewards/reward_model/mean": -0.89983731508255, "rewards/reward_model/std": 1.4320958852767944, "step": 17, "step_time": 168.78324813907966 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.45947265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.05908203125, "completions/mean_terminated_length": 54.10749816894531, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.176318216137588, "epoch": 0.007413509060955519, "frac_reward_zero_std": 0.0, "grad_norm": 1.0268303155899048, "kl": 0.0010050315393073106, "learning_rate": 2.0987654320987656e-07, "loss": 0.0845, "num_tokens": 8726801.0, "reward": -0.7434755563735962, "reward_std": 1.1786913871765137, "rewards/reward_model/mean": -0.7434755563735962, "rewards/reward_model/std": 1.4701310396194458, "step": 18, "step_time": 168.51915573468432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4658203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.00439453125, "completions/mean_terminated_length": 53.12705993652344, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0752989565953612, "epoch": 0.007825370675453048, "frac_reward_zero_std": 0.0, "grad_norm": 1.0904563665390015, "kl": 0.00123250130627639, "learning_rate": 2.2222222222222222e-07, "loss": 0.0885, "num_tokens": 9180858.0, "reward": -0.8568893074989319, "reward_std": 1.1963412761688232, "rewards/reward_model/mean": -0.8568893074989319, "rewards/reward_model/std": 1.5186042785644531, "step": 19, "step_time": 170.01141701499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4033203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.22021484375, "completions/mean_terminated_length": 52.95172119140625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.200795284938067, "epoch": 0.008237232289950576, "frac_reward_zero_std": 0.0, "grad_norm": 1.0073621273040771, "kl": 0.0012237756848207937, "learning_rate": 2.3456790123456793e-07, "loss": 0.0782, "num_tokens": 9702557.0, "reward": -0.9474191069602966, "reward_std": 1.101952314376831, "rewards/reward_model/mean": -0.9474191665649414, "rewards/reward_model/std": 1.514784336090088, "step": 20, "step_time": 168.7904914407991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.44189453125, "completions/max_length": 128.0, "completions/max_terminated_length": 127.0, "completions/mean_length": 85.07275390625, "completions/mean_terminated_length": 51.08399200439453, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.1343746068887413, "epoch": 0.008649093904448105, "frac_reward_zero_std": 0.0, "grad_norm": 1.1044775247573853, "kl": 0.0011666002192214364, "learning_rate": 2.469135802469136e-07, "loss": 0.0659, "num_tokens": 10182002.0, "reward": -0.8981258869171143, "reward_std": 1.1897304058074951, "rewards/reward_model/mean": -0.8981258869171143, "rewards/reward_model/std": 1.4881244897842407, "step": 21, "step_time": 168.61277754418552 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4580078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.984375, "completions/mean_terminated_length": 56.0144157409668, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.060287212021649, "epoch": 0.009060955518945634, "frac_reward_zero_std": 0.0, "grad_norm": 0.9674479961395264, "kl": 0.0012853052542141086, "learning_rate": 2.5925925925925923e-07, "loss": 0.0795, "num_tokens": 10599858.0, "reward": -0.7459607720375061, "reward_std": 1.18560791015625, "rewards/reward_model/mean": -0.7459607720375061, "rewards/reward_model/std": 1.4447804689407349, "step": 22, "step_time": 168.57235636515543 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.45751953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.20947265625, "completions/mean_terminated_length": 52.80738067626953, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.9992840560153127, "epoch": 0.009472817133443162, "frac_reward_zero_std": 0.0, "grad_norm": 1.2746193408966064, "kl": 0.001352380840899059, "learning_rate": 2.7160493827160497e-07, "loss": 0.0805, "num_tokens": 11135295.0, "reward": -0.9941644668579102, "reward_std": 1.2033442258834839, "rewards/reward_model/mean": -0.9941644668579102, "rewards/reward_model/std": 1.5118839740753174, "step": 23, "step_time": 168.93097670795396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.458984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.12744140625, "completions/mean_terminated_length": 54.300540924072266, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 2.1036753226071596, "epoch": 0.009884678747940691, "frac_reward_zero_std": 0.0, "grad_norm": 0.9719477295875549, "kl": 0.0014479562626092957, "learning_rate": 2.839506172839506e-07, "loss": 0.0792, "num_tokens": 11647428.0, "reward": -0.7246302366256714, "reward_std": 1.1223700046539307, "rewards/reward_model/mean": -0.7246302366256714, "rewards/reward_model/std": 1.4486252069473267, "step": 24, "step_time": 168.1220847275108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.48876953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.43017578125, "completions/mean_terminated_length": 58.42311477661133, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0290252747945487, "epoch": 0.01029654036243822, "frac_reward_zero_std": 0.0, "grad_norm": 1.036043643951416, "kl": 0.0014065650616430503, "learning_rate": 2.962962962962963e-07, "loss": 0.0464, "num_tokens": 12175925.0, "reward": -0.8139803409576416, "reward_std": 1.18918776512146, "rewards/reward_model/mean": -0.8139803409576416, "rewards/reward_model/std": 1.5184983015060425, "step": 25, "step_time": 169.08092289417982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.48193359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.38720703125, "completions/mean_terminated_length": 57.3279914855957, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0865674833767116, "epoch": 0.01070840197693575, "frac_reward_zero_std": 0.0, "grad_norm": 0.9221011996269226, "kl": 0.0016496309149260924, "learning_rate": 3.08641975308642e-07, "loss": 0.0459, "num_tokens": 12671022.0, "reward": -0.6815944910049438, "reward_std": 1.1987043619155884, "rewards/reward_model/mean": -0.6815944910049438, "rewards/reward_model/std": 1.503211259841919, "step": 26, "step_time": 169.66068721655756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.39208984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.84814453125, "completions/mean_terminated_length": 53.726104736328125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.100129804573953, "epoch": 0.011120263591433279, "frac_reward_zero_std": 0.0, "grad_norm": 1.0829551219940186, "kl": 0.0020542718434626295, "learning_rate": 3.2098765432098767e-07, "loss": 0.1111, "num_tokens": 13159479.0, "reward": -0.7841147780418396, "reward_std": 1.1083781719207764, "rewards/reward_model/mean": -0.7841147780418396, "rewards/reward_model/std": 1.398116946220398, "step": 27, "step_time": 170.63890342088416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41845703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.29150390625, "completions/mean_terminated_length": 52.84046936035156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0639419481158257, "epoch": 0.011532125205930808, "frac_reward_zero_std": 0.0, "grad_norm": 0.9612188935279846, "kl": 0.002871143702122936, "learning_rate": 3.333333333333333e-07, "loss": 0.0511, "num_tokens": 13624972.0, "reward": -0.41133514046669006, "reward_std": 1.0870225429534912, "rewards/reward_model/mean": -0.41133514046669006, "rewards/reward_model/std": 1.3928031921386719, "step": 28, "step_time": 169.01845826301724 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3896484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.62353515625, "completions/mean_terminated_length": 53.65519714355469, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0633218064904213, "epoch": 0.011943986820428337, "frac_reward_zero_std": 0.0, "grad_norm": 1.0220043659210205, "kl": 0.004001651409907936, "learning_rate": 3.4567901234567904e-07, "loss": 0.0608, "num_tokens": 14084265.0, "reward": -0.5280731916427612, "reward_std": 1.139591097831726, "rewards/reward_model/mean": -0.5280731916427612, "rewards/reward_model/std": 1.5284217596054077, "step": 29, "step_time": 169.68904952565208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.75439453125, "completions/mean_terminated_length": 51.321163177490234, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.053064794279635, "epoch": 0.012355848434925865, "frac_reward_zero_std": 0.0, "grad_norm": 1.0159528255462646, "kl": 0.004774259470650577, "learning_rate": 3.580246913580247e-07, "loss": 0.0826, "num_tokens": 14545778.0, "reward": -0.8308598399162292, "reward_std": 1.1439062356948853, "rewards/reward_model/mean": -0.8308598399162292, "rewards/reward_model/std": 1.4677071571350098, "step": 30, "step_time": 169.4162016301416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.40478515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.13525390625, "completions/mean_terminated_length": 54.30434799194336, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.9778149635531008, "epoch": 0.012767710049423394, "frac_reward_zero_std": 0.0, "grad_norm": 0.9643027186393738, "kl": 0.004632160428627685, "learning_rate": 3.7037037037037036e-07, "loss": 0.0324, "num_tokens": 15021479.0, "reward": -0.5928993225097656, "reward_std": 1.0915915966033936, "rewards/reward_model/mean": -0.5928993225097656, "rewards/reward_model/std": 1.4171936511993408, "step": 31, "step_time": 169.55369784962386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.39404296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.93603515625, "completions/mean_terminated_length": 51.981468200683594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.9114997563883662, "epoch": 0.013179571663920923, "frac_reward_zero_std": 0.0, "grad_norm": 0.9297622442245483, "kl": 0.005185256256481807, "learning_rate": 3.8271604938271605e-07, "loss": 0.0641, "num_tokens": 15490468.0, "reward": -0.4294321537017822, "reward_std": 1.1095049381256104, "rewards/reward_model/mean": -0.4294321537017822, "rewards/reward_model/std": 1.4001518487930298, "step": 32, "step_time": 169.97963417787105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4033203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.73388671875, "completions/mean_terminated_length": 55.48854446411133, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.9920633286237717, "epoch": 0.013591433278418451, "frac_reward_zero_std": 0.0, "grad_norm": 0.9739150404930115, "kl": 0.0050489629365984, "learning_rate": 3.950617283950618e-07, "loss": 0.069, "num_tokens": 16033027.0, "reward": -0.5853164792060852, "reward_std": 1.1397128105163574, "rewards/reward_model/mean": -0.5853164792060852, "rewards/reward_model/std": 1.4342437982559204, "step": 33, "step_time": 170.0918092643842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.33349609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.16943359375, "completions/mean_terminated_length": 56.236629486083984, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.052212963812053, "epoch": 0.01400329489291598, "frac_reward_zero_std": 0.0, "grad_norm": 1.0108827352523804, "kl": 0.0070345894837373635, "learning_rate": 4.0740740740740737e-07, "loss": 0.0717, "num_tokens": 16443422.0, "reward": -0.40320760011672974, "reward_std": 1.023691177368164, "rewards/reward_model/mean": -0.40320760011672974, "rewards/reward_model/std": 1.3064631223678589, "step": 34, "step_time": 168.22575595136732 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3896484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.1884765625, "completions/mean_terminated_length": 52.94239807128906, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.8943170690909028, "epoch": 0.014415156507413509, "frac_reward_zero_std": 0.0, "grad_norm": 0.9463284015655518, "kl": 0.006503033908302314, "learning_rate": 4.197530864197531e-07, "loss": 0.0771, "num_tokens": 16958848.0, "reward": -0.46641844511032104, "reward_std": 1.1392958164215088, "rewards/reward_model/mean": -0.46641844511032104, "rewards/reward_model/std": 1.3904635906219482, "step": 35, "step_time": 169.57979472074658 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2724609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 69.291015625, "completions/mean_terminated_length": 47.3046989440918, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 2.0284548006020486, "epoch": 0.014827018121911038, "frac_reward_zero_std": 0.0, "grad_norm": 1.202287197113037, "kl": 0.008711300118193321, "learning_rate": 4.320987654320988e-07, "loss": 0.0853, "num_tokens": 17445812.0, "reward": -0.5143425464630127, "reward_std": 1.080782175064087, "rewards/reward_model/mean": -0.5143425464630127, "rewards/reward_model/std": 1.3849540948867798, "step": 36, "step_time": 169.69654387421906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3369140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.77197265625, "completions/mean_terminated_length": 52.25110626220703, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.8757861303165555, "epoch": 0.015238879736408566, "frac_reward_zero_std": 0.0, "grad_norm": 0.8937506675720215, "kl": 0.010500569681425986, "learning_rate": 4.4444444444444444e-07, "loss": 0.0466, "num_tokens": 17886785.0, "reward": -0.2941930890083313, "reward_std": 1.089874267578125, "rewards/reward_model/mean": -0.2941930890083313, "rewards/reward_model/std": 1.3422448635101318, "step": 37, "step_time": 168.3831845112145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.33203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.78515625, "completions/mean_terminated_length": 54.32163619995117, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.8339524874463677, "epoch": 0.015650741350906095, "frac_reward_zero_std": 0.0, "grad_norm": 0.8777432441711426, "kl": 0.014370948238138226, "learning_rate": 4.567901234567901e-07, "loss": 0.0431, "num_tokens": 18363593.0, "reward": -0.21549299359321594, "reward_std": 1.0654486417770386, "rewards/reward_model/mean": -0.21549299359321594, "rewards/reward_model/std": 1.286303997039795, "step": 38, "step_time": 168.6560257449746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3623046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.3076171875, "completions/mean_terminated_length": 53.211334228515625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.809513804037124, "epoch": 0.016062602965403624, "frac_reward_zero_std": 0.0, "grad_norm": 0.850346028804779, "kl": 0.019628787720648688, "learning_rate": 4.6913580246913586e-07, "loss": -0.0144, "num_tokens": 18782015.0, "reward": -0.19260446727275848, "reward_std": 1.0799050331115723, "rewards/reward_model/mean": -0.19260446727275848, "rewards/reward_model/std": 1.4198755025863647, "step": 39, "step_time": 169.06077374424785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3212890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.080078125, "completions/mean_terminated_length": 54.44892120361328, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.856378594879061, "epoch": 0.016474464579901153, "frac_reward_zero_std": 0.0, "grad_norm": 0.8225836753845215, "kl": 0.023546985856228275, "learning_rate": 4.814814814814815e-07, "loss": 0.0268, "num_tokens": 19238691.0, "reward": -0.22110876441001892, "reward_std": 1.0441968441009521, "rewards/reward_model/mean": -0.22110876441001892, "rewards/reward_model/std": 1.3271934986114502, "step": 40, "step_time": 169.5594472438097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.33251953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.1416015625, "completions/mean_terminated_length": 51.805416107177734, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.764324402436614, "epoch": 0.01688632619439868, "frac_reward_zero_std": 0.0, "grad_norm": 0.7773052453994751, "kl": 0.029280464848852716, "learning_rate": 4.938271604938272e-07, "loss": 0.0194, "num_tokens": 19755301.0, "reward": -0.12512998282909393, "reward_std": 1.0090844631195068, "rewards/reward_model/mean": -0.12512998282909393, "rewards/reward_model/std": 1.2345008850097656, "step": 41, "step_time": 170.39410974271595 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27490234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 72.548828125, "completions/mean_terminated_length": 51.52592468261719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.758555585052818, "epoch": 0.01729818780889621, "frac_reward_zero_std": 0.0, "grad_norm": 0.8362958431243896, "kl": 0.03495379232481355, "learning_rate": 5.061728395061729e-07, "loss": 0.0014, "num_tokens": 20199209.0, "reward": -0.034443896263837814, "reward_std": 1.0466477870941162, "rewards/reward_model/mean": -0.034443896263837814, "rewards/reward_model/std": 1.2755711078643799, "step": 42, "step_time": 170.17393092392012 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25830078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.35302734375, "completions/mean_terminated_length": 55.6701774597168, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.6830066749826074, "epoch": 0.01771004942339374, "frac_reward_zero_std": 0.0, "grad_norm": 0.9207571148872375, "kl": 0.03826815243519377, "learning_rate": 5.185185185185185e-07, "loss": 0.0197, "num_tokens": 20667900.0, "reward": -0.03724297881126404, "reward_std": 0.9730924367904663, "rewards/reward_model/mean": -0.03724297881126404, "rewards/reward_model/std": 1.1648329496383667, "step": 43, "step_time": 168.8539799619466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3427734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.59326171875, "completions/mean_terminated_length": 58.911590576171875, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.722061135340482, "epoch": 0.018121911037891267, "frac_reward_zero_std": 0.0, "grad_norm": 0.691871166229248, "kl": 0.035297417802212294, "learning_rate": 5.308641975308642e-07, "loss": 0.0252, "num_tokens": 21084443.0, "reward": 0.1364922821521759, "reward_std": 0.9992862939834595, "rewards/reward_model/mean": 0.1364922821521759, "rewards/reward_model/std": 1.338813066482544, "step": 44, "step_time": 168.30778062017635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2216796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 70.23193359375, "completions/mean_terminated_length": 53.77854537963867, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.6960708745755255, "epoch": 0.018533772652388796, "frac_reward_zero_std": 0.0, "grad_norm": 0.8769946694374084, "kl": 0.04808991262325435, "learning_rate": 5.432098765432099e-07, "loss": -0.0016, "num_tokens": 21510710.0, "reward": 0.1898983120918274, "reward_std": 0.9757044911384583, "rewards/reward_model/mean": 0.1898983120918274, "rewards/reward_model/std": 1.1677379608154297, "step": 45, "step_time": 169.03864477854222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.9912109375, "completions/mean_terminated_length": 58.42255401611328, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.6502066934481263, "epoch": 0.018945634266886325, "frac_reward_zero_std": 0.0, "grad_norm": 0.6699737906455994, "kl": 0.04364871226789546, "learning_rate": 5.555555555555555e-07, "loss": 0.0298, "num_tokens": 21968036.0, "reward": 0.14871619641780853, "reward_std": 0.8983126878738403, "rewards/reward_model/mean": 0.14871619641780853, "rewards/reward_model/std": 1.1425597667694092, "step": 46, "step_time": 169.17142802104354 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27294921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.318359375, "completions/mean_terminated_length": 52.789791107177734, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.806841244455427, "epoch": 0.019357495881383854, "frac_reward_zero_std": 0.0, "grad_norm": 0.7548915147781372, "kl": 0.050439021695638075, "learning_rate": 5.679012345679012e-07, "loss": 0.0261, "num_tokens": 22428752.0, "reward": 0.18833398818969727, "reward_std": 0.9490935802459717, "rewards/reward_model/mean": 0.18833398818969727, "rewards/reward_model/std": 1.218595027923584, "step": 47, "step_time": 169.8053262718022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.23583984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 67.080078125, "completions/mean_terminated_length": 48.278594970703125, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.7256725700572133, "epoch": 0.019769357495881382, "frac_reward_zero_std": 0.0, "grad_norm": 0.8408772349357605, "kl": 0.05761563615669729, "learning_rate": 5.80246913580247e-07, "loss": -0.0234, "num_tokens": 22883444.0, "reward": 0.10819900035858154, "reward_std": 0.9136756062507629, "rewards/reward_model/mean": 0.10819900035858154, "rewards/reward_model/std": 1.13023042678833, "step": 48, "step_time": 170.28967663506046 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.25927734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 71.54296875, "completions/mean_terminated_length": 51.78114700317383, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.676765794865787, "epoch": 0.02018121911037891, "frac_reward_zero_std": 0.0, "grad_norm": 0.6633880138397217, "kl": 0.05726497815339826, "learning_rate": 5.925925925925926e-07, "loss": 0.0077, "num_tokens": 23311980.0, "reward": 0.31039929389953613, "reward_std": 0.8826955556869507, "rewards/reward_model/mean": 0.31039929389953613, "rewards/reward_model/std": 1.1599924564361572, "step": 49, "step_time": 169.17084869695827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2392578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.46142578125, "completions/mean_terminated_length": 56.30873107910156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5921450154855847, "epoch": 0.02059308072487644, "frac_reward_zero_std": 0.0, "grad_norm": 0.7272253632545471, "kl": 0.05662718684470747, "learning_rate": 6.049382716049383e-07, "loss": -0.0038, "num_tokens": 23729245.0, "reward": 0.2335912585258484, "reward_std": 0.9175702929496765, "rewards/reward_model/mean": 0.2335912585258484, "rewards/reward_model/std": 1.1314274072647095, "step": 50, "step_time": 169.32695539435372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.26220703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.47705078125, "completions/mean_terminated_length": 59.521507263183594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.630979296285659, "epoch": 0.021004942339373972, "frac_reward_zero_std": 0.0, "grad_norm": 0.7209051847457886, "kl": 0.0596031873501488, "learning_rate": 6.17283950617284e-07, "loss": -0.019, "num_tokens": 24171054.0, "reward": 0.3881710171699524, "reward_std": 0.9779696464538574, "rewards/reward_model/mean": 0.3881710171699524, "rewards/reward_model/std": 1.2501736879348755, "step": 51, "step_time": 169.39474018104374 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.35888671875, "completions/mean_terminated_length": 62.0467529296875, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.5277671799995005, "epoch": 0.0214168039538715, "frac_reward_zero_std": 0.0, "grad_norm": 0.6657931208610535, "kl": 0.04347534721819102, "learning_rate": 6.296296296296296e-07, "loss": 0.0084, "num_tokens": 24640845.0, "reward": 0.40088099241256714, "reward_std": 0.8522671461105347, "rewards/reward_model/mean": 0.40088099241256714, "rewards/reward_model/std": 1.1760755777359009, "step": 52, "step_time": 169.99416326358914 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.26708984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.86572265625, "completions/mean_terminated_length": 58.23118209838867, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.5355965252965689, "epoch": 0.02182866556836903, "frac_reward_zero_std": 0.0, "grad_norm": 0.7246338725090027, "kl": 0.059198625254794024, "learning_rate": 6.419753086419753e-07, "loss": 0.0065, "num_tokens": 25146234.0, "reward": 0.32493141293525696, "reward_std": 0.8951080441474915, "rewards/reward_model/mean": 0.32493141293525696, "rewards/reward_model/std": 1.109892725944519, "step": 53, "step_time": 170.4134237067774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.26220703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.32373046875, "completions/mean_terminated_length": 59.313697814941406, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4214782847557217, "epoch": 0.022240527182866558, "frac_reward_zero_std": 0.0, "grad_norm": 0.663506805896759, "kl": 0.07469483163731638, "learning_rate": 6.54320987654321e-07, "loss": -0.0106, "num_tokens": 25556273.0, "reward": 0.5579333305358887, "reward_std": 0.8257571458816528, "rewards/reward_model/mean": 0.5579333305358887, "rewards/reward_model/std": 1.0652962923049927, "step": 54, "step_time": 168.6097109238617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2158203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 67.8369140625, "completions/mean_terminated_length": 51.278953552246094, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.4409256265498698, "epoch": 0.022652388797364087, "frac_reward_zero_std": 0.0, "grad_norm": 0.6998105645179749, "kl": 0.09127819760760758, "learning_rate": 6.666666666666666e-07, "loss": -0.0145, "num_tokens": 26009315.0, "reward": 0.36190831661224365, "reward_std": 0.8500241637229919, "rewards/reward_model/mean": 0.36190831661224365, "rewards/reward_model/std": 1.0737853050231934, "step": 55, "step_time": 169.63903413154185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 71.69873046875, "completions/mean_terminated_length": 55.20643997192383, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4477170635946095, "epoch": 0.023064250411861616, "frac_reward_zero_std": 0.0, "grad_norm": 0.6856215596199036, "kl": 0.10684883118665311, "learning_rate": 6.790123456790124e-07, "loss": -0.0154, "num_tokens": 26453082.0, "reward": 0.5452687740325928, "reward_std": 0.7654911875724792, "rewards/reward_model/mean": 0.5452687740325928, "rewards/reward_model/std": 0.9965056777000427, "step": 56, "step_time": 167.3982848683372 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.31689453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.6259765625, "completions/mean_terminated_length": 58.6490364074707, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.3876500492915511, "epoch": 0.023476112026359144, "frac_reward_zero_std": 0.0, "grad_norm": 0.5802730321884155, "kl": 0.08226650860888185, "learning_rate": 6.913580246913581e-07, "loss": -0.0124, "num_tokens": 26859036.0, "reward": 0.4985049366950989, "reward_std": 0.8241320252418518, "rewards/reward_model/mean": 0.4985049366950989, "rewards/reward_model/std": 1.177066683769226, "step": 57, "step_time": 168.99145932588726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.29150390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.41162109375, "completions/mean_terminated_length": 62.24327850341797, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3736186842434108, "epoch": 0.023887973640856673, "frac_reward_zero_std": 0.0, "grad_norm": 0.672706127166748, "kl": 0.0943700206480571, "learning_rate": 7.037037037037037e-07, "loss": 0.0084, "num_tokens": 27318439.0, "reward": 0.7748833894729614, "reward_std": 0.8400471806526184, "rewards/reward_model/mean": 0.7748833894729614, "rewards/reward_model/std": 1.0984324216842651, "step": 58, "step_time": 168.8087218273431 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.30419921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.111328125, "completions/mean_terminated_length": 57.737545013427734, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.3578353270422667, "epoch": 0.024299835255354202, "frac_reward_zero_std": 0.0, "grad_norm": 0.6414256691932678, "kl": 0.10844983752758708, "learning_rate": 7.160493827160494e-07, "loss": 0.0049, "num_tokens": 27845483.0, "reward": 0.6581840515136719, "reward_std": 0.7642059326171875, "rewards/reward_model/mean": 0.6581840515136719, "rewards/reward_model/std": 1.0230196714401245, "step": 59, "step_time": 170.09878712054342 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.20947265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 68.9013671875, "completions/mean_terminated_length": 53.24150848388672, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.359937054105103, "epoch": 0.02471169686985173, "frac_reward_zero_std": 0.0, "grad_norm": 0.6327216029167175, "kl": 0.12389619748864789, "learning_rate": 7.283950617283951e-07, "loss": -0.0019, "num_tokens": 28360609.0, "reward": 0.5726691484451294, "reward_std": 0.7263065576553345, "rewards/reward_model/mean": 0.5726691484451294, "rewards/reward_model/std": 1.0532201528549194, "step": 60, "step_time": 169.033332105726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21728515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.28515625, "completions/mean_terminated_length": 59.37367248535156, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2873307822737843, "epoch": 0.02512355848434926, "frac_reward_zero_std": 0.0, "grad_norm": 0.6471663117408752, "kl": 0.11947597128164489, "learning_rate": 7.407407407407407e-07, "loss": -0.018, "num_tokens": 28796649.0, "reward": 0.8057171106338501, "reward_std": 0.6930927038192749, "rewards/reward_model/mean": 0.8057171106338501, "rewards/reward_model/std": 0.9504708647727966, "step": 61, "step_time": 170.3676045727916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27880859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.5634765625, "completions/mean_terminated_length": 62.22477722167969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.4397811936214566, "epoch": 0.025535420098846788, "frac_reward_zero_std": 0.0, "grad_norm": 0.5935730934143066, "kl": 0.11211827301303856, "learning_rate": 7.530864197530865e-07, "loss": 0.005, "num_tokens": 29272715.0, "reward": 0.5376583337783813, "reward_std": 0.7316970825195312, "rewards/reward_model/mean": 0.5376583337783813, "rewards/reward_model/std": 1.0817116498947144, "step": 62, "step_time": 170.07809142861515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.27978515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.00634765625, "completions/mean_terminated_length": 65.52745819091797, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2591755213215947, "epoch": 0.025947281713344317, "frac_reward_zero_std": 0.0, "grad_norm": 0.6332337260246277, "kl": 0.09935169246455189, "learning_rate": 7.654320987654321e-07, "loss": -0.0047, "num_tokens": 29675416.0, "reward": 0.8634133338928223, "reward_std": 0.7280638217926025, "rewards/reward_model/mean": 0.8634133338928223, "rewards/reward_model/std": 1.0552853345870972, "step": 63, "step_time": 169.48511258373037 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.23291015625, "completions/mean_terminated_length": 64.30420684814453, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.3228048181626946, "epoch": 0.026359143327841845, "frac_reward_zero_std": 0.0, "grad_norm": 0.5896514058113098, "kl": 0.10629570209130179, "learning_rate": 7.777777777777778e-07, "loss": 0.0018, "num_tokens": 30103349.0, "reward": 0.7413797378540039, "reward_std": 0.6787456274032593, "rewards/reward_model/mean": 0.7413797378540039, "rewards/reward_model/std": 0.9844362735748291, "step": 64, "step_time": 168.49784950073808 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2080078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.34521484375, "completions/mean_terminated_length": 64.04130554199219, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.3859816826879978, "epoch": 0.026771004942339374, "frac_reward_zero_std": 0.0, "grad_norm": 0.569113552570343, "kl": 0.1355650291661732, "learning_rate": 7.901234567901236e-07, "loss": 0.0008, "num_tokens": 30598360.0, "reward": 0.8138879537582397, "reward_std": 0.6921124458312988, "rewards/reward_model/mean": 0.8138879537582397, "rewards/reward_model/std": 1.008180856704712, "step": 65, "step_time": 168.94378049625084 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2333984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.275390625, "completions/mean_terminated_length": 60.52738952636719, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.276806804118678, "epoch": 0.027182866556836903, "frac_reward_zero_std": 0.0, "grad_norm": 0.584193229675293, "kl": 0.12889757197990548, "learning_rate": 8.024691358024692e-07, "loss": 0.0109, "num_tokens": 31028524.0, "reward": 0.7695643901824951, "reward_std": 0.7420451641082764, "rewards/reward_model/mean": 0.7695643901824951, "rewards/reward_model/std": 1.1982769966125488, "step": 66, "step_time": 169.0462037078105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2314453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.75390625, "completions/mean_terminated_length": 67.82718658447266, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2658436398487538, "epoch": 0.02759472817133443, "frac_reward_zero_std": 0.0, "grad_norm": 0.5793888568878174, "kl": 0.10512553945591208, "learning_rate": 8.148148148148147e-07, "loss": 0.019, "num_tokens": 31459860.0, "reward": 0.9362199306488037, "reward_std": 0.6280190944671631, "rewards/reward_model/mean": 0.9362199306488037, "rewards/reward_model/std": 1.003322958946228, "step": 67, "step_time": 168.24778978247195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.20068359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.294921875, "completions/mean_terminated_length": 62.06230926513672, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3218755372799933, "epoch": 0.02800658978583196, "frac_reward_zero_std": 0.0, "grad_norm": 0.619716465473175, "kl": 0.13950362912146375, "learning_rate": 8.271604938271605e-07, "loss": 0.0032, "num_tokens": 31900336.0, "reward": 0.7856715321540833, "reward_std": 0.6523309946060181, "rewards/reward_model/mean": 0.7856715321540833, "rewards/reward_model/std": 0.9243690371513367, "step": 68, "step_time": 168.62237379932776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 72.62060546875, "completions/mean_terminated_length": 59.84074783325195, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2817874399479479, "epoch": 0.02841845140032949, "frac_reward_zero_std": 0.0, "grad_norm": 0.6988398432731628, "kl": 0.1405531533237081, "learning_rate": 8.395061728395062e-07, "loss": 0.0001, "num_tokens": 32349991.0, "reward": 0.7539228200912476, "reward_std": 0.6927404403686523, "rewards/reward_model/mean": 0.7539228200912476, "rewards/reward_model/std": 1.1138005256652832, "step": 69, "step_time": 168.95463426411152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.18310546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.68505859375, "completions/mean_terminated_length": 65.18290710449219, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.3052547052502632, "epoch": 0.028830313014827018, "frac_reward_zero_std": 0.0, "grad_norm": 0.5859233140945435, "kl": 0.14809596522536594, "learning_rate": 8.518518518518518e-07, "loss": 0.006, "num_tokens": 32799778.0, "reward": 0.899767279624939, "reward_std": 0.6600509881973267, "rewards/reward_model/mean": 0.899767279624939, "rewards/reward_model/std": 1.0378800630569458, "step": 70, "step_time": 168.27934673754498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.21435546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.1396484375, "completions/mean_terminated_length": 63.26289749145508, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2957828119397163, "epoch": 0.029242174629324547, "frac_reward_zero_std": 0.0, "grad_norm": 0.6063217520713806, "kl": 0.1499464159278432, "learning_rate": 8.641975308641976e-07, "loss": 0.0009, "num_tokens": 33258528.0, "reward": 0.9532963037490845, "reward_std": 0.5860557556152344, "rewards/reward_model/mean": 0.9532963037490845, "rewards/reward_model/std": 0.9753101468086243, "step": 71, "step_time": 168.12876597139984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.123046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 71.07666015625, "completions/mean_terminated_length": 63.08964538574219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2831250824965537, "epoch": 0.029654036243822075, "frac_reward_zero_std": 0.0, "grad_norm": 0.650992214679718, "kl": 0.14023015335260425, "learning_rate": 8.765432098765433e-07, "loss": 0.0053, "num_tokens": 33708125.0, "reward": 0.8464133739471436, "reward_std": 0.5981078147888184, "rewards/reward_model/mean": 0.8464133739471436, "rewards/reward_model/std": 0.9848034977912903, "step": 72, "step_time": 168.81821045372635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1435546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 70.923828125, "completions/mean_terminated_length": 61.356895446777344, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.27306635864079, "epoch": 0.030065897858319604, "frac_reward_zero_std": 0.0, "grad_norm": 0.6549474000930786, "kl": 0.15989831037586555, "learning_rate": 8.888888888888889e-07, "loss": -0.0137, "num_tokens": 34163265.0, "reward": 0.9670735001564026, "reward_std": 0.590969979763031, "rewards/reward_model/mean": 0.9670735001564026, "rewards/reward_model/std": 0.9453141689300537, "step": 73, "step_time": 169.4882780299522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.10107421875, "completions/mean_terminated_length": 66.77362060546875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2641989213880152, "epoch": 0.030477759472817133, "frac_reward_zero_std": 0.0, "grad_norm": 0.5717378854751587, "kl": 0.13075158167339396, "learning_rate": 9.012345679012347e-07, "loss": 0.0049, "num_tokens": 34635568.0, "reward": 1.0525561571121216, "reward_std": 0.5589165687561035, "rewards/reward_model/mean": 1.0525561571121216, "rewards/reward_model/std": 0.8849756121635437, "step": 74, "step_time": 169.85129849473014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.18017578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.9130859375, "completions/mean_terminated_length": 65.6855239868164, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2528326134197414, "epoch": 0.03088962108731466, "frac_reward_zero_std": 0.0, "grad_norm": 0.5799550414085388, "kl": 0.13488844226230867, "learning_rate": 9.135802469135802e-07, "loss": 0.0233, "num_tokens": 35042110.0, "reward": 1.0121339559555054, "reward_std": 0.6093316078186035, "rewards/reward_model/mean": 1.0121339559555054, "rewards/reward_model/std": 0.9795147776603699, "step": 75, "step_time": 168.39488552790135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 72.35791015625, "completions/mean_terminated_length": 65.24944305419922, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2186853648163378, "epoch": 0.03130148270181219, "frac_reward_zero_std": 0.0, "grad_norm": 0.6350732445716858, "kl": 0.13198584888596088, "learning_rate": 9.259259259259259e-07, "loss": 0.0245, "num_tokens": 35454619.0, "reward": 1.1278910636901855, "reward_std": 0.6185814738273621, "rewards/reward_model/mean": 1.1278910636901855, "rewards/reward_model/std": 0.9232901930809021, "step": 76, "step_time": 169.25788368703797 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13134765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 70.81689453125, "completions/mean_terminated_length": 62.170318603515625, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.247056140564382, "epoch": 0.03171334431630972, "frac_reward_zero_std": 0.0, "grad_norm": 0.759410560131073, "kl": 0.16601154587988276, "learning_rate": 9.382716049382717e-07, "loss": -0.0003, "num_tokens": 35845316.0, "reward": 1.0192276239395142, "reward_std": 0.5931369066238403, "rewards/reward_model/mean": 1.0192276239395142, "rewards/reward_model/std": 0.9772949814796448, "step": 77, "step_time": 168.0306376479566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09521484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 70.13916015625, "completions/mean_terminated_length": 64.05018615722656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2540333659853786, "epoch": 0.03212520593080725, "frac_reward_zero_std": 0.0, "grad_norm": 0.6740376353263855, "kl": 0.1378997444990091, "learning_rate": 9.506172839506173e-07, "loss": -0.013, "num_tokens": 36287137.0, "reward": 0.9819941520690918, "reward_std": 0.604373574256897, "rewards/reward_model/mean": 0.9819941520690918, "rewards/reward_model/std": 0.9436709880828857, "step": 78, "step_time": 168.8950103893876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09130859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 68.37744140625, "completions/mean_terminated_length": 62.3863525390625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1720305329654366, "epoch": 0.032537067545304776, "frac_reward_zero_std": 0.0, "grad_norm": 0.6466286182403564, "kl": 0.12133494461886585, "learning_rate": 9.62962962962963e-07, "loss": -0.0048, "num_tokens": 36719654.0, "reward": 1.1645737886428833, "reward_std": 0.5557790398597717, "rewards/reward_model/mean": 1.1645737886428833, "rewards/reward_model/std": 0.9391114711761475, "step": 79, "step_time": 169.75694013293833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13330078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 69.42236328125, "completions/mean_terminated_length": 60.412960052490234, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2074508473742753, "epoch": 0.032948929159802305, "frac_reward_zero_std": 0.0, "grad_norm": 0.7178550362586975, "kl": 0.16269512700091582, "learning_rate": 9.753086419753088e-07, "loss": 0.0024, "num_tokens": 37198599.0, "reward": 1.0070809125900269, "reward_std": 0.6197090148925781, "rewards/reward_model/mean": 1.0070809125900269, "rewards/reward_model/std": 0.8695884943008423, "step": 80, "step_time": 170.65315298642963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10888671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 69.23583984375, "completions/mean_terminated_length": 62.05534362792969, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1959036465268582, "epoch": 0.033360790774299834, "frac_reward_zero_std": 0.0, "grad_norm": 0.6767128705978394, "kl": 0.1640322696766816, "learning_rate": 9.876543209876544e-07, "loss": -0.0102, "num_tokens": 37627498.0, "reward": 1.051206111907959, "reward_std": 0.6312122344970703, "rewards/reward_model/mean": 1.051206111907959, "rewards/reward_model/std": 1.006866455078125, "step": 81, "step_time": 169.01685216045007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.1484375, "completions/mean_terminated_length": 66.8294906616211, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2353556689340621, "epoch": 0.03377265238879736, "frac_reward_zero_std": 0.0, "grad_norm": 0.6121835708618164, "kl": 0.13194930272584315, "learning_rate": 1e-06, "loss": -0.0084, "num_tokens": 38077466.0, "reward": 1.1334960460662842, "reward_std": 0.5429809093475342, "rewards/reward_model/mean": 1.1334960460662842, "rewards/reward_model/std": 1.0062233209609985, "step": 82, "step_time": 169.77917499747127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1455078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.54931640625, "completions/mean_terminated_length": 65.44742584228516, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2487247881945223, "epoch": 0.03418451400329489, "frac_reward_zero_std": 0.0, "grad_norm": 0.8223838210105896, "kl": 0.1559760042873677, "learning_rate": 1.0123456790123457e-06, "loss": 0.0192, "num_tokens": 38560095.0, "reward": 1.0274322032928467, "reward_std": 0.6149877309799194, "rewards/reward_model/mean": 1.0274322032928467, "rewards/reward_model/std": 0.8848612308502197, "step": 83, "step_time": 168.7728981245309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0693359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 68.9453125, "completions/mean_terminated_length": 64.54563903808594, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.203527741599828, "epoch": 0.03459637561779242, "frac_reward_zero_std": 0.0, "grad_norm": 0.6430616974830627, "kl": 0.16221579984994605, "learning_rate": 1.0246913580246913e-06, "loss": -0.0054, "num_tokens": 38989743.0, "reward": 1.1352043151855469, "reward_std": 0.5808489918708801, "rewards/reward_model/mean": 1.1352043151855469, "rewards/reward_model/std": 1.0034772157669067, "step": 84, "step_time": 167.36356884567067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08544921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.13720703125, "completions/mean_terminated_length": 68.01121520996094, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1932638001162559, "epoch": 0.03500823723228995, "frac_reward_zero_std": 0.0, "grad_norm": 0.618569552898407, "kl": 0.1509321930789156, "learning_rate": 1.037037037037037e-06, "loss": -0.0105, "num_tokens": 39413960.0, "reward": 1.1917307376861572, "reward_std": 0.6115972995758057, "rewards/reward_model/mean": 1.1917307376861572, "rewards/reward_model/std": 0.8577749729156494, "step": 85, "step_time": 169.4022615076974 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.16455078125, "completions/mean_terminated_length": 65.91948699951172, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1511160423979163, "epoch": 0.03542009884678748, "frac_reward_zero_std": 0.0, "grad_norm": 0.6170347929000854, "kl": 0.16613518859958276, "learning_rate": 1.0493827160493827e-06, "loss": 0.0068, "num_tokens": 39897081.0, "reward": 1.2492460012435913, "reward_std": 0.6051790714263916, "rewards/reward_model/mean": 1.2492460012435913, "rewards/reward_model/std": 0.8991779685020447, "step": 86, "step_time": 168.7876625736244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.091796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.81884765625, "completions/mean_terminated_length": 68.34247589111328, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.216100089251995, "epoch": 0.035831960461285006, "frac_reward_zero_std": 0.0, "grad_norm": 0.6116214394569397, "kl": 0.17693783454888035, "learning_rate": 1.0617283950617285e-06, "loss": 0.014, "num_tokens": 40304870.0, "reward": 1.327715277671814, "reward_std": 0.5273313522338867, "rewards/reward_model/mean": 1.327715277671814, "rewards/reward_model/std": 0.8829416036605835, "step": 87, "step_time": 170.00215818034485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10498046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.087890625, "completions/mean_terminated_length": 76.70267486572266, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2015210629906505, "epoch": 0.036243822075782535, "frac_reward_zero_std": 0.0, "grad_norm": 0.5836432576179504, "kl": 0.14079495580517687, "learning_rate": 1.074074074074074e-06, "loss": 0.0043, "num_tokens": 40738362.0, "reward": 1.2116522789001465, "reward_std": 0.5785905122756958, "rewards/reward_model/mean": 1.2116522789001465, "rewards/reward_model/std": 0.8599736094474792, "step": 88, "step_time": 170.04560359567404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07080078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 71.1552734375, "completions/mean_terminated_length": 66.82395935058594, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.1739569688215852, "epoch": 0.036655683690280064, "frac_reward_zero_std": 0.0, "grad_norm": 0.5943244099617004, "kl": 0.16060514625860378, "learning_rate": 1.0864197530864199e-06, "loss": -0.0115, "num_tokens": 41174584.0, "reward": 1.079056739807129, "reward_std": 0.5492511987686157, "rewards/reward_model/mean": 1.079056739807129, "rewards/reward_model/std": 0.8875714540481567, "step": 89, "step_time": 170.3232544688508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.50341796875, "completions/mean_terminated_length": 68.63350677490234, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2288584825582802, "epoch": 0.03706754530477759, "frac_reward_zero_std": 0.0, "grad_norm": 0.6955690979957581, "kl": 0.18743510471540503, "learning_rate": 1.0987654320987655e-06, "loss": 0.0392, "num_tokens": 41634879.0, "reward": 1.2760483026504517, "reward_std": 0.5275530219078064, "rewards/reward_model/mean": 1.2760483026504517, "rewards/reward_model/std": 0.8526185154914856, "step": 90, "step_time": 169.28955688048154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.103515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.4619140625, "completions/mean_terminated_length": 71.62635803222656, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1926879836246371, "epoch": 0.03747940691927512, "frac_reward_zero_std": 0.0, "grad_norm": 0.6305184364318848, "kl": 0.16981972678331658, "learning_rate": 1.111111111111111e-06, "loss": -0.0032, "num_tokens": 42093777.0, "reward": 1.3892216682434082, "reward_std": 0.5210399627685547, "rewards/reward_model/mean": 1.3892216682434082, "rewards/reward_model/std": 0.8532023429870605, "step": 91, "step_time": 169.93193591805175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.02099609375, "completions/mean_terminated_length": 74.65526580810547, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.220891160191968, "epoch": 0.03789126853377265, "frac_reward_zero_std": 0.0, "grad_norm": 0.5791942477226257, "kl": 0.13990605663275346, "learning_rate": 1.1234567901234568e-06, "loss": 0.021, "num_tokens": 42533916.0, "reward": 1.3777389526367188, "reward_std": 0.5628249049186707, "rewards/reward_model/mean": 1.3777389526367188, "rewards/reward_model/std": 0.8695874214172363, "step": 92, "step_time": 168.67672005156055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13427734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.66845703125, "completions/mean_terminated_length": 74.48223876953125, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.1679968070238829, "epoch": 0.03830313014827018, "frac_reward_zero_std": 0.0, "grad_norm": 0.577627956867218, "kl": 0.14946075380430557, "learning_rate": 1.1358024691358024e-06, "loss": 0.0005, "num_tokens": 42977269.0, "reward": 1.2400810718536377, "reward_std": 0.5488580465316772, "rewards/reward_model/mean": 1.2400810718536377, "rewards/reward_model/std": 0.8818415999412537, "step": 93, "step_time": 169.11300712404773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1591796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.02783203125, "completions/mean_terminated_length": 81.64982604980469, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2256716336123645, "epoch": 0.03871499176276771, "frac_reward_zero_std": 0.0, "grad_norm": 1.3020151853561401, "kl": 0.1575723221176304, "learning_rate": 1.1481481481481482e-06, "loss": 0.0078, "num_tokens": 43468782.0, "reward": 1.2878694534301758, "reward_std": 0.5034958124160767, "rewards/reward_model/mean": 1.2878694534301758, "rewards/reward_model/std": 0.8000524640083313, "step": 94, "step_time": 168.66197129152715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.146484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.29296875, "completions/mean_terminated_length": 74.4485092163086, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2125889593735337, "epoch": 0.039126853377265236, "frac_reward_zero_std": 0.0, "grad_norm": 0.5775403380393982, "kl": 0.17348909995052963, "learning_rate": 1.160493827160494e-06, "loss": 0.0291, "num_tokens": 43925606.0, "reward": 1.3064830303192139, "reward_std": 0.5317621231079102, "rewards/reward_model/mean": 1.3064830303192139, "rewards/reward_model/std": 0.8767746090888977, "step": 95, "step_time": 168.66238435404375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11376953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.8466796875, "completions/mean_terminated_length": 78.17851257324219, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2149158080574125, "epoch": 0.039538714991762765, "frac_reward_zero_std": 0.0, "grad_norm": 0.5917447209358215, "kl": 0.16629101379658096, "learning_rate": 1.1728395061728396e-06, "loss": 0.0372, "num_tokens": 44365228.0, "reward": 1.310151219367981, "reward_std": 0.5394536852836609, "rewards/reward_model/mean": 1.310151219367981, "rewards/reward_model/std": 0.8472654223442078, "step": 96, "step_time": 170.65063601452857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.087890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.22119140625, "completions/mean_terminated_length": 76.7136001586914, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2331861625425518, "epoch": 0.039950576606260293, "frac_reward_zero_std": 0.0, "grad_norm": 0.6160045266151428, "kl": 0.18450818251585588, "learning_rate": 1.1851851851851852e-06, "loss": 0.0239, "num_tokens": 44809841.0, "reward": 1.388469934463501, "reward_std": 0.47841960191726685, "rewards/reward_model/mean": 1.388469934463501, "rewards/reward_model/std": 0.7604539394378662, "step": 97, "step_time": 168.5139070255682 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14404296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.03857421875, "completions/mean_terminated_length": 78.97718811035156, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2734931902959943, "epoch": 0.04036243822075782, "frac_reward_zero_std": 0.0, "grad_norm": 0.8496507406234741, "kl": 0.1486935554712545, "learning_rate": 1.197530864197531e-06, "loss": 0.024, "num_tokens": 45291392.0, "reward": 1.273500680923462, "reward_std": 0.5131819844245911, "rewards/reward_model/mean": 1.273500680923462, "rewards/reward_model/std": 0.8183842301368713, "step": 98, "step_time": 169.52931605745107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07861328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.30078125, "completions/mean_terminated_length": 71.8897705078125, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.210193380014971, "epoch": 0.04077429983525535, "frac_reward_zero_std": 0.0, "grad_norm": 0.6533998847007751, "kl": 0.16968276555417106, "learning_rate": 1.2098765432098765e-06, "loss": 0.0227, "num_tokens": 45769608.0, "reward": 1.374595046043396, "reward_std": 0.5025352835655212, "rewards/reward_model/mean": 1.374595046043396, "rewards/reward_model/std": 0.7355093359947205, "step": 99, "step_time": 170.76728575211018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10595703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.27490234375, "completions/mean_terminated_length": 74.61878204345703, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.1873215795494616, "epoch": 0.04118616144975288, "frac_reward_zero_std": 0.0, "grad_norm": 0.5959328413009644, "kl": 0.14610896681551822, "learning_rate": 1.2222222222222221e-06, "loss": 0.0218, "num_tokens": 46193339.0, "reward": 1.393322229385376, "reward_std": 0.5245035886764526, "rewards/reward_model/mean": 1.393322229385376, "rewards/reward_model/std": 0.8767962455749512, "step": 100, "step_time": 169.11898464756086 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.0029296875, "completions/mean_terminated_length": 74.7643814086914, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1970643543172628, "epoch": 0.04159802306425041, "frac_reward_zero_std": 0.0, "grad_norm": 0.6192975640296936, "kl": 0.17013628454878926, "learning_rate": 1.234567901234568e-06, "loss": 0.0146, "num_tokens": 46603041.0, "reward": 1.3850064277648926, "reward_std": 0.5491656064987183, "rewards/reward_model/mean": 1.3850064277648926, "rewards/reward_model/std": 0.8517816662788391, "step": 101, "step_time": 172.5300747868605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09130859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.30126953125, "completions/mean_terminated_length": 74.40784454345703, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2366106095723808, "epoch": 0.042009884678747944, "frac_reward_zero_std": 0.0, "grad_norm": 0.6139182448387146, "kl": 0.15838144707959145, "learning_rate": 1.2469135802469135e-06, "loss": 0.0044, "num_tokens": 47074410.0, "reward": 1.4079811573028564, "reward_std": 0.5216450691223145, "rewards/reward_model/mean": 1.4079811573028564, "rewards/reward_model/std": 0.7702781558036804, "step": 102, "step_time": 168.7905627740547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.34375, "completions/mean_terminated_length": 70.88135528564453, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2070911666378379, "epoch": 0.04242174629324547, "frac_reward_zero_std": 0.0, "grad_norm": 0.6438104510307312, "kl": 0.18365478227497078, "learning_rate": 1.2592592592592593e-06, "loss": 0.0099, "num_tokens": 47573482.0, "reward": 1.379532814025879, "reward_std": 0.5358700156211853, "rewards/reward_model/mean": 1.379532814025879, "rewards/reward_model/std": 0.8822428584098816, "step": 103, "step_time": 169.78911154950038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12841796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.322265625, "completions/mean_terminated_length": 73.2974853515625, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.2451989527326077, "epoch": 0.042833607907743, "frac_reward_zero_std": 0.0, "grad_norm": 0.5943405628204346, "kl": 0.15326940282830037, "learning_rate": 1.271604938271605e-06, "loss": 0.0054, "num_tokens": 48002462.0, "reward": 1.2383322715759277, "reward_std": 0.5271602869033813, "rewards/reward_model/mean": 1.2383322715759277, "rewards/reward_model/std": 1.0347553491592407, "step": 104, "step_time": 168.55086909374222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 72.16455078125, "completions/mean_terminated_length": 68.03618621826172, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1801623778883368, "epoch": 0.04324546952224053, "frac_reward_zero_std": 0.0, "grad_norm": 0.6378849148750305, "kl": 0.15821812994545326, "learning_rate": 1.2839506172839507e-06, "loss": 0.0035, "num_tokens": 48392175.0, "reward": 1.5423574447631836, "reward_std": 0.4824356138706207, "rewards/reward_model/mean": 1.5423574447631836, "rewards/reward_model/std": 0.8302909135818481, "step": 105, "step_time": 169.42736366018653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0830078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.6953125, "completions/mean_terminated_length": 74.23216247558594, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.3169332989491522, "epoch": 0.04365733113673806, "frac_reward_zero_std": 0.0, "grad_norm": 0.5943061113357544, "kl": 0.17450666631339118, "learning_rate": 1.2962962962962962e-06, "loss": 0.0041, "num_tokens": 48840319.0, "reward": 1.3128201961517334, "reward_std": 0.5468074679374695, "rewards/reward_model/mean": 1.3128201961517334, "rewards/reward_model/std": 0.9597176909446716, "step": 106, "step_time": 169.50068523269147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08642578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.5673828125, "completions/mean_terminated_length": 68.41796112060547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1956646356265992, "epoch": 0.04406919275123559, "frac_reward_zero_std": 0.0, "grad_norm": 0.9529135823249817, "kl": 0.14988234487827867, "learning_rate": 1.308641975308642e-06, "loss": 0.0263, "num_tokens": 49281225.0, "reward": 1.289332628250122, "reward_std": 0.5048444867134094, "rewards/reward_model/mean": 1.289332628250122, "rewards/reward_model/std": 0.8929986953735352, "step": 107, "step_time": 169.95979618094862 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1025390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.716796875, "completions/mean_terminated_length": 75.31446838378906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2545830043964088, "epoch": 0.044481054365733116, "frac_reward_zero_std": 0.0, "grad_norm": 0.5836382508277893, "kl": 0.14862596249440685, "learning_rate": 1.3209876543209876e-06, "loss": 0.0258, "num_tokens": 49792933.0, "reward": 1.3871095180511475, "reward_std": 0.5084824562072754, "rewards/reward_model/mean": 1.3871095180511475, "rewards/reward_model/std": 0.8852909207344055, "step": 108, "step_time": 169.20244881836697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04638671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 70.8515625, "completions/mean_terminated_length": 68.07168579101562, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2631548107601702, "epoch": 0.044892915980230645, "frac_reward_zero_std": 0.0, "grad_norm": 0.6485495567321777, "kl": 0.1719318167597521, "learning_rate": 1.3333333333333332e-06, "loss": -0.0057, "num_tokens": 50257109.0, "reward": 1.5582630634307861, "reward_std": 0.5006577968597412, "rewards/reward_model/mean": 1.5582630634307861, "rewards/reward_model/std": 0.8305342197418213, "step": 109, "step_time": 168.78190125897527 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 71.658203125, "completions/mean_terminated_length": 68.33712768554688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2364499233663082, "epoch": 0.045304777594728174, "frac_reward_zero_std": 0.0, "grad_norm": 0.588637113571167, "kl": 0.16918645988334902, "learning_rate": 1.345679012345679e-06, "loss": 0.0182, "num_tokens": 50728345.0, "reward": 1.4181112051010132, "reward_std": 0.5059062242507935, "rewards/reward_model/mean": 1.4181112051010132, "rewards/reward_model/std": 0.9040238857269287, "step": 110, "step_time": 169.8461561407894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.97265625, "completions/mean_terminated_length": 75.17507934570312, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.3138902625069022, "epoch": 0.0457166392092257, "frac_reward_zero_std": 0.0, "grad_norm": 0.5418975353240967, "kl": 0.16301694582216442, "learning_rate": 1.3580246913580248e-06, "loss": 0.0118, "num_tokens": 51243937.0, "reward": 1.2584481239318848, "reward_std": 0.4887618124485016, "rewards/reward_model/mean": 1.2584481239318848, "rewards/reward_model/std": 0.9598668813705444, "step": 111, "step_time": 169.55967817036435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10009765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.0, "completions/mean_terminated_length": 71.32718658447266, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2457992839626968, "epoch": 0.04612850082372323, "frac_reward_zero_std": 0.0, "grad_norm": 0.7095631957054138, "kl": 0.18068589709582739, "learning_rate": 1.3703703703703704e-06, "loss": 0.0037, "num_tokens": 51729121.0, "reward": 1.4246959686279297, "reward_std": 0.44748643040657043, "rewards/reward_model/mean": 1.4246959686279297, "rewards/reward_model/std": 0.8918141722679138, "step": 112, "step_time": 169.78249773895368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.623046875, "completions/mean_terminated_length": 69.17062377929688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2187957542482764, "epoch": 0.04654036243822076, "frac_reward_zero_std": 0.0, "grad_norm": 0.9400354623794556, "kl": 0.22693650811561383, "learning_rate": 1.3827160493827162e-06, "loss": 0.0122, "num_tokens": 52150877.0, "reward": 1.5303287506103516, "reward_std": 0.4950566291809082, "rewards/reward_model/mean": 1.5303287506103516, "rewards/reward_model/std": 0.8175535798072815, "step": 113, "step_time": 169.6679522804916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.8798828125, "completions/mean_terminated_length": 73.89456939697266, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2598798810504377, "epoch": 0.04695222405271829, "frac_reward_zero_std": 0.0, "grad_norm": 0.5663942098617554, "kl": 0.18427117861574516, "learning_rate": 1.3950617283950617e-06, "loss": 0.0113, "num_tokens": 52610119.0, "reward": 1.4505566358566284, "reward_std": 0.47873079776763916, "rewards/reward_model/mean": 1.4505566358566284, "rewards/reward_model/std": 0.9025180339813232, "step": 114, "step_time": 168.86495931399986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.5009765625, "completions/mean_terminated_length": 73.12487030029297, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2603700500912964, "epoch": 0.04736408566721582, "frac_reward_zero_std": 0.0, "grad_norm": 0.594477653503418, "kl": 0.1864922004751861, "learning_rate": 1.4074074074074073e-06, "loss": 0.0007, "num_tokens": 53070697.0, "reward": 1.5104587078094482, "reward_std": 0.4391752779483795, "rewards/reward_model/mean": 1.5104587078094482, "rewards/reward_model/std": 0.7633559703826904, "step": 115, "step_time": 170.47059550089762 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08544921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.56005859375, "completions/mean_terminated_length": 75.03417205810547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2630685805343091, "epoch": 0.047775947281713346, "frac_reward_zero_std": 0.0, "grad_norm": 0.5976677536964417, "kl": 0.17307385959429666, "learning_rate": 1.4197530864197531e-06, "loss": 0.0185, "num_tokens": 53542148.0, "reward": 1.419188141822815, "reward_std": 0.4496381878852844, "rewards/reward_model/mean": 1.419188141822815, "rewards/reward_model/std": 0.8634824156761169, "step": 116, "step_time": 169.13070647930726 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.076171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.9912109375, "completions/mean_terminated_length": 79.2801284790039, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.3025836027227342, "epoch": 0.048187808896210875, "frac_reward_zero_std": 0.0, "grad_norm": 0.5448313355445862, "kl": 0.1677903521922417, "learning_rate": 1.4320987654320987e-06, "loss": 0.0104, "num_tokens": 53986130.0, "reward": 1.4435430765151978, "reward_std": 0.4609750807285309, "rewards/reward_model/mean": 1.4435430765151978, "rewards/reward_model/std": 0.7846410870552063, "step": 117, "step_time": 169.9405871666968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0830078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.50244140625, "completions/mean_terminated_length": 74.0218276977539, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.288459766190499, "epoch": 0.048599670510708404, "frac_reward_zero_std": 0.0, "grad_norm": 0.5833011865615845, "kl": 0.1983755056571681, "learning_rate": 1.4444444444444445e-06, "loss": 0.0079, "num_tokens": 54439799.0, "reward": 1.4345738887786865, "reward_std": 0.48493313789367676, "rewards/reward_model/mean": 1.4345738887786865, "rewards/reward_model/std": 0.8746789693832397, "step": 118, "step_time": 169.1852500126697 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07373046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.51513671875, "completions/mean_terminated_length": 74.576171875, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.224245477002114, "epoch": 0.04901153212520593, "frac_reward_zero_std": 0.0, "grad_norm": 0.6017325520515442, "kl": 0.18918700981885195, "learning_rate": 1.4567901234567903e-06, "loss": 0.0202, "num_tokens": 54875606.0, "reward": 1.5676627159118652, "reward_std": 0.4177808165550232, "rewards/reward_model/mean": 1.5676627159118652, "rewards/reward_model/std": 0.854965329170227, "step": 119, "step_time": 169.29946460714564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10595703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.9853515625, "completions/mean_terminated_length": 72.05789184570312, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.209789669374004, "epoch": 0.04942339373970346, "frac_reward_zero_std": 0.0, "grad_norm": 0.8375149369239807, "kl": 0.20605222514132038, "learning_rate": 1.4691358024691359e-06, "loss": 0.016, "num_tokens": 55328024.0, "reward": 1.4581849575042725, "reward_std": 0.4875085949897766, "rewards/reward_model/mean": 1.4581849575042725, "rewards/reward_model/std": 0.9500890374183655, "step": 120, "step_time": 169.78462026640773 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11181640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.04736328125, "completions/mean_terminated_length": 78.51402282714844, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.232045111246407, "epoch": 0.04983525535420099, "frac_reward_zero_std": 0.0, "grad_norm": 0.5587739944458008, "kl": 0.18681989194010384, "learning_rate": 1.4814814814814815e-06, "loss": 0.0177, "num_tokens": 55771033.0, "reward": 1.4740581512451172, "reward_std": 0.4508310556411743, "rewards/reward_model/mean": 1.4740581512451172, "rewards/reward_model/std": 0.8630524277687073, "step": 121, "step_time": 168.61212700419128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08349609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.2998046875, "completions/mean_terminated_length": 71.58977508544922, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2581788282841444, "epoch": 0.05024711696869852, "frac_reward_zero_std": 0.0, "grad_norm": 0.5757594704627991, "kl": 0.19532652158522978, "learning_rate": 1.4938271604938272e-06, "loss": 0.011, "num_tokens": 56215583.0, "reward": 1.4256713390350342, "reward_std": 0.4282228946685791, "rewards/reward_model/mean": 1.4256713390350342, "rewards/reward_model/std": 0.9199231863021851, "step": 122, "step_time": 168.55926717165858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0947265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.01416015625, "completions/mean_terminated_length": 72.78370666503906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2156934614758939, "epoch": 0.05065897858319605, "frac_reward_zero_std": 0.0, "grad_norm": 0.5999395847320557, "kl": 0.19728930137353018, "learning_rate": 1.506172839506173e-06, "loss": 0.0113, "num_tokens": 56669980.0, "reward": 1.6116740703582764, "reward_std": 0.4245319962501526, "rewards/reward_model/mean": 1.6116740703582764, "rewards/reward_model/std": 0.7885070443153381, "step": 123, "step_time": 168.9781666644849 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.9921875, "completions/mean_terminated_length": 72.8738784790039, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2410368812270463, "epoch": 0.051070840197693576, "frac_reward_zero_std": 0.0, "grad_norm": 0.5885641574859619, "kl": 0.22556541379890405, "learning_rate": 1.5185185185185186e-06, "loss": 0.0278, "num_tokens": 57086316.0, "reward": 1.7070322036743164, "reward_std": 0.4219028055667877, "rewards/reward_model/mean": 1.7070322036743164, "rewards/reward_model/std": 0.7904437184333801, "step": 124, "step_time": 168.52942496724427 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14599609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.66748046875, "completions/mean_terminated_length": 74.91766357421875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2711665122769773, "epoch": 0.051482701812191105, "frac_reward_zero_std": 0.0, "grad_norm": 0.5652002096176147, "kl": 0.21976503301993944, "learning_rate": 1.5308641975308642e-06, "loss": 0.025, "num_tokens": 57550787.0, "reward": 1.5571563243865967, "reward_std": 0.43854257464408875, "rewards/reward_model/mean": 1.5571563243865967, "rewards/reward_model/std": 0.9019820690155029, "step": 125, "step_time": 169.8311795401387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.79931640625, "completions/mean_terminated_length": 76.47174835205078, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2508959705010056, "epoch": 0.051894563426688634, "frac_reward_zero_std": 0.0, "grad_norm": 0.587785542011261, "kl": 0.2117367441242095, "learning_rate": 1.5432098765432098e-06, "loss": 0.0343, "num_tokens": 57980552.0, "reward": 1.6625767946243286, "reward_std": 0.41725289821624756, "rewards/reward_model/mean": 1.6625767946243286, "rewards/reward_model/std": 0.781819760799408, "step": 126, "step_time": 170.32025544391945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.45458984375, "completions/mean_terminated_length": 71.35691833496094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2673575547523797, "epoch": 0.05230642504118616, "frac_reward_zero_std": 0.0, "grad_norm": 0.663336992263794, "kl": 0.2514068058517296, "learning_rate": 1.5555555555555556e-06, "loss": 0.007, "num_tokens": 58421643.0, "reward": 1.6073158979415894, "reward_std": 0.431104838848114, "rewards/reward_model/mean": 1.6073158979415894, "rewards/reward_model/std": 0.7830482125282288, "step": 127, "step_time": 167.69548717467114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05712890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.8486328125, "completions/mean_terminated_length": 76.93112182617188, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.3203896265476942, "epoch": 0.05271828665568369, "frac_reward_zero_std": 0.0, "grad_norm": 0.5664064288139343, "kl": 0.21401835474534892, "learning_rate": 1.5679012345679012e-06, "loss": 0.0218, "num_tokens": 58904437.0, "reward": 1.5991525650024414, "reward_std": 0.4234989285469055, "rewards/reward_model/mean": 1.5991525650024414, "rewards/reward_model/std": 0.8423987030982971, "step": 128, "step_time": 170.09795808279887 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0693359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.70556640625, "completions/mean_terminated_length": 72.88404846191406, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2859586202539504, "epoch": 0.05313014827018122, "frac_reward_zero_std": 0.0, "grad_norm": 0.5938490033149719, "kl": 0.23812477025785483, "learning_rate": 1.5802469135802472e-06, "loss": 0.0198, "num_tokens": 59368826.0, "reward": 1.4986733198165894, "reward_std": 0.48006802797317505, "rewards/reward_model/mean": 1.4986733198165894, "rewards/reward_model/std": 0.9735957384109497, "step": 129, "step_time": 168.860689394176 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07666015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.9814453125, "completions/mean_terminated_length": 74.91168975830078, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.210738726425916, "epoch": 0.05354200988467875, "frac_reward_zero_std": 0.0, "grad_norm": 0.5862839818000793, "kl": 0.21009712052182294, "learning_rate": 1.5925925925925927e-06, "loss": 0.0074, "num_tokens": 59805460.0, "reward": 1.6455633640289307, "reward_std": 0.4051080346107483, "rewards/reward_model/mean": 1.6455633640289307, "rewards/reward_model/std": 0.902286946773529, "step": 130, "step_time": 169.57672298140824 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0595703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.908203125, "completions/mean_terminated_length": 72.6085205078125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2665375941433012, "epoch": 0.05395387149917628, "frac_reward_zero_std": 0.0, "grad_norm": 0.5997110605239868, "kl": 0.2307097713346593, "learning_rate": 1.6049382716049383e-06, "loss": 0.0136, "num_tokens": 60283128.0, "reward": 1.6560347080230713, "reward_std": 0.3901210129261017, "rewards/reward_model/mean": 1.6560347080230713, "rewards/reward_model/std": 0.7428885698318481, "step": 131, "step_time": 170.24099622154608 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.75537109375, "completions/mean_terminated_length": 72.64718627929688, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.280121157411486, "epoch": 0.054365733113673806, "frac_reward_zero_std": 0.0, "grad_norm": 0.6451330780982971, "kl": 0.24510033716796897, "learning_rate": 1.617283950617284e-06, "loss": 0.0219, "num_tokens": 60704675.0, "reward": 1.5663776397705078, "reward_std": 0.4158337414264679, "rewards/reward_model/mean": 1.5663776397705078, "rewards/reward_model/std": 0.8126929402351379, "step": 132, "step_time": 168.72953157825395 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03466796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 72.27392578125, "completions/mean_terminated_length": 70.27263641357422, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.255036068148911, "epoch": 0.054777594728171335, "frac_reward_zero_std": 0.0, "grad_norm": 0.5956186056137085, "kl": 0.22500447847414762, "learning_rate": 1.6296296296296295e-06, "loss": 0.0051, "num_tokens": 61135252.0, "reward": 1.5705558061599731, "reward_std": 0.4293729066848755, "rewards/reward_model/mean": 1.5705558061599731, "rewards/reward_model/std": 0.8589349985122681, "step": 133, "step_time": 169.56890073092654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0849609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.0068359375, "completions/mean_terminated_length": 76.64353942871094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.22501984029077, "epoch": 0.05518945634266886, "frac_reward_zero_std": 0.0, "grad_norm": 0.6271244883537292, "kl": 0.19278798706363887, "learning_rate": 1.6419753086419753e-06, "loss": 0.0213, "num_tokens": 61582082.0, "reward": 1.4659600257873535, "reward_std": 0.4655250310897827, "rewards/reward_model/mean": 1.4659600257873535, "rewards/reward_model/std": 0.8552236557006836, "step": 134, "step_time": 169.07396916579455 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06396484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.24267578125, "completions/mean_terminated_length": 73.77412414550781, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2544868639670312, "epoch": 0.05560131795716639, "frac_reward_zero_std": 0.0, "grad_norm": 0.55135577917099, "kl": 0.21390190807869658, "learning_rate": 1.654320987654321e-06, "loss": 0.016, "num_tokens": 62051859.0, "reward": 1.7084484100341797, "reward_std": 0.42418497800827026, "rewards/reward_model/mean": 1.7084484100341797, "rewards/reward_model/std": 0.8365797996520996, "step": 135, "step_time": 168.55205999454483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06396484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.744140625, "completions/mean_terminated_length": 73.24151611328125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2688625399023294, "epoch": 0.05601317957166392, "frac_reward_zero_std": 0.0, "grad_norm": 0.6378998160362244, "kl": 0.23519006837159395, "learning_rate": 1.6666666666666669e-06, "loss": 0.0138, "num_tokens": 62528583.0, "reward": 1.6745352745056152, "reward_std": 0.4303410053253174, "rewards/reward_model/mean": 1.6745352745056152, "rewards/reward_model/std": 0.8760103583335876, "step": 136, "step_time": 167.9896323014982 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0771484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.08544921875, "completions/mean_terminated_length": 78.2470932006836, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.298474219162017, "epoch": 0.05642504118616145, "frac_reward_zero_std": 0.0, "grad_norm": 0.5870314240455627, "kl": 0.19887266453588381, "learning_rate": 1.6790123456790125e-06, "loss": 0.0049, "num_tokens": 63017686.0, "reward": 1.5558466911315918, "reward_std": 0.41865676641464233, "rewards/reward_model/mean": 1.5558466911315918, "rewards/reward_model/std": 0.8572432994842529, "step": 137, "step_time": 168.25818043760955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.044921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.47802734375, "completions/mean_terminated_length": 71.96063232421875, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2788015282712877, "epoch": 0.05683690280065898, "frac_reward_zero_std": 0.0, "grad_norm": 1.0465470552444458, "kl": 0.28733671543886885, "learning_rate": 1.691358024691358e-06, "loss": 0.0283, "num_tokens": 63458441.0, "reward": 1.8364841938018799, "reward_std": 0.3914712369441986, "rewards/reward_model/mean": 1.8364841938018799, "rewards/reward_model/std": 0.8168711066246033, "step": 138, "step_time": 167.18661137623712 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.74072265625, "completions/mean_terminated_length": 73.87821960449219, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2473736926913261, "epoch": 0.05724876441515651, "frac_reward_zero_std": 0.0, "grad_norm": 0.5875552892684937, "kl": 0.22631295106839389, "learning_rate": 1.7037037037037036e-06, "loss": 0.0339, "num_tokens": 63941110.0, "reward": 1.70628821849823, "reward_std": 0.4453471899032593, "rewards/reward_model/mean": 1.70628821849823, "rewards/reward_model/std": 0.8872243762016296, "step": 139, "step_time": 169.1617161957547 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08740234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.11279296875, "completions/mean_terminated_length": 74.43070983886719, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2679406760726124, "epoch": 0.057660626029654036, "frac_reward_zero_std": 0.0, "grad_norm": 0.6072338819503784, "kl": 0.2250406270613894, "learning_rate": 1.7160493827160492e-06, "loss": 0.0179, "num_tokens": 64478525.0, "reward": 1.731923222541809, "reward_std": 0.3971579968929291, "rewards/reward_model/mean": 1.731923222541809, "rewards/reward_model/std": 0.8387468457221985, "step": 140, "step_time": 169.69765338627622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06689453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.95458984375, "completions/mean_terminated_length": 74.3668212890625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.3288813619874418, "epoch": 0.058072487644151564, "frac_reward_zero_std": 0.0, "grad_norm": 0.9750301241874695, "kl": 0.22281288140220568, "learning_rate": 1.7283950617283952e-06, "loss": 0.0086, "num_tokens": 64943904.0, "reward": 1.5512080192565918, "reward_std": 0.4345375895500183, "rewards/reward_model/mean": 1.5512080192565918, "rewards/reward_model/std": 0.8531954288482666, "step": 141, "step_time": 169.35747446445748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09033203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.90966796875, "completions/mean_terminated_length": 71.8362808227539, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2554950527846813, "epoch": 0.05848434925864909, "frac_reward_zero_std": 0.0, "grad_norm": 0.5846207141876221, "kl": 0.22971162866451778, "learning_rate": 1.7407407407407408e-06, "loss": 0.0123, "num_tokens": 65435015.0, "reward": 1.6992193460464478, "reward_std": 0.38921838998794556, "rewards/reward_model/mean": 1.6992193460464478, "rewards/reward_model/std": 0.8802086710929871, "step": 142, "step_time": 169.55797945754603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.7138671875, "completions/mean_terminated_length": 74.08013153076172, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.241849954240024, "epoch": 0.05889621087314662, "frac_reward_zero_std": 0.0, "grad_norm": 0.5389864444732666, "kl": 0.19780789304059, "learning_rate": 1.7530864197530866e-06, "loss": 0.0162, "num_tokens": 65893181.0, "reward": 1.657914638519287, "reward_std": 0.43264538049697876, "rewards/reward_model/mean": 1.657914638519287, "rewards/reward_model/std": 0.9311876893043518, "step": 143, "step_time": 167.69665700104088 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.701171875, "completions/mean_terminated_length": 75.52391052246094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.3291778746061027, "epoch": 0.05930807248764415, "frac_reward_zero_std": 0.0, "grad_norm": 0.5641889572143555, "kl": 0.23325793404364958, "learning_rate": 1.7654320987654322e-06, "loss": 0.0106, "num_tokens": 66306777.0, "reward": 1.6250760555267334, "reward_std": 0.45928484201431274, "rewards/reward_model/mean": 1.6250760555267334, "rewards/reward_model/std": 0.8440907001495361, "step": 144, "step_time": 167.09111699229106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.5625, "completions/mean_terminated_length": 73.81069946289062, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.2597985244356096, "epoch": 0.05971993410214168, "frac_reward_zero_std": 0.0, "grad_norm": 0.6075507402420044, "kl": 0.2127083102823235, "learning_rate": 1.7777777777777777e-06, "loss": -0.0003, "num_tokens": 66685465.0, "reward": 1.637428641319275, "reward_std": 0.4271353781223297, "rewards/reward_model/mean": 1.637428641319275, "rewards/reward_model/std": 0.9868490099906921, "step": 145, "step_time": 168.623529009521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.47509765625, "completions/mean_terminated_length": 75.8676986694336, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2390871203970164, "epoch": 0.06013179571663921, "frac_reward_zero_std": 0.0, "grad_norm": 0.738397479057312, "kl": 0.29930633143521845, "learning_rate": 1.7901234567901233e-06, "loss": 0.0243, "num_tokens": 67148166.0, "reward": 1.8085708618164062, "reward_std": 0.4322406053543091, "rewards/reward_model/mean": 1.8085708618164062, "rewards/reward_model/std": 0.8529988527297974, "step": 146, "step_time": 168.20390673354268 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.037109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.32373046875, "completions/mean_terminated_length": 74.33214569091797, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.2929235147312284, "epoch": 0.06054365733113674, "frac_reward_zero_std": 0.0, "grad_norm": 0.6655780673027039, "kl": 0.22947472316445783, "learning_rate": 1.8024691358024693e-06, "loss": 0.0168, "num_tokens": 67600669.0, "reward": 1.878852128982544, "reward_std": 0.38836777210235596, "rewards/reward_model/mean": 1.878852128982544, "rewards/reward_model/std": 0.9328538179397583, "step": 147, "step_time": 169.04876817949116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.455078125, "completions/mean_terminated_length": 73.3164291381836, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.300009768921882, "epoch": 0.060955518945634266, "frac_reward_zero_std": 0.0, "grad_norm": 0.5745586156845093, "kl": 0.23608133487869054, "learning_rate": 1.814814814814815e-06, "loss": 0.0142, "num_tokens": 68108001.0, "reward": 1.627516269683838, "reward_std": 0.43401244282722473, "rewards/reward_model/mean": 1.627516269683838, "rewards/reward_model/std": 0.9561209082603455, "step": 148, "step_time": 167.96074909390882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.375, "completions/mean_terminated_length": 73.63188171386719, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2789768348447978, "epoch": 0.061367380560131794, "frac_reward_zero_std": 0.0, "grad_norm": 0.6077526807785034, "kl": 0.2504324442707002, "learning_rate": 1.8271604938271605e-06, "loss": 0.0138, "num_tokens": 68588705.0, "reward": 1.7919840812683105, "reward_std": 0.40561193227767944, "rewards/reward_model/mean": 1.7919840812683105, "rewards/reward_model/std": 0.807359516620636, "step": 149, "step_time": 167.94775600917637 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0693359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.6728515625, "completions/mean_terminated_length": 77.14690399169922, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.286247756332159, "epoch": 0.06177924217462932, "frac_reward_zero_std": 0.0, "grad_norm": 0.5670334100723267, "kl": 0.22240044514182955, "learning_rate": 1.839506172839506e-06, "loss": 0.0155, "num_tokens": 69043619.0, "reward": 1.5120161771774292, "reward_std": 0.4330858290195465, "rewards/reward_model/mean": 1.5120161771774292, "rewards/reward_model/std": 0.8432108759880066, "step": 150, "step_time": 168.81714980350807 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.8125, "completions/mean_terminated_length": 76.70686340332031, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2792057455517352, "epoch": 0.06219110378912685, "frac_reward_zero_std": 0.0, "grad_norm": 0.5719607472419739, "kl": 0.25582361221313477, "learning_rate": 1.8518518518518519e-06, "loss": 0.0076, "num_tokens": 69484995.0, "reward": 1.7927849292755127, "reward_std": 0.41597551107406616, "rewards/reward_model/mean": 1.7927849292755127, "rewards/reward_model/std": 0.9223642945289612, "step": 151, "step_time": 167.4020181344822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0498046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.515625, "completions/mean_terminated_length": 73.81706237792969, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.227274117525667, "epoch": 0.06260296540362438, "frac_reward_zero_std": 0.0, "grad_norm": 0.7039728164672852, "kl": 0.26282261524465866, "learning_rate": 1.8641975308641975e-06, "loss": 0.0056, "num_tokens": 69909667.0, "reward": 1.8002865314483643, "reward_std": 0.41272586584091187, "rewards/reward_model/mean": 1.8002865314483643, "rewards/reward_model/std": 0.8000503778457642, "step": 152, "step_time": 167.92274621222168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06103515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.96826171875, "completions/mean_terminated_length": 76.8460693359375, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.273845980875194, "epoch": 0.06301482701812192, "frac_reward_zero_std": 0.0, "grad_norm": 0.5925673246383667, "kl": 0.22481732474989258, "learning_rate": 1.8765432098765435e-06, "loss": 0.0193, "num_tokens": 70384674.0, "reward": 1.8214460611343384, "reward_std": 0.413318932056427, "rewards/reward_model/mean": 1.8214460611343384, "rewards/reward_model/std": 0.742709219455719, "step": 153, "step_time": 170.70164536684752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04638671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.90673828125, "completions/mean_terminated_length": 73.37275695800781, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2647815430536866, "epoch": 0.06342668863261944, "frac_reward_zero_std": 0.0, "grad_norm": 0.6434057354927063, "kl": 0.30792183699668385, "learning_rate": 1.888888888888889e-06, "loss": 0.0344, "num_tokens": 70848451.0, "reward": 1.8151272535324097, "reward_std": 0.40342289209365845, "rewards/reward_model/mean": 1.8151272535324097, "rewards/reward_model/std": 0.8678472638130188, "step": 154, "step_time": 168.5453836512752 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.47216796875, "completions/mean_terminated_length": 74.96002197265625, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.269783977419138, "epoch": 0.06383855024711697, "frac_reward_zero_std": 0.0, "grad_norm": 0.6314256191253662, "kl": 0.2555277651408687, "learning_rate": 1.9012345679012346e-06, "loss": 0.0251, "num_tokens": 71328554.0, "reward": 1.7019236087799072, "reward_std": 0.4237982928752899, "rewards/reward_model/mean": 1.7019236087799072, "rewards/reward_model/std": 0.913361132144928, "step": 155, "step_time": 169.69989513559267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09521484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.9814453125, "completions/mean_terminated_length": 83.77010345458984, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.262259678915143, "epoch": 0.0642504118616145, "frac_reward_zero_std": 0.0, "grad_norm": 0.5448090434074402, "kl": 0.22080097059370019, "learning_rate": 1.9135802469135804e-06, "loss": 0.0095, "num_tokens": 71843364.0, "reward": 1.6485573053359985, "reward_std": 0.442721962928772, "rewards/reward_model/mean": 1.6485573053359985, "rewards/reward_model/std": 0.9059621691703796, "step": 156, "step_time": 168.91886990657076 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.076171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.45654296875, "completions/mean_terminated_length": 79.78382873535156, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2155877850018442, "epoch": 0.06466227347611203, "frac_reward_zero_std": 0.0, "grad_norm": 0.6289539933204651, "kl": 0.2453711698180996, "learning_rate": 1.925925925925926e-06, "loss": 0.026, "num_tokens": 72273771.0, "reward": 1.784334659576416, "reward_std": 0.4161246418952942, "rewards/reward_model/mean": 1.784334659576416, "rewards/reward_model/std": 0.757505476474762, "step": 157, "step_time": 167.44150482024997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08154296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.76904296875, "completions/mean_terminated_length": 77.66453552246094, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.235411421628669, "epoch": 0.06507413509060955, "frac_reward_zero_std": 0.0, "grad_norm": 0.5421144366264343, "kl": 0.22543006701744162, "learning_rate": 1.9382716049382716e-06, "loss": 0.0316, "num_tokens": 72800146.0, "reward": 1.7696501016616821, "reward_std": 0.3846498727798462, "rewards/reward_model/mean": 1.7696501016616821, "rewards/reward_model/std": 0.8624175786972046, "step": 158, "step_time": 169.24183974647895 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08642578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.5419921875, "completions/mean_terminated_length": 76.05238342285156, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.185378910973668, "epoch": 0.06548599670510709, "frac_reward_zero_std": 0.0, "grad_norm": 0.5627242922782898, "kl": 0.23744830160285346, "learning_rate": 1.9506172839506176e-06, "loss": 0.0326, "num_tokens": 73252072.0, "reward": 1.8059046268463135, "reward_std": 0.3923932611942291, "rewards/reward_model/mean": 1.8059046268463135, "rewards/reward_model/std": 0.8797404766082764, "step": 159, "step_time": 169.05729650333524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.080078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.8115234375, "completions/mean_terminated_length": 77.79087829589844, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.23535214853473, "epoch": 0.06589785831960461, "frac_reward_zero_std": 0.0, "grad_norm": 0.5441705584526062, "kl": 0.24602960981428623, "learning_rate": 1.962962962962963e-06, "loss": 0.0213, "num_tokens": 73739494.0, "reward": 1.7673776149749756, "reward_std": 0.4512333571910858, "rewards/reward_model/mean": 1.7673776149749756, "rewards/reward_model/std": 0.9724159240722656, "step": 160, "step_time": 168.9636302953586 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.20166015625, "completions/mean_terminated_length": 74.46351623535156, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2300576204434037, "epoch": 0.06630971993410215, "frac_reward_zero_std": 0.0, "grad_norm": 0.5943037271499634, "kl": 0.24898978223791346, "learning_rate": 1.9753086419753087e-06, "loss": 0.0222, "num_tokens": 74167715.0, "reward": 1.926592469215393, "reward_std": 0.41215306520462036, "rewards/reward_model/mean": 1.926592469215393, "rewards/reward_model/std": 1.001451015472412, "step": 161, "step_time": 167.6384666627273 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 73.8603515625, "completions/mean_terminated_length": 69.85736846923828, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.198827504646033, "epoch": 0.06672158154859967, "frac_reward_zero_std": 0.0, "grad_norm": 0.5532698035240173, "kl": 0.27067929052282125, "learning_rate": 1.9876543209876543e-06, "loss": 0.0219, "num_tokens": 74647397.0, "reward": 1.9008184671401978, "reward_std": 0.38047462701797485, "rewards/reward_model/mean": 1.9008184671401978, "rewards/reward_model/std": 0.9279819130897522, "step": 162, "step_time": 167.92858559498563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0458984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.6513671875, "completions/mean_terminated_length": 73.133056640625, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2209027321077883, "epoch": 0.0671334431630972, "frac_reward_zero_std": 0.0, "grad_norm": 0.6697596907615662, "kl": 0.27582967511261813, "learning_rate": 2e-06, "loss": 0.0013, "num_tokens": 75100251.0, "reward": 1.778894305229187, "reward_std": 0.40786850452423096, "rewards/reward_model/mean": 1.778894305229187, "rewards/reward_model/std": 0.9986056089401245, "step": 163, "step_time": 168.67196059180424 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.04296875, "completions/mean_terminated_length": 73.45977020263672, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1774590322747827, "epoch": 0.06754530477759473, "frac_reward_zero_std": 0.0, "grad_norm": 0.5625696182250977, "kl": 0.24044052005046979, "learning_rate": 2.012345679012346e-06, "loss": 0.0079, "num_tokens": 75646451.0, "reward": 1.668849229812622, "reward_std": 0.40883883833885193, "rewards/reward_model/mean": 1.668849229812622, "rewards/reward_model/std": 0.9833309650421143, "step": 164, "step_time": 168.17798956111073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.30810546875, "completions/mean_terminated_length": 75.3790054321289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2012779735960066, "epoch": 0.06795716639209226, "frac_reward_zero_std": 0.0, "grad_norm": 0.647919774055481, "kl": 0.24133505855570547, "learning_rate": 2.0246913580246915e-06, "loss": -0.0057, "num_tokens": 76075114.0, "reward": 1.8032076358795166, "reward_std": 0.4206048548221588, "rewards/reward_model/mean": 1.8032076358795166, "rewards/reward_model/std": 0.8603482842445374, "step": 165, "step_time": 167.6303828060627 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0712890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.9677734375, "completions/mean_terminated_length": 79.51103973388672, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2237120219506323, "epoch": 0.06836902800658978, "frac_reward_zero_std": 0.0, "grad_norm": 0.5906450748443604, "kl": 0.23315264994744211, "learning_rate": 2.037037037037037e-06, "loss": 0.0018, "num_tokens": 76518888.0, "reward": 1.6204566955566406, "reward_std": 0.43912777304649353, "rewards/reward_model/mean": 1.6204566955566406, "rewards/reward_model/std": 0.8727616667747498, "step": 166, "step_time": 169.20850368216634 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04345703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.6572265625, "completions/mean_terminated_length": 76.41551971435547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1896002353169024, "epoch": 0.06878088962108732, "frac_reward_zero_std": 0.0, "grad_norm": 0.6026791930198669, "kl": 0.27406785456696525, "learning_rate": 2.0493827160493827e-06, "loss": 0.0206, "num_tokens": 77015562.0, "reward": 1.8125802278518677, "reward_std": 0.3493039608001709, "rewards/reward_model/mean": 1.8125802278518677, "rewards/reward_model/std": 0.8256182074546814, "step": 167, "step_time": 167.97593408357352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.49560546875, "completions/mean_terminated_length": 77.59119415283203, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.2243428956717253, "epoch": 0.06919275123558484, "frac_reward_zero_std": 0.0, "grad_norm": 0.5614490509033203, "kl": 0.27035874902503565, "learning_rate": 2.0617283950617282e-06, "loss": 0.0263, "num_tokens": 77461185.0, "reward": 1.7474470138549805, "reward_std": 0.345928430557251, "rewards/reward_model/mean": 1.7474470138549805, "rewards/reward_model/std": 0.783444344997406, "step": 168, "step_time": 167.4764309921302 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.9755859375, "completions/mean_terminated_length": 82.66261291503906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2144196771550924, "epoch": 0.06960461285008238, "frac_reward_zero_std": 0.0, "grad_norm": 0.7678112387657166, "kl": 0.2838464966916945, "learning_rate": 2.074074074074074e-06, "loss": 0.0225, "num_tokens": 77914863.0, "reward": 1.836325764656067, "reward_std": 0.39190569519996643, "rewards/reward_model/mean": 1.836325764656067, "rewards/reward_model/std": 0.8917631506919861, "step": 169, "step_time": 167.54137054365128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1064453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.44287109375, "completions/mean_terminated_length": 78.13497924804688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.215761701343581, "epoch": 0.0700164744645799, "frac_reward_zero_std": 0.0, "grad_norm": 0.551048219203949, "kl": 0.2747056516527664, "learning_rate": 2.08641975308642e-06, "loss": 0.0378, "num_tokens": 78387770.0, "reward": 1.7706103324890137, "reward_std": 0.3842095136642456, "rewards/reward_model/mean": 1.7706103324890137, "rewards/reward_model/std": 0.9329609274864197, "step": 170, "step_time": 168.65306692710146 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11474609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.2822265625, "completions/mean_terminated_length": 80.87478637695312, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.2296040374785662, "epoch": 0.07042833607907743, "frac_reward_zero_std": 0.0, "grad_norm": 0.6304351091384888, "kl": 0.2613049374194816, "learning_rate": 2.0987654320987654e-06, "loss": 0.0244, "num_tokens": 78848220.0, "reward": 1.8255634307861328, "reward_std": 0.3948723077774048, "rewards/reward_model/mean": 1.8255634307861328, "rewards/reward_model/std": 0.8326601982116699, "step": 171, "step_time": 168.58243151567876 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1396484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.66064453125, "completions/mean_terminated_length": 83.43757629394531, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1852441588416696, "epoch": 0.07084019769357495, "frac_reward_zero_std": 0.0, "grad_norm": 0.5359068512916565, "kl": 0.25176908579305746, "learning_rate": 2.1111111111111114e-06, "loss": 0.0259, "num_tokens": 79300933.0, "reward": 1.806993007659912, "reward_std": 0.37374427914619446, "rewards/reward_model/mean": 1.806993007659912, "rewards/reward_model/std": 0.8301866054534912, "step": 172, "step_time": 168.35271108709276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13720703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.8330078125, "completions/mean_terminated_length": 79.12733459472656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2571615048218518, "epoch": 0.07125205930807249, "frac_reward_zero_std": 0.0, "grad_norm": 0.5572459101676941, "kl": 0.2960540017229505, "learning_rate": 2.123456790123457e-06, "loss": 0.0209, "num_tokens": 79797039.0, "reward": 1.8621840476989746, "reward_std": 0.40364766120910645, "rewards/reward_model/mean": 1.8621840476989746, "rewards/reward_model/std": 0.9432704448699951, "step": 173, "step_time": 167.40115015720949 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0693359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.65771484375, "completions/mean_terminated_length": 76.05613708496094, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.2446925514377654, "epoch": 0.07166392092257001, "frac_reward_zero_std": 0.0, "grad_norm": 0.5800526142120361, "kl": 0.2971960572176613, "learning_rate": 2.1358024691358026e-06, "loss": 0.0368, "num_tokens": 80245682.0, "reward": 1.9920142889022827, "reward_std": 0.3743855655193329, "rewards/reward_model/mean": 1.9920142889022827, "rewards/reward_model/std": 0.9964386820793152, "step": 174, "step_time": 168.70972929289564 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.859375, "completions/mean_terminated_length": 78.10645294189453, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2962157218717039, "epoch": 0.07207578253706755, "frac_reward_zero_std": 0.0, "grad_norm": 0.6903020143508911, "kl": 0.2973472154699266, "learning_rate": 2.148148148148148e-06, "loss": 0.0265, "num_tokens": 80705938.0, "reward": 1.7820935249328613, "reward_std": 0.37934696674346924, "rewards/reward_model/mean": 1.7820935249328613, "rewards/reward_model/std": 0.902191162109375, "step": 175, "step_time": 169.36274442402646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.068359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.72021484375, "completions/mean_terminated_length": 78.32441711425781, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 1.27019348484464, "epoch": 0.07248764415156507, "frac_reward_zero_std": 0.0, "grad_norm": 0.5628058910369873, "kl": 0.2966971327550709, "learning_rate": 2.160493827160494e-06, "loss": 0.0268, "num_tokens": 81138997.0, "reward": 1.7612276077270508, "reward_std": 0.41708946228027344, "rewards/reward_model/mean": 1.7612276077270508, "rewards/reward_model/std": 0.9293939471244812, "step": 176, "step_time": 169.23447634931654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0537109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.7333984375, "completions/mean_terminated_length": 74.88029479980469, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2512029868084937, "epoch": 0.0728995057660626, "frac_reward_zero_std": 0.0, "grad_norm": 0.5666512250900269, "kl": 0.30354723357595503, "learning_rate": 2.1728395061728397e-06, "loss": 0.0181, "num_tokens": 81605171.0, "reward": 1.9841545820236206, "reward_std": 0.3431292474269867, "rewards/reward_model/mean": 1.9841545820236206, "rewards/reward_model/std": 0.8817236423492432, "step": 177, "step_time": 168.3823292935267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07666015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.712890625, "completions/mean_terminated_length": 71.37175750732422, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.263297796715051, "epoch": 0.07331136738056013, "frac_reward_zero_std": 0.0, "grad_norm": 0.6307470798492432, "kl": 0.3325087181874551, "learning_rate": 2.1851851851851853e-06, "loss": 0.0256, "num_tokens": 82068551.0, "reward": 1.8173794746398926, "reward_std": 0.3982235789299011, "rewards/reward_model/mean": 1.8173794746398926, "rewards/reward_model/std": 1.106950283050537, "step": 178, "step_time": 167.39255077391863 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06103515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.169921875, "completions/mean_terminated_length": 75.99583435058594, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.271299744490534, "epoch": 0.07372322899505766, "frac_reward_zero_std": 0.0, "grad_norm": 0.6578369736671448, "kl": 0.33446174120763317, "learning_rate": 2.197530864197531e-06, "loss": 0.0284, "num_tokens": 82517795.0, "reward": 1.7375619411468506, "reward_std": 0.3846127390861511, "rewards/reward_model/mean": 1.7375619411468506, "rewards/reward_model/std": 0.8862913250923157, "step": 179, "step_time": 168.1992179742083 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0419921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.02197265625, "completions/mean_terminated_length": 77.91896057128906, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 1.2483481515664607, "epoch": 0.07413509060955518, "frac_reward_zero_std": 0.0, "grad_norm": 0.5584097504615784, "kl": 0.30480305349919945, "learning_rate": 2.2098765432098765e-06, "loss": 0.0213, "num_tokens": 82938224.0, "reward": 1.9653817415237427, "reward_std": 0.37414827942848206, "rewards/reward_model/mean": 1.9653817415237427, "rewards/reward_model/std": 0.8811721205711365, "step": 180, "step_time": 167.49799311021343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02587890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.77294921875, "completions/mean_terminated_length": 73.35889434814453, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2098995437845588, "epoch": 0.07454695222405272, "frac_reward_zero_std": 0.0, "grad_norm": 0.7208864688873291, "kl": 0.3534813959267922, "learning_rate": 2.222222222222222e-06, "loss": 0.0354, "num_tokens": 83381023.0, "reward": 2.026184320449829, "reward_std": 0.3664648234844208, "rewards/reward_model/mean": 2.026184320449829, "rewards/reward_model/std": 0.780437707901001, "step": 181, "step_time": 168.94889666279778 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02685546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.79931640625, "completions/mean_terminated_length": 73.33116149902344, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.2554190349765122, "epoch": 0.07495881383855024, "frac_reward_zero_std": 0.0, "grad_norm": 0.6133463382720947, "kl": 0.3488931337487884, "learning_rate": 2.234567901234568e-06, "loss": -0.0, "num_tokens": 83815236.0, "reward": 2.015033006668091, "reward_std": 0.35350939631462097, "rewards/reward_model/mean": 2.015033006668091, "rewards/reward_model/std": 1.0514765977859497, "step": 182, "step_time": 168.09779750416055 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0869140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.35546875, "completions/mean_terminated_length": 74.72513580322266, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.199432454770431, "epoch": 0.07537067545304778, "frac_reward_zero_std": 0.0, "grad_norm": 0.6157463192939758, "kl": 0.3277318494510837, "learning_rate": 2.2469135802469137e-06, "loss": 0.0357, "num_tokens": 84297404.0, "reward": 1.8543448448181152, "reward_std": 0.391369104385376, "rewards/reward_model/mean": 1.8543448448181152, "rewards/reward_model/std": 0.9162046909332275, "step": 183, "step_time": 167.68394365813583 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0869140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.412109375, "completions/mean_terminated_length": 80.26309967041016, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2098595479037613, "epoch": 0.0757825370675453, "frac_reward_zero_std": 0.0, "grad_norm": 0.5533583760261536, "kl": 0.31157187622739, "learning_rate": 2.2592592592592592e-06, "loss": 0.0297, "num_tokens": 84797320.0, "reward": 1.829999327659607, "reward_std": 0.39016926288604736, "rewards/reward_model/mean": 1.829999327659607, "rewards/reward_model/std": 0.919818103313446, "step": 184, "step_time": 169.95830805273727 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08642578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.021484375, "completions/mean_terminated_length": 72.1988296508789, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.216996781527996, "epoch": 0.07619439868204284, "frac_reward_zero_std": 0.0, "grad_norm": 0.5928105115890503, "kl": 0.3597461999161169, "learning_rate": 2.271604938271605e-06, "loss": 0.0241, "num_tokens": 85345332.0, "reward": 1.7781141996383667, "reward_std": 0.40030625462532043, "rewards/reward_model/mean": 1.7781143188476562, "rewards/reward_model/std": 1.0846326351165771, "step": 185, "step_time": 168.48916833195835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.26416015625, "completions/mean_terminated_length": 80.54025268554688, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.1919847468379885, "epoch": 0.07660626029654036, "frac_reward_zero_std": 0.0, "grad_norm": 0.5492737889289856, "kl": 0.30091724882368, "learning_rate": 2.2839506172839504e-06, "loss": 0.0307, "num_tokens": 85790705.0, "reward": 1.9125442504882812, "reward_std": 0.45980480313301086, "rewards/reward_model/mean": 1.9125442504882812, "rewards/reward_model/std": 0.884122371673584, "step": 186, "step_time": 168.23294077534229 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.7451171875, "completions/mean_terminated_length": 75.3208999633789, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1483043180778623, "epoch": 0.07701812191103789, "frac_reward_zero_std": 0.0, "grad_norm": 0.5400401949882507, "kl": 0.31244106526719406, "learning_rate": 2.2962962962962964e-06, "loss": 0.0363, "num_tokens": 86241287.0, "reward": 1.9626367092132568, "reward_std": 0.37234199047088623, "rewards/reward_model/mean": 1.9626367092132568, "rewards/reward_model/std": 0.9630360007286072, "step": 187, "step_time": 168.14725243346766 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.87939453125, "completions/mean_terminated_length": 78.75338745117188, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.2079863131511956, "epoch": 0.07742998352553541, "frac_reward_zero_std": 0.0, "grad_norm": 0.582582414150238, "kl": 0.3190855036955327, "learning_rate": 2.308641975308642e-06, "loss": 0.0175, "num_tokens": 86728272.0, "reward": 1.9138296842575073, "reward_std": 0.3768790364265442, "rewards/reward_model/mean": 1.9138296842575073, "rewards/reward_model/std": 1.0191558599472046, "step": 188, "step_time": 168.40571281081066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11376953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.12548828125, "completions/mean_terminated_length": 79.62149047851562, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.169044690206647, "epoch": 0.07784184514003295, "frac_reward_zero_std": 0.0, "grad_norm": 0.5737549662590027, "kl": 0.31175463960971683, "learning_rate": 2.320987654320988e-06, "loss": 0.0287, "num_tokens": 87171473.0, "reward": 1.9407716989517212, "reward_std": 0.4355354905128479, "rewards/reward_model/mean": 1.9407716989517212, "rewards/reward_model/std": 1.0851298570632935, "step": 189, "step_time": 167.8595218854025 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.955078125, "completions/mean_terminated_length": 79.68505859375, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.1897474566940218, "epoch": 0.07825370675453047, "frac_reward_zero_std": 0.0, "grad_norm": 0.5536565184593201, "kl": 0.32399573898874223, "learning_rate": 2.3333333333333336e-06, "loss": 0.0254, "num_tokens": 87624149.0, "reward": 1.953005313873291, "reward_std": 0.39668840169906616, "rewards/reward_model/mean": 1.953005313873291, "rewards/reward_model/std": 0.9170160889625549, "step": 190, "step_time": 167.70779052050784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07958984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.31494140625, "completions/mean_terminated_length": 83.79681396484375, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1992020884063095, "epoch": 0.07866556836902801, "frac_reward_zero_std": 0.0, "grad_norm": 0.534305214881897, "kl": 0.2823739905725233, "learning_rate": 2.345679012345679e-06, "loss": 0.0207, "num_tokens": 88065530.0, "reward": 1.8692455291748047, "reward_std": 0.3706667721271515, "rewards/reward_model/mean": 1.8692455291748047, "rewards/reward_model/std": 0.7108120918273926, "step": 191, "step_time": 168.080767756328 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.57177734375, "completions/mean_terminated_length": 77.01155090332031, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2336446167901158, "epoch": 0.07907742998352553, "frac_reward_zero_std": 0.0, "grad_norm": 0.7091127634048462, "kl": 0.3430620247963816, "learning_rate": 2.3580246913580247e-06, "loss": 0.0272, "num_tokens": 88569293.0, "reward": 1.8590911626815796, "reward_std": 0.3997899293899536, "rewards/reward_model/mean": 1.8590911626815796, "rewards/reward_model/std": 1.0301216840744019, "step": 192, "step_time": 168.8303936952725 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05224609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.47802734375, "completions/mean_terminated_length": 74.69293975830078, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2118491139262915, "epoch": 0.07948929159802307, "frac_reward_zero_std": 0.0, "grad_norm": 0.6019130349159241, "kl": 0.35962342529091984, "learning_rate": 2.3703703703703703e-06, "loss": 0.0434, "num_tokens": 88987872.0, "reward": 2.0519325733184814, "reward_std": 0.383076012134552, "rewards/reward_model/mean": 2.0519325733184814, "rewards/reward_model/std": 0.8245954513549805, "step": 193, "step_time": 166.53060166956857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03369140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.05859375, "completions/mean_terminated_length": 73.21273040771484, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.186548251658678, "epoch": 0.07990115321252059, "frac_reward_zero_std": 0.0, "grad_norm": 0.7078003287315369, "kl": 0.3844323490629904, "learning_rate": 2.3827160493827163e-06, "loss": 0.0161, "num_tokens": 89383064.0, "reward": 2.240997076034546, "reward_std": 0.3178167939186096, "rewards/reward_model/mean": 2.240997076034546, "rewards/reward_model/std": 0.84669429063797, "step": 194, "step_time": 167.91490666009486 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0439453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.45703125, "completions/mean_terminated_length": 75.1338119506836, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1919686342589557, "epoch": 0.08031301482701812, "frac_reward_zero_std": 0.0, "grad_norm": 0.5621170401573181, "kl": 0.3380963269737549, "learning_rate": 2.395061728395062e-06, "loss": 0.0135, "num_tokens": 89828864.0, "reward": 1.9205503463745117, "reward_std": 0.37170130014419556, "rewards/reward_model/mean": 1.9205503463745117, "rewards/reward_model/std": 0.847759485244751, "step": 195, "step_time": 167.42564077628776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.94970703125, "completions/mean_terminated_length": 75.2721176147461, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.1974371783435345, "epoch": 0.08072487644151564, "frac_reward_zero_std": 0.0, "grad_norm": 0.6827316880226135, "kl": 0.3753180764615536, "learning_rate": 2.4074074074074075e-06, "loss": 0.0271, "num_tokens": 90260377.0, "reward": 2.031513214111328, "reward_std": 0.3799358606338501, "rewards/reward_model/mean": 2.031513214111328, "rewards/reward_model/std": 1.0462548732757568, "step": 196, "step_time": 166.99446870852262 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.01416015625, "completions/mean_terminated_length": 76.08504486083984, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2283128183335066, "epoch": 0.08113673805601318, "frac_reward_zero_std": 0.0, "grad_norm": 0.570841372013092, "kl": 0.3397949834470637, "learning_rate": 2.419753086419753e-06, "loss": 0.0326, "num_tokens": 90748310.0, "reward": 1.893890380859375, "reward_std": 0.3679511547088623, "rewards/reward_model/mean": 1.893890380859375, "rewards/reward_model/std": 0.8536400198936462, "step": 197, "step_time": 167.43677308317274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.32080078125, "completions/mean_terminated_length": 76.90056610107422, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.2077249235007912, "epoch": 0.0815485996705107, "frac_reward_zero_std": 0.0, "grad_norm": 0.5746736526489258, "kl": 0.3641646616742946, "learning_rate": 2.4320987654320987e-06, "loss": 0.029, "num_tokens": 91213543.0, "reward": 2.129655361175537, "reward_std": 0.3387081027030945, "rewards/reward_model/mean": 2.129655361175537, "rewards/reward_model/std": 0.9924630522727966, "step": 198, "step_time": 168.12879333738238 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08642578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.84912109375, "completions/mean_terminated_length": 76.38856506347656, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.106790901394561, "epoch": 0.08196046128500824, "frac_reward_zero_std": 0.0, "grad_norm": 0.6154218316078186, "kl": 0.3942809420404956, "learning_rate": 2.4444444444444442e-06, "loss": 0.0462, "num_tokens": 91686258.0, "reward": 2.0349197387695312, "reward_std": 0.38085871934890747, "rewards/reward_model/mean": 2.0349197387695312, "rewards/reward_model/std": 1.0781173706054688, "step": 199, "step_time": 166.76508231228217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.064453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.89501953125, "completions/mean_terminated_length": 78.71868896484375, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.1731834115926176, "epoch": 0.08237232289950576, "frac_reward_zero_std": 0.0, "grad_norm": 0.5466514825820923, "kl": 0.34953623195178807, "learning_rate": 2.4567901234567902e-06, "loss": 0.0346, "num_tokens": 92113083.0, "reward": 2.178508758544922, "reward_std": 0.3616424798965454, "rewards/reward_model/mean": 2.178508758544922, "rewards/reward_model/std": 0.908944845199585, "step": 200, "step_time": 168.23503723321483 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.12939453125, "completions/mean_terminated_length": 87.78885650634766, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1966260885819793, "epoch": 0.0827841845140033, "frac_reward_zero_std": 0.0, "grad_norm": 0.5196940898895264, "kl": 0.28262469364563003, "learning_rate": 2.469135802469136e-06, "loss": 0.0235, "num_tokens": 92564260.0, "reward": 1.8757667541503906, "reward_std": 0.4259279668331146, "rewards/reward_model/mean": 1.8757667541503906, "rewards/reward_model/std": 0.8404105305671692, "step": 201, "step_time": 172.01673823036253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.79638671875, "completions/mean_terminated_length": 75.15836334228516, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1660795253701508, "epoch": 0.08319604612850082, "frac_reward_zero_std": 0.0, "grad_norm": 0.7824254035949707, "kl": 0.4240683880052529, "learning_rate": 2.4814814814814814e-06, "loss": 0.0469, "num_tokens": 92977699.0, "reward": 2.2998664379119873, "reward_std": 0.33475711941719055, "rewards/reward_model/mean": 2.2998664379119873, "rewards/reward_model/std": 0.9457482099533081, "step": 202, "step_time": 167.70637829741463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.51171875, "completions/mean_terminated_length": 80.49919891357422, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1729191986378282, "epoch": 0.08360790774299835, "frac_reward_zero_std": 0.0, "grad_norm": 0.5323017239570618, "kl": 0.3277348914416507, "learning_rate": 2.493827160493827e-06, "loss": 0.0315, "num_tokens": 93425211.0, "reward": 1.9932093620300293, "reward_std": 0.3924778699874878, "rewards/reward_model/mean": 1.9932093620300293, "rewards/reward_model/std": 0.8923009037971497, "step": 203, "step_time": 168.36044597998261 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.099609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.85546875, "completions/mean_terminated_length": 83.41431427001953, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1917262603528798, "epoch": 0.08401976935749589, "frac_reward_zero_std": 0.0, "grad_norm": 0.5234535932540894, "kl": 0.29382882232312113, "learning_rate": 2.506172839506173e-06, "loss": 0.0349, "num_tokens": 93968435.0, "reward": 1.9223933219909668, "reward_std": 0.3960828185081482, "rewards/reward_model/mean": 1.9223933219909668, "rewards/reward_model/std": 0.9306033253669739, "step": 204, "step_time": 167.21074503008276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12646484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.11865234375, "completions/mean_terminated_length": 77.76579284667969, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1593736838549376, "epoch": 0.08443163097199341, "frac_reward_zero_std": 0.0, "grad_norm": 0.5419563055038452, "kl": 0.34318836219608784, "learning_rate": 2.5185185185185186e-06, "loss": 0.0449, "num_tokens": 94424550.0, "reward": 1.9915204048156738, "reward_std": 0.3865254521369934, "rewards/reward_model/mean": 1.9915204048156738, "rewards/reward_model/std": 1.0041513442993164, "step": 205, "step_time": 167.1998031665571 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.93603515625, "completions/mean_terminated_length": 78.43447875976562, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.1889956025406718, "epoch": 0.08484349258649095, "frac_reward_zero_std": 0.0, "grad_norm": 0.5670483112335205, "kl": 0.31639264163095504, "learning_rate": 2.5308641975308646e-06, "loss": 0.0344, "num_tokens": 94887075.0, "reward": 1.9125852584838867, "reward_std": 0.43281179666519165, "rewards/reward_model/mean": 1.9125852584838867, "rewards/reward_model/std": 1.021222710609436, "step": 206, "step_time": 167.30011925753206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.056640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.73291015625, "completions/mean_terminated_length": 78.95497131347656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2472544298507273, "epoch": 0.08525535420098847, "frac_reward_zero_std": 0.0, "grad_norm": 0.5876907706260681, "kl": 0.32494840066647157, "learning_rate": 2.54320987654321e-06, "loss": 0.0391, "num_tokens": 95410784.0, "reward": 1.8113906383514404, "reward_std": 0.39986446499824524, "rewards/reward_model/mean": 1.8113906383514404, "rewards/reward_model/std": 0.9175825119018555, "step": 207, "step_time": 167.5429368270561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.7998046875, "completions/mean_terminated_length": 76.95863342285156, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.2154350047931075, "epoch": 0.085667215815486, "frac_reward_zero_std": 0.0, "grad_norm": 0.5915288329124451, "kl": 0.35894436069065705, "learning_rate": 2.5555555555555557e-06, "loss": 0.0364, "num_tokens": 95812070.0, "reward": 2.111616611480713, "reward_std": 0.41880661249160767, "rewards/reward_model/mean": 2.111616611480713, "rewards/reward_model/std": 0.876736044883728, "step": 208, "step_time": 167.34378886129707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05126953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.0830078125, "completions/mean_terminated_length": 78.547607421875, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.225630244705826, "epoch": 0.08607907742998352, "frac_reward_zero_std": 0.0, "grad_norm": 0.5733460783958435, "kl": 0.32834174192976207, "learning_rate": 2.5679012345679013e-06, "loss": 0.0234, "num_tokens": 96282704.0, "reward": 1.9388561248779297, "reward_std": 0.36063337326049805, "rewards/reward_model/mean": 1.9388561248779297, "rewards/reward_model/std": 0.922111988067627, "step": 209, "step_time": 167.41245729569346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.45458984375, "completions/mean_terminated_length": 78.31989288330078, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 1.1803404518868774, "epoch": 0.08649093904448106, "frac_reward_zero_std": 0.0, "grad_norm": 0.6085997819900513, "kl": 0.33247560617746785, "learning_rate": 2.580246913580247e-06, "loss": 0.0154, "num_tokens": 96739539.0, "reward": 2.0350303649902344, "reward_std": 0.3522924482822418, "rewards/reward_model/mean": 2.0350303649902344, "rewards/reward_model/std": 0.8716661930084229, "step": 210, "step_time": 168.13221714179963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04150390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.4814453125, "completions/mean_terminated_length": 74.25064086914062, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1280691751744598, "epoch": 0.08690280065897858, "frac_reward_zero_std": 0.0, "grad_norm": 1.545676827430725, "kl": 0.41590086941141635, "learning_rate": 2.5925925925925925e-06, "loss": 0.0235, "num_tokens": 97194957.0, "reward": 2.0548627376556396, "reward_std": 0.3438347280025482, "rewards/reward_model/mean": 2.0548627376556396, "rewards/reward_model/std": 0.8671165704727173, "step": 211, "step_time": 167.26268136315048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.208984375, "completions/mean_terminated_length": 78.06326293945312, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 1.2129651608411223, "epoch": 0.08731466227347612, "frac_reward_zero_std": 0.0, "grad_norm": 0.601740300655365, "kl": 0.3178666002349928, "learning_rate": 2.6049382716049385e-06, "loss": 0.0127, "num_tokens": 97614169.0, "reward": 2.004199981689453, "reward_std": 0.41419386863708496, "rewards/reward_model/mean": 2.004199981689453, "rewards/reward_model/std": 0.7952732443809509, "step": 212, "step_time": 168.1062852027826 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.8017578125, "completions/mean_terminated_length": 75.30599975585938, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.1689555132761598, "epoch": 0.08772652388797364, "frac_reward_zero_std": 0.0, "grad_norm": 0.5558601021766663, "kl": 0.3709078943938948, "learning_rate": 2.617283950617284e-06, "loss": 0.0015, "num_tokens": 98118819.0, "reward": 2.0427756309509277, "reward_std": 0.36541327834129333, "rewards/reward_model/mean": 2.0427756309509277, "rewards/reward_model/std": 1.0435569286346436, "step": 213, "step_time": 167.87519831489772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.2958984375, "completions/mean_terminated_length": 78.44041442871094, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 1.182969830231741, "epoch": 0.08813838550247118, "frac_reward_zero_std": 0.0, "grad_norm": 0.5668360590934753, "kl": 0.3610283025773242, "learning_rate": 2.6296296296296297e-06, "loss": 0.0188, "num_tokens": 98580097.0, "reward": 1.9900280237197876, "reward_std": 0.3760533332824707, "rewards/reward_model/mean": 1.9900280237197876, "rewards/reward_model/std": 0.9317620396614075, "step": 214, "step_time": 167.24053863668814 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.77001953125, "completions/mean_terminated_length": 80.77215576171875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1422891535330564, "epoch": 0.0885502471169687, "frac_reward_zero_std": 0.0, "grad_norm": 0.5320437550544739, "kl": 0.3234372254810296, "learning_rate": 2.6419753086419752e-06, "loss": 0.0309, "num_tokens": 99010666.0, "reward": 2.0549404621124268, "reward_std": 0.3534768223762512, "rewards/reward_model/mean": 2.0549404621124268, "rewards/reward_model/std": 0.8009667992591858, "step": 215, "step_time": 168.64242574665695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.68798828125, "completions/mean_terminated_length": 80.00054168701172, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1598168034106493, "epoch": 0.08896210873146623, "frac_reward_zero_std": 0.0, "grad_norm": 0.538616955280304, "kl": 0.37402690842282027, "learning_rate": 2.654320987654321e-06, "loss": 0.0238, "num_tokens": 99513515.0, "reward": 2.0404818058013916, "reward_std": 0.3708900511264801, "rewards/reward_model/mean": 2.0404818058013916, "rewards/reward_model/std": 0.9694047570228577, "step": 216, "step_time": 166.2653929819353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1337890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.43212890625, "completions/mean_terminated_length": 74.23957061767578, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0760659761726856, "epoch": 0.08937397034596375, "frac_reward_zero_std": 0.0, "grad_norm": 0.5696491599082947, "kl": 0.40011614008108154, "learning_rate": 2.6666666666666664e-06, "loss": 0.0536, "num_tokens": 99924640.0, "reward": 2.265016555786133, "reward_std": 0.358880877494812, "rewards/reward_model/mean": 2.265016555786133, "rewards/reward_model/std": 0.9844642281532288, "step": 217, "step_time": 167.61541163828224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09619140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.93212890625, "completions/mean_terminated_length": 73.70988464355469, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1136725812684745, "epoch": 0.08978583196046129, "frac_reward_zero_std": 0.0, "grad_norm": 0.5942970514297485, "kl": 0.4367644570884295, "learning_rate": 2.6790123456790124e-06, "loss": 0.0488, "num_tokens": 100375509.0, "reward": 2.337205410003662, "reward_std": 0.32930994033813477, "rewards/reward_model/mean": 2.337205410003662, "rewards/reward_model/std": 1.0812623500823975, "step": 218, "step_time": 167.20669629890472 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10791015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.7216796875, "completions/mean_terminated_length": 81.728515625, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.2016150718554854, "epoch": 0.09019769357495881, "frac_reward_zero_std": 0.0, "grad_norm": 0.5580322742462158, "kl": 0.3770898795919493, "learning_rate": 2.691358024691358e-06, "loss": 0.0296, "num_tokens": 100807835.0, "reward": 1.9961779117584229, "reward_std": 0.40818697214126587, "rewards/reward_model/mean": 1.9961779117584229, "rewards/reward_model/std": 1.0633736848831177, "step": 219, "step_time": 166.09151289006695 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09912109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.98828125, "completions/mean_terminated_length": 82.47587585449219, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.176664667436853, "epoch": 0.09060955518945635, "frac_reward_zero_std": 0.0, "grad_norm": 0.59111487865448, "kl": 0.34581953659653664, "learning_rate": 2.703703703703704e-06, "loss": 0.0294, "num_tokens": 101313475.0, "reward": 1.971434235572815, "reward_std": 0.3868926763534546, "rewards/reward_model/mean": 1.971434235572815, "rewards/reward_model/std": 1.0751652717590332, "step": 220, "step_time": 167.64113991288468 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.662109375, "completions/mean_terminated_length": 85.03253173828125, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.2570797046646476, "epoch": 0.09102141680395387, "frac_reward_zero_std": 0.0, "grad_norm": 0.5188368558883667, "kl": 0.320189309946727, "learning_rate": 2.7160493827160496e-06, "loss": 0.0345, "num_tokens": 101836879.0, "reward": 1.9701588153839111, "reward_std": 0.35449203848838806, "rewards/reward_model/mean": 1.9701588153839111, "rewards/reward_model/std": 0.857776403427124, "step": 221, "step_time": 167.400555992499 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02490234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.978515625, "completions/mean_terminated_length": 79.77766418457031, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 1.2087533662561327, "epoch": 0.0914332784184514, "frac_reward_zero_std": 0.0, "grad_norm": 0.5409842729568481, "kl": 0.3594117659959011, "learning_rate": 2.728395061728395e-06, "loss": 0.034, "num_tokens": 102308483.0, "reward": 1.9811162948608398, "reward_std": 0.3334912061691284, "rewards/reward_model/mean": 1.9811162948608398, "rewards/reward_model/std": 1.0253175497055054, "step": 222, "step_time": 167.75764168938622 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.00537109375, "completions/mean_terminated_length": 80.00572967529297, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1627340822014958, "epoch": 0.09184514003294893, "frac_reward_zero_std": 0.0, "grad_norm": 0.6176626086235046, "kl": 0.3442263937322423, "learning_rate": 2.7407407407407407e-06, "loss": 0.0269, "num_tokens": 102745390.0, "reward": 2.0517289638519287, "reward_std": 0.3462464511394501, "rewards/reward_model/mean": 2.0517289638519287, "rewards/reward_model/std": 0.841692328453064, "step": 223, "step_time": 167.44764357246459 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0224609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.63623046875, "completions/mean_terminated_length": 76.47901916503906, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.2233956214040518, "epoch": 0.09225700164744646, "frac_reward_zero_std": 0.0, "grad_norm": 0.6192571520805359, "kl": 0.40509118349291384, "learning_rate": 2.7530864197530867e-06, "loss": 0.0267, "num_tokens": 103187077.0, "reward": 2.1590912342071533, "reward_std": 0.2999606728553772, "rewards/reward_model/mean": 2.1590912342071533, "rewards/reward_model/std": 0.8668639063835144, "step": 224, "step_time": 166.09988987259567 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0244140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.7412109375, "completions/mean_terminated_length": 74.43343353271484, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1649149996228516, "epoch": 0.09266886326194398, "frac_reward_zero_std": 0.0, "grad_norm": 0.6238582730293274, "kl": 0.4190884438576177, "learning_rate": 2.7654320987654323e-06, "loss": 0.0489, "num_tokens": 103666067.0, "reward": 2.0492851734161377, "reward_std": 0.33458924293518066, "rewards/reward_model/mean": 2.0492851734161377, "rewards/reward_model/std": 0.9674420356750488, "step": 225, "step_time": 167.92315621161833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04638671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.6025390625, "completions/mean_terminated_length": 82.4915542602539, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.235764770070091, "epoch": 0.09308072487644152, "frac_reward_zero_std": 0.0, "grad_norm": 0.5285966992378235, "kl": 0.3762934444239363, "learning_rate": 2.777777777777778e-06, "loss": 0.034, "num_tokens": 104144741.0, "reward": 2.1850292682647705, "reward_std": 0.3341595530509949, "rewards/reward_model/mean": 2.1850292682647705, "rewards/reward_model/std": 1.0103217363357544, "step": 226, "step_time": 167.12626477750018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03466796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.47119140625, "completions/mean_terminated_length": 80.83611297607422, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.1625184018630534, "epoch": 0.09349258649093904, "frac_reward_zero_std": 0.0, "grad_norm": 0.5667609572410583, "kl": 0.40189889882458374, "learning_rate": 2.7901234567901235e-06, "loss": 0.0368, "num_tokens": 104569002.0, "reward": 2.187929391860962, "reward_std": 0.31522464752197266, "rewards/reward_model/mean": 2.187929391860962, "rewards/reward_model/std": 0.8487621545791626, "step": 227, "step_time": 166.7178283939138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04443359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.81640625, "completions/mean_terminated_length": 78.62238311767578, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1048777429386973, "epoch": 0.09390444810543658, "frac_reward_zero_std": 0.0, "grad_norm": 0.5987794995307922, "kl": 0.4451111367670819, "learning_rate": 2.802469135802469e-06, "loss": 0.0421, "num_tokens": 104999506.0, "reward": 2.2165136337280273, "reward_std": 0.34235715866088867, "rewards/reward_model/mean": 2.2165136337280273, "rewards/reward_model/std": 0.9973713755607605, "step": 228, "step_time": 168.3438419145532 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.2177734375, "completions/mean_terminated_length": 80.03242492675781, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.082367680966854, "epoch": 0.0943163097199341, "frac_reward_zero_std": 0.0, "grad_norm": 0.6990885734558105, "kl": 0.4529536569607444, "learning_rate": 2.8148148148148147e-06, "loss": 0.042, "num_tokens": 105464528.0, "reward": 2.198648452758789, "reward_std": 0.3203347623348236, "rewards/reward_model/mean": 2.198648452758789, "rewards/reward_model/std": 1.0034140348434448, "step": 229, "step_time": 167.59184094239026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.064453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.90869140625, "completions/mean_terminated_length": 78.7332992553711, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 1.088235519360751, "epoch": 0.09472817133443164, "frac_reward_zero_std": 0.0, "grad_norm": 0.5619035363197327, "kl": 0.44760972406947985, "learning_rate": 2.8271604938271607e-06, "loss": 0.0547, "num_tokens": 105912053.0, "reward": 2.3557634353637695, "reward_std": 0.32759857177734375, "rewards/reward_model/mean": 2.3557634353637695, "rewards/reward_model/std": 1.004636287689209, "step": 230, "step_time": 168.46039228746668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.51708984375, "completions/mean_terminated_length": 85.98880767822266, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.1501870406791568, "epoch": 0.09514003294892916, "frac_reward_zero_std": 0.0, "grad_norm": 0.5444980263710022, "kl": 0.3612911127274856, "learning_rate": 2.8395061728395062e-06, "loss": 0.0557, "num_tokens": 106414328.0, "reward": 2.0176854133605957, "reward_std": 0.3583607077598572, "rewards/reward_model/mean": 2.0176854133605957, "rewards/reward_model/std": 0.9017461538314819, "step": 231, "step_time": 167.52982168877497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09716796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.95361328125, "completions/mean_terminated_length": 84.75121307373047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1595689423847944, "epoch": 0.09555189456342669, "frac_reward_zero_std": 0.0, "grad_norm": 0.5628121495246887, "kl": 0.3798269954859279, "learning_rate": 2.851851851851852e-06, "loss": 0.0524, "num_tokens": 106905401.0, "reward": 2.108987808227539, "reward_std": 0.3625732362270355, "rewards/reward_model/mean": 2.108987808227539, "rewards/reward_model/std": 0.7929613590240479, "step": 232, "step_time": 167.03865774115548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.134765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.900390625, "completions/mean_terminated_length": 86.27765655517578, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.160016889916733, "epoch": 0.09596375617792421, "frac_reward_zero_std": 0.0, "grad_norm": 0.5226653814315796, "kl": 0.3490075214067474, "learning_rate": 2.8641975308641974e-06, "loss": 0.0431, "num_tokens": 107406349.0, "reward": 1.9204034805297852, "reward_std": 0.39888080954551697, "rewards/reward_model/mean": 1.9204034805297852, "rewards/reward_model/std": 0.9537256956100464, "step": 233, "step_time": 166.9014555583708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.1337890625, "completions/mean_terminated_length": 85.36582946777344, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1650826949626207, "epoch": 0.09637561779242175, "frac_reward_zero_std": 0.0, "grad_norm": 0.5264082551002502, "kl": 0.35566855262732133, "learning_rate": 2.876543209876543e-06, "loss": 0.0493, "num_tokens": 107868863.0, "reward": 2.024616241455078, "reward_std": 0.3620162904262543, "rewards/reward_model/mean": 2.024616241455078, "rewards/reward_model/std": 0.8900324702262878, "step": 234, "step_time": 167.88250127714127 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.380859375, "completions/mean_terminated_length": 83.30361938476562, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.205029504839331, "epoch": 0.09678747940691927, "frac_reward_zero_std": 0.0, "grad_norm": 0.6892660856246948, "kl": 0.38163267134223133, "learning_rate": 2.888888888888889e-06, "loss": 0.0227, "num_tokens": 108317035.0, "reward": 2.0172483921051025, "reward_std": 0.38704532384872437, "rewards/reward_model/mean": 2.0172483921051025, "rewards/reward_model/std": 0.891554594039917, "step": 235, "step_time": 167.89702927460894 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04150390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.03564453125, "completions/mean_terminated_length": 80.04534149169922, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.1944723164197057, "epoch": 0.09719934102141681, "frac_reward_zero_std": 0.0, "grad_norm": 2.6619279384613037, "kl": 0.518526881467551, "learning_rate": 2.9012345679012346e-06, "loss": 0.0396, "num_tokens": 108770132.0, "reward": 2.091853141784668, "reward_std": 0.33212810754776, "rewards/reward_model/mean": 2.091853141784668, "rewards/reward_model/std": 0.9218842387199402, "step": 236, "step_time": 167.77288166387007 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02587890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.46142578125, "completions/mean_terminated_length": 75.09223175048828, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1901515398640186, "epoch": 0.09761120263591433, "frac_reward_zero_std": 0.0, "grad_norm": 5.474021911621094, "kl": 0.35753101215232164, "learning_rate": 2.9135802469135806e-06, "loss": 0.018, "num_tokens": 109288453.0, "reward": 2.084144115447998, "reward_std": 0.32210126519203186, "rewards/reward_model/mean": 2.084144115447998, "rewards/reward_model/std": 0.9429020285606384, "step": 237, "step_time": 167.31716302549466 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02099609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.61767578125, "completions/mean_terminated_length": 75.51570892333984, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.2653391831554472, "epoch": 0.09802306425041186, "frac_reward_zero_std": 0.0, "grad_norm": 0.6623149514198303, "kl": 0.3646582751534879, "learning_rate": 2.925925925925926e-06, "loss": 0.0163, "num_tokens": 109699702.0, "reward": 2.0571460723876953, "reward_std": 0.35120493173599243, "rewards/reward_model/mean": 2.0571460723876953, "rewards/reward_model/std": 0.8974202871322632, "step": 238, "step_time": 168.29958866722882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.56982421875, "completions/mean_terminated_length": 73.50547790527344, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2013407927006483, "epoch": 0.09843492586490939, "frac_reward_zero_std": 0.0, "grad_norm": 0.7111081480979919, "kl": 0.39180356194265187, "learning_rate": 2.9382716049382717e-06, "loss": 0.0131, "num_tokens": 110214437.0, "reward": 2.0890862941741943, "reward_std": 0.33617299795150757, "rewards/reward_model/mean": 2.0890862941741943, "rewards/reward_model/std": 1.0048047304153442, "step": 239, "step_time": 167.48866623546928 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02880859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.50830078125, "completions/mean_terminated_length": 74.98089599609375, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1762982227373868, "epoch": 0.09884678747940692, "frac_reward_zero_std": 0.0, "grad_norm": 0.6593844890594482, "kl": 0.3651020047836937, "learning_rate": 2.9506172839506173e-06, "loss": 0.0101, "num_tokens": 110658518.0, "reward": 2.118110179901123, "reward_std": 0.3363790214061737, "rewards/reward_model/mean": 2.118110179901123, "rewards/reward_model/std": 0.8251715302467346, "step": 240, "step_time": 166.6864103130065 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0263671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.087890625, "completions/mean_terminated_length": 74.68204498291016, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.2127992340829223, "epoch": 0.09925864909390444, "frac_reward_zero_std": 0.0, "grad_norm": 0.868339478969574, "kl": 0.3826256996835582, "learning_rate": 2.962962962962963e-06, "loss": 0.0163, "num_tokens": 111124170.0, "reward": 2.044410228729248, "reward_std": 0.37149369716644287, "rewards/reward_model/mean": 2.044410228729248, "rewards/reward_model/std": 0.8633520007133484, "step": 241, "step_time": 166.3835293748416 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.048828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.9794921875, "completions/mean_terminated_length": 74.3603744506836, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1714741026517004, "epoch": 0.09967051070840198, "frac_reward_zero_std": 0.0, "grad_norm": 0.6249910593032837, "kl": 0.4107258401927538, "learning_rate": 2.975308641975309e-06, "loss": 0.0355, "num_tokens": 111572576.0, "reward": 1.9775002002716064, "reward_std": 0.3719671070575714, "rewards/reward_model/mean": 1.9775002002716064, "rewards/reward_model/std": 0.9445093870162964, "step": 242, "step_time": 166.6616400028579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11767578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.34130859375, "completions/mean_terminated_length": 76.25180053710938, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1609450306277722, "epoch": 0.1000823723228995, "frac_reward_zero_std": 0.0, "grad_norm": 0.5364700555801392, "kl": 0.414590252621565, "learning_rate": 2.9876543209876545e-06, "loss": 0.0226, "num_tokens": 112018651.0, "reward": 2.025747299194336, "reward_std": 0.37331199645996094, "rewards/reward_model/mean": 2.025747299194336, "rewards/reward_model/std": 0.928956925868988, "step": 243, "step_time": 166.3363236086443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.2705078125, "completions/mean_terminated_length": 80.09529876708984, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1745273335836828, "epoch": 0.10049423393739704, "frac_reward_zero_std": 0.0, "grad_norm": 0.5619068741798401, "kl": 0.3952896823757328, "learning_rate": 3e-06, "loss": 0.0459, "num_tokens": 112527877.0, "reward": 2.0009818077087402, "reward_std": 0.4256266951560974, "rewards/reward_model/mean": 2.0009818077087402, "rewards/reward_model/std": 1.0109403133392334, "step": 244, "step_time": 167.12481802189723 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.77880859375, "completions/mean_terminated_length": 75.62001037597656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.111538388300687, "epoch": 0.10090609555189456, "frac_reward_zero_std": 0.0, "grad_norm": 0.564782440662384, "kl": 0.4492376520065591, "learning_rate": 2.9999984495489793e-06, "loss": 0.0577, "num_tokens": 112995328.0, "reward": 2.193532705307007, "reward_std": 0.3593786060810089, "rewards/reward_model/mean": 2.193532705307007, "rewards/reward_model/std": 1.060076355934143, "step": 245, "step_time": 167.63375080144033 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.150390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.51171875, "completions/mean_terminated_length": 76.81378936767578, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.053921447834, "epoch": 0.1013179571663921, "frac_reward_zero_std": 0.0, "grad_norm": 6.696301460266113, "kl": 0.7351259617717005, "learning_rate": 2.999993798199121e-06, "loss": 0.0789, "num_tokens": 113463832.0, "reward": 2.124671459197998, "reward_std": 0.40309929847717285, "rewards/reward_model/mean": 2.124671459197998, "rewards/reward_model/std": 1.0214053392410278, "step": 246, "step_time": 166.34345931280404 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15283203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.97412109375, "completions/mean_terminated_length": 76.03170013427734, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.109812049428001, "epoch": 0.10172981878088962, "frac_reward_zero_std": 0.0, "grad_norm": 0.6046229004859924, "kl": 0.47344940464245155, "learning_rate": 2.9999860459600426e-06, "loss": 0.0572, "num_tokens": 113953699.0, "reward": 2.166167974472046, "reward_std": 0.3859254717826843, "rewards/reward_model/mean": 2.166167974472046, "rewards/reward_model/std": 0.978531002998352, "step": 247, "step_time": 167.3546497602947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14794921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.8193359375, "completions/mean_terminated_length": 79.66876220703125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.118449567584321, "epoch": 0.10214168039538715, "frac_reward_zero_std": 0.0, "grad_norm": 0.5672058463096619, "kl": 0.4395463067339733, "learning_rate": 2.9999751928477685e-06, "loss": 0.0567, "num_tokens": 114459089.0, "reward": 2.143787384033203, "reward_std": 0.3989766240119934, "rewards/reward_model/mean": 2.143787384033203, "rewards/reward_model/std": 1.0720252990722656, "step": 248, "step_time": 166.55423203017563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14306640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.94775390625, "completions/mean_terminated_length": 83.5948715209961, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1065816211048514, "epoch": 0.10255354200988467, "frac_reward_zero_std": 0.0, "grad_norm": 0.6922656297683716, "kl": 0.3973483861191198, "learning_rate": 2.999961238884735e-06, "loss": 0.0525, "num_tokens": 114932070.0, "reward": 2.3289365768432617, "reward_std": 0.36888283491134644, "rewards/reward_model/mean": 2.3289365768432617, "rewards/reward_model/std": 0.8522553443908691, "step": 249, "step_time": 166.28053965605795 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11767578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.57861328125, "completions/mean_terminated_length": 84.454345703125, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1614748241845518, "epoch": 0.10296540362438221, "frac_reward_zero_std": 0.0, "grad_norm": 0.5837370157241821, "kl": 0.35857224953360856, "learning_rate": 2.99994418409979e-06, "loss": 0.0374, "num_tokens": 115451719.0, "reward": 1.9058700799942017, "reward_std": 0.41814398765563965, "rewards/reward_model/mean": 1.9058700799942017, "rewards/reward_model/std": 0.8954830765724182, "step": 250, "step_time": 167.45548058999702 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.076171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.8818359375, "completions/mean_terminated_length": 79.16173553466797, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1528221073094755, "epoch": 0.10337726523887973, "frac_reward_zero_std": 0.0, "grad_norm": 0.526090681552887, "kl": 0.42182266688905656, "learning_rate": 2.999924028528189e-06, "loss": 0.0353, "num_tokens": 115904981.0, "reward": 2.217320442199707, "reward_std": 0.3572084307670593, "rewards/reward_model/mean": 2.217320442199707, "rewards/reward_model/std": 1.0017460584640503, "step": 251, "step_time": 166.7643887377344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05322265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.9921875, "completions/mean_terminated_length": 73.06859588623047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.174781663576141, "epoch": 0.10378912685337727, "frac_reward_zero_std": 0.0, "grad_norm": 0.6146409511566162, "kl": 0.4185088180238381, "learning_rate": 2.9999007722115995e-06, "loss": 0.0498, "num_tokens": 116310085.0, "reward": 2.0703132152557373, "reward_std": 0.3669997453689575, "rewards/reward_model/mean": 2.0703132152557373, "rewards/reward_model/std": 0.920939564704895, "step": 252, "step_time": 165.6844850042835 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.15380859375, "completions/mean_terminated_length": 73.70751190185547, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1874840632081032, "epoch": 0.10420098846787479, "frac_reward_zero_std": 0.0, "grad_norm": 0.5613296031951904, "kl": 0.40776463254587725, "learning_rate": 2.999874415198099e-06, "loss": 0.0217, "num_tokens": 116813600.0, "reward": 1.977123498916626, "reward_std": 0.3521531820297241, "rewards/reward_model/mean": 1.977123498916626, "rewards/reward_model/std": 1.1046346426010132, "step": 253, "step_time": 167.25275180488825 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03662109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.5439453125, "completions/mean_terminated_length": 76.6639633178711, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2282694249879569, "epoch": 0.10461285008237232, "frac_reward_zero_std": 0.0, "grad_norm": 0.5822910666465759, "kl": 0.36772315396228805, "learning_rate": 2.999844957542173e-06, "loss": 0.0244, "num_tokens": 117268602.0, "reward": 2.044971227645874, "reward_std": 0.3567642569541931, "rewards/reward_model/mean": 2.044971227645874, "rewards/reward_model/std": 0.8489285707473755, "step": 254, "step_time": 168.09533910546452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.26171875, "completions/mean_terminated_length": 79.76519775390625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2026057564653456, "epoch": 0.10502471169686985, "frac_reward_zero_std": 0.0, "grad_norm": 0.548676073551178, "kl": 0.388615106290672, "learning_rate": 2.99981239930472e-06, "loss": 0.0274, "num_tokens": 117749618.0, "reward": 2.0184969902038574, "reward_std": 0.3759860396385193, "rewards/reward_model/mean": 2.0184969902038574, "rewards/reward_model/std": 0.9887039065361023, "step": 255, "step_time": 166.4025080227293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05029296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.638671875, "completions/mean_terminated_length": 72.86581420898438, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2006305053364486, "epoch": 0.10543657331136738, "frac_reward_zero_std": 0.0, "grad_norm": 0.5897360444068909, "kl": 0.4212786242715083, "learning_rate": 2.9997767405530456e-06, "loss": 0.0378, "num_tokens": 118195214.0, "reward": 2.0807132720947266, "reward_std": 0.3527105450630188, "rewards/reward_model/mean": 2.0807132720947266, "rewards/reward_model/std": 0.9851901531219482, "step": 256, "step_time": 166.69172981195152 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0361328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 75.103515625, "completions/mean_terminated_length": 73.12056732177734, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1398502422962338, "epoch": 0.1058484349258649, "frac_reward_zero_std": 0.0, "grad_norm": 1.6856611967086792, "kl": 0.4659574901452288, "learning_rate": 2.999737981360867e-06, "loss": 0.0382, "num_tokens": 118674178.0, "reward": 2.0986170768737793, "reward_std": 0.3780984580516815, "rewards/reward_model/mean": 2.0986170768737793, "rewards/reward_model/std": 1.0228615999221802, "step": 257, "step_time": 168.47519267862663 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03662109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.48779296875, "completions/mean_terminated_length": 78.68170928955078, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1821080185472965, "epoch": 0.10626029654036244, "frac_reward_zero_std": 0.0, "grad_norm": 0.5905833840370178, "kl": 0.37560061120893806, "learning_rate": 2.9996961218083088e-06, "loss": 0.0163, "num_tokens": 119116617.0, "reward": 2.1928091049194336, "reward_std": 0.4030284285545349, "rewards/reward_model/mean": 2.1928091049194336, "rewards/reward_model/std": 0.8955987691879272, "step": 258, "step_time": 166.74717063084245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07861328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.70263671875, "completions/mean_terminated_length": 85.34976196289062, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 1.1852269642986357, "epoch": 0.10667215815485996, "frac_reward_zero_std": 0.0, "grad_norm": 0.5435870885848999, "kl": 0.33758779475465417, "learning_rate": 2.999651161981906e-06, "loss": 0.0383, "num_tokens": 119587304.0, "reward": 1.9972866773605347, "reward_std": 0.3740314841270447, "rewards/reward_model/mean": 1.9972866773605347, "rewards/reward_model/std": 0.8846590518951416, "step": 259, "step_time": 167.4595838827081 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.01513671875, "completions/mean_terminated_length": 77.03842163085938, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.103387565817684, "epoch": 0.1070840197693575, "frac_reward_zero_std": 0.0, "grad_norm": 0.6193649172782898, "kl": 0.4352249257499352, "learning_rate": 2.9996031019746035e-06, "loss": 0.0446, "num_tokens": 120056583.0, "reward": 2.2137465476989746, "reward_std": 0.3558496832847595, "rewards/reward_model/mean": 2.2137465476989746, "rewards/reward_model/std": 0.9672927856445312, "step": 260, "step_time": 166.36369473906234 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.75537109375, "completions/mean_terminated_length": 77.62287139892578, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0919575728476048, "epoch": 0.10749588138385502, "frac_reward_zero_std": 0.0, "grad_norm": 0.5283170342445374, "kl": 0.42035381792811677, "learning_rate": 2.9995519418857535e-06, "loss": 0.0368, "num_tokens": 120506578.0, "reward": 2.1689672470092773, "reward_std": 0.3427453637123108, "rewards/reward_model/mean": 2.1689672470092773, "rewards/reward_model/std": 0.9175665378570557, "step": 261, "step_time": 166.67317268624902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06689453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.7890625, "completions/mean_terminated_length": 78.47618865966797, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1333514552097768, "epoch": 0.10790774299835255, "frac_reward_zero_std": 0.0, "grad_norm": 0.6854379177093506, "kl": 0.4108184980577789, "learning_rate": 2.9994976818211173e-06, "loss": 0.0365, "num_tokens": 120998850.0, "reward": 2.0521130561828613, "reward_std": 0.39874687790870667, "rewards/reward_model/mean": 2.0521130561828613, "rewards/reward_model/std": 0.9544030427932739, "step": 262, "step_time": 167.96281497785822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04248046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.38232421875, "completions/mean_terminated_length": 79.31413269042969, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.079210513504222, "epoch": 0.10831960461285008, "frac_reward_zero_std": 0.0, "grad_norm": 0.5797022581100464, "kl": 0.40812746091978624, "learning_rate": 2.999440321892866e-06, "loss": 0.0389, "num_tokens": 121462225.0, "reward": 2.0903613567352295, "reward_std": 0.3557789921760559, "rewards/reward_model/mean": 2.0903613567352295, "rewards/reward_model/std": 0.8900156021118164, "step": 263, "step_time": 166.4291472081095 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.7666015625, "completions/mean_terminated_length": 79.11450958251953, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0689460325520486, "epoch": 0.10873146622734761, "frac_reward_zero_std": 0.0, "grad_norm": 0.566057026386261, "kl": 0.4403340080752969, "learning_rate": 2.9993798622195772e-06, "loss": 0.0412, "num_tokens": 121912019.0, "reward": 2.2504732608795166, "reward_std": 0.34994006156921387, "rewards/reward_model/mean": 2.2504732608795166, "rewards/reward_model/std": 1.0079203844070435, "step": 264, "step_time": 167.9702100143768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03271484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.81298828125, "completions/mean_terminated_length": 75.08177947998047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.076825467403978, "epoch": 0.10914332784184513, "frac_reward_zero_std": 0.0, "grad_norm": 0.8230749368667603, "kl": 0.48995525762438774, "learning_rate": 2.9993163029262376e-06, "loss": 0.0407, "num_tokens": 122376948.0, "reward": 2.248335361480713, "reward_std": 0.3329167366027832, "rewards/reward_model/mean": 2.248335361480713, "rewards/reward_model/std": 1.0766096115112305, "step": 265, "step_time": 166.21424208022654 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08056640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.712890625, "completions/mean_terminated_length": 77.65692901611328, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.0614131812471896, "epoch": 0.10955518945634267, "frac_reward_zero_std": 0.0, "grad_norm": 0.5787990689277649, "kl": 0.4573960801353678, "learning_rate": 2.999249644144241e-06, "loss": 0.0535, "num_tokens": 122841416.0, "reward": 2.229977607727051, "reward_std": 0.36523738503456116, "rewards/reward_model/mean": 2.229977607727051, "rewards/reward_model/std": 0.9276270866394043, "step": 266, "step_time": 165.4688211195171 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.095703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.6962890625, "completions/mean_terminated_length": 84.53671264648438, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1332264912780374, "epoch": 0.10996705107084019, "frac_reward_zero_std": 0.0, "grad_norm": 0.566692054271698, "kl": 0.39690410619368777, "learning_rate": 2.9991798860113893e-06, "loss": 0.0483, "num_tokens": 123355258.0, "reward": 2.195420503616333, "reward_std": 0.34465447068214417, "rewards/reward_model/mean": 2.195420503616333, "rewards/reward_model/std": 0.9330472350120544, "step": 267, "step_time": 167.1648590057157 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.47509765625, "completions/mean_terminated_length": 75.11835479736328, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.0139270399231464, "epoch": 0.11037891268533773, "frac_reward_zero_std": 0.0, "grad_norm": 0.5762166976928711, "kl": 0.5479846252710558, "learning_rate": 2.9991070286718906e-06, "loss": 0.0463, "num_tokens": 123832487.0, "reward": 2.425419807434082, "reward_std": 0.295541375875473, "rewards/reward_model/mean": 2.425419807434082, "rewards/reward_model/std": 1.1402605772018433, "step": 268, "step_time": 166.71831385185942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.89453125, "completions/mean_terminated_length": 76.79418182373047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0537686191964895, "epoch": 0.11079077429983525, "frac_reward_zero_std": 0.0, "grad_norm": 0.6760122179985046, "kl": 0.5037357417168096, "learning_rate": 2.9990310722763615e-06, "loss": 0.0483, "num_tokens": 124345967.0, "reward": 2.305006980895996, "reward_std": 0.31295880675315857, "rewards/reward_model/mean": 2.305006980895996, "rewards/reward_model/std": 0.9712084531784058, "step": 269, "step_time": 167.11943473527208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.376953125, "completions/mean_terminated_length": 75.20623016357422, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0368919281754643, "epoch": 0.11120263591433278, "frac_reward_zero_std": 0.0, "grad_norm": 0.5720655918121338, "kl": 0.5112295243307017, "learning_rate": 2.9989520169818234e-06, "loss": 0.0403, "num_tokens": 124788755.0, "reward": 2.3633298873901367, "reward_std": 0.28035101294517517, "rewards/reward_model/mean": 2.3633298873901367, "rewards/reward_model/std": 0.9284332394599915, "step": 270, "step_time": 166.87855655979365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06103515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.66796875, "completions/mean_terminated_length": 78.65626525878906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1737296553328633, "epoch": 0.11161449752883032, "frac_reward_zero_std": 0.0, "grad_norm": 0.5821009874343872, "kl": 0.4506463960278779, "learning_rate": 2.9988698629517054e-06, "loss": 0.0573, "num_tokens": 125241931.0, "reward": 2.2463903427124023, "reward_std": 0.32008451223373413, "rewards/reward_model/mean": 2.2463903427124023, "rewards/reward_model/std": 0.8837316632270813, "step": 271, "step_time": 167.66402456723154 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0458984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.60791015625, "completions/mean_terminated_length": 78.32804107666016, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.126048056408763, "epoch": 0.11202635914332784, "frac_reward_zero_std": 0.0, "grad_norm": 0.6286892890930176, "kl": 0.47393987770192325, "learning_rate": 2.9987846103558418e-06, "loss": 0.0624, "num_tokens": 125700616.0, "reward": 2.2652668952941895, "reward_std": 0.34473103284835815, "rewards/reward_model/mean": 2.2652668952941895, "rewards/reward_model/std": 0.996942400932312, "step": 272, "step_time": 167.9426286201924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.78662109375, "completions/mean_terminated_length": 83.08934783935547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1949954982846975, "epoch": 0.11243822075782538, "frac_reward_zero_std": 0.0, "grad_norm": 0.5814276933670044, "kl": 0.426611682982184, "learning_rate": 2.998696259370472e-06, "loss": 0.0313, "num_tokens": 126128595.0, "reward": 2.149965286254883, "reward_std": 0.34451329708099365, "rewards/reward_model/mean": 2.149965286254883, "rewards/reward_model/std": 0.9439492225646973, "step": 273, "step_time": 165.48718547355384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.52880859375, "completions/mean_terminated_length": 82.36750793457031, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 1.2326417269650847, "epoch": 0.1128500823723229, "frac_reward_zero_std": 0.0, "grad_norm": 0.651888906955719, "kl": 0.45543873618589714, "learning_rate": 2.998604810178242e-06, "loss": 0.0475, "num_tokens": 126610094.0, "reward": 2.243598699569702, "reward_std": 0.32785987854003906, "rewards/reward_model/mean": 2.243598699569702, "rewards/reward_model/std": 0.897570788860321, "step": 274, "step_time": 167.30028676893562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04248046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.392578125, "completions/mean_terminated_length": 81.41356658935547, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.1482101476285607, "epoch": 0.11326194398682043, "frac_reward_zero_std": 0.0, "grad_norm": 1.389794111251831, "kl": 0.4680080755497329, "learning_rate": 2.998510262968201e-06, "loss": 0.0465, "num_tokens": 127074002.0, "reward": 2.239393711090088, "reward_std": 0.33927756547927856, "rewards/reward_model/mean": 2.239393711090088, "rewards/reward_model/std": 0.8706660270690918, "step": 275, "step_time": 166.43037084210664 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.06005859375, "completions/mean_terminated_length": 86.99683380126953, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.1874423588160425, "epoch": 0.11367380560131796, "frac_reward_zero_std": 0.0, "grad_norm": 0.8038768768310547, "kl": 0.4064391045249067, "learning_rate": 2.998412617935804e-06, "loss": 0.0403, "num_tokens": 127563277.0, "reward": 2.2046656608581543, "reward_std": 0.3385607600212097, "rewards/reward_model/mean": 2.2046656608581543, "rewards/reward_model/std": 0.8827455639839172, "step": 276, "step_time": 167.62961031217128 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08349609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.60498046875, "completions/mean_terminated_length": 82.83377838134766, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1939080201555043, "epoch": 0.11408566721581549, "frac_reward_zero_std": 0.0, "grad_norm": 0.5087087154388428, "kl": 0.4431017763563432, "learning_rate": 2.9983118752829087e-06, "loss": 0.0431, "num_tokens": 128041700.0, "reward": 2.291957378387451, "reward_std": 0.33698388934135437, "rewards/reward_model/mean": 2.291957378387451, "rewards/reward_model/std": 1.0976485013961792, "step": 277, "step_time": 166.47902009263635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08740234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.5908203125, "completions/mean_terminated_length": 83.720703125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1856307329144329, "epoch": 0.11449752883031301, "frac_reward_zero_std": 0.0, "grad_norm": 0.5273477435112, "kl": 0.4514982245163992, "learning_rate": 2.998208035217778e-06, "loss": 0.0537, "num_tokens": 128530430.0, "reward": 2.2459394931793213, "reward_std": 0.3567132353782654, "rewards/reward_model/mean": 2.2459394931793213, "rewards/reward_model/std": 0.8966877460479736, "step": 278, "step_time": 166.6428160974756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0654296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.43603515625, "completions/mean_terminated_length": 77.1060562133789, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1816288942936808, "epoch": 0.11490939044481055, "frac_reward_zero_std": 0.0, "grad_norm": 0.5808203220367432, "kl": 0.5039720781496726, "learning_rate": 2.9981010979550773e-06, "loss": 0.0495, "num_tokens": 128993531.0, "reward": 2.115367889404297, "reward_std": 0.36884111166000366, "rewards/reward_model/mean": 2.115367889404297, "rewards/reward_model/std": 0.9640926122665405, "step": 279, "step_time": 166.85618913453072 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.947265625, "completions/mean_terminated_length": 80.1927490234375, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1779224942438304, "epoch": 0.11532125205930807, "frac_reward_zero_std": 0.0, "grad_norm": 0.5532060265541077, "kl": 0.47714378812815994, "learning_rate": 2.9979910637158735e-06, "loss": 0.052, "num_tokens": 129492847.0, "reward": 2.195665121078491, "reward_std": 0.34004709124565125, "rewards/reward_model/mean": 2.195665121078491, "rewards/reward_model/std": 1.0143131017684937, "step": 280, "step_time": 167.7095952462405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.97265625, "completions/mean_terminated_length": 81.74278259277344, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2372127475682646, "epoch": 0.11573311367380561, "frac_reward_zero_std": 0.0, "grad_norm": 1.3116786479949951, "kl": 0.5048430786118843, "learning_rate": 2.997877932727638e-06, "loss": 0.0641, "num_tokens": 129983095.0, "reward": 2.325448989868164, "reward_std": 0.3475065231323242, "rewards/reward_model/mean": 2.325448989868164, "rewards/reward_model/std": 0.9131367802619934, "step": 281, "step_time": 166.77392387297004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.4853515625, "completions/mean_terminated_length": 81.56266784667969, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2391732432879508, "epoch": 0.11614497528830313, "frac_reward_zero_std": 0.0, "grad_norm": 0.5712860226631165, "kl": 0.4339372126851231, "learning_rate": 2.997761705224242e-06, "loss": 0.0537, "num_tokens": 130549177.0, "reward": 2.1727700233459473, "reward_std": 0.3783659338951111, "rewards/reward_model/mean": 2.1727700233459473, "rewards/reward_model/std": 1.0410562753677368, "step": 282, "step_time": 168.2725958460942 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.46875, "completions/mean_terminated_length": 82.44351196289062, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2441872442141175, "epoch": 0.11655683690280066, "frac_reward_zero_std": 0.0, "grad_norm": 0.5652570128440857, "kl": 0.44935083464952186, "learning_rate": 2.9976423814459597e-06, "loss": 0.0488, "num_tokens": 130991641.0, "reward": 2.15010666847229, "reward_std": 0.37734052538871765, "rewards/reward_model/mean": 2.15010666847229, "rewards/reward_model/std": 0.9091908931732178, "step": 283, "step_time": 167.22230136068538 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0888671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.19775390625, "completions/mean_terminated_length": 79.92550659179688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2174382915254682, "epoch": 0.11696869851729819, "frac_reward_zero_std": 0.0, "grad_norm": 0.5168125033378601, "kl": 0.4481718390015885, "learning_rate": 2.9975199616394645e-06, "loss": 0.0547, "num_tokens": 131471470.0, "reward": 2.2211289405822754, "reward_std": 0.37207257747650146, "rewards/reward_model/mean": 2.2211289405822754, "rewards/reward_model/std": 0.9916544556617737, "step": 284, "step_time": 166.56063405843452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.60302734375, "completions/mean_terminated_length": 77.83513641357422, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.233087460277602, "epoch": 0.11738056013179572, "frac_reward_zero_std": 0.0, "grad_norm": 0.5638722777366638, "kl": 0.43769514810992405, "learning_rate": 2.9973944460578314e-06, "loss": 0.0468, "num_tokens": 131946625.0, "reward": 2.0859298706054688, "reward_std": 0.343211829662323, "rewards/reward_model/mean": 2.0859298706054688, "rewards/reward_model/std": 0.9097768068313599, "step": 285, "step_time": 165.99169771838933 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.07861328125, "completions/mean_terminated_length": 84.50908660888672, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.2405860875733197, "epoch": 0.11779242174629324, "frac_reward_zero_std": 0.0, "grad_norm": 0.5505926012992859, "kl": 0.3984885466634296, "learning_rate": 2.997265834960535e-06, "loss": 0.0487, "num_tokens": 132390946.0, "reward": 2.1617085933685303, "reward_std": 0.360080361366272, "rewards/reward_model/mean": 2.1617085933685303, "rewards/reward_model/std": 0.9202823042869568, "step": 286, "step_time": 166.77291592443362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06494140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.9892578125, "completions/mean_terminated_length": 75.58538055419922, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.1980928278062493, "epoch": 0.11820428336079078, "frac_reward_zero_std": 0.0, "grad_norm": 0.6096040606498718, "kl": 0.5170614552916959, "learning_rate": 2.9971341286134486e-06, "loss": 0.0396, "num_tokens": 132828428.0, "reward": 2.378830909729004, "reward_std": 0.33530116081237793, "rewards/reward_model/mean": 2.378830909729004, "rewards/reward_model/std": 1.0353243350982666, "step": 287, "step_time": 166.52288179518655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.29443359375, "completions/mean_terminated_length": 80.56114196777344, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2639109750743955, "epoch": 0.1186161449752883, "frac_reward_zero_std": 0.0, "grad_norm": 0.5939512252807617, "kl": 0.4585163014708087, "learning_rate": 2.996999327288844e-06, "loss": 0.0354, "num_tokens": 133285703.0, "reward": 2.1381678581237793, "reward_std": 0.36385589838027954, "rewards/reward_model/mean": 2.1381678581237793, "rewards/reward_model/std": 0.9374520778656006, "step": 288, "step_time": 166.6009632977657 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07275390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.96484375, "completions/mean_terminated_length": 79.43128204345703, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1612094941083342, "epoch": 0.11902800658978584, "frac_reward_zero_std": 0.0, "grad_norm": 0.5733307003974915, "kl": 0.48431827931199223, "learning_rate": 2.9968614312653924e-06, "loss": 0.0436, "num_tokens": 133731007.0, "reward": 2.301260471343994, "reward_std": 0.33408987522125244, "rewards/reward_model/mean": 2.301260471343994, "rewards/reward_model/std": 0.9408568143844604, "step": 289, "step_time": 166.66861878242344 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 78.62744140625, "completions/mean_terminated_length": 75.93254852294922, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.1165203212294728, "epoch": 0.11943986820428336, "frac_reward_zero_std": 0.0, "grad_norm": 0.5708264112472534, "kl": 0.5367161537287757, "learning_rate": 2.996720440828162e-06, "loss": 0.0443, "num_tokens": 134168484.0, "reward": 2.335646152496338, "reward_std": 0.33389145135879517, "rewards/reward_model/mean": 2.335646152496338, "rewards/reward_model/std": 1.0379271507263184, "step": 290, "step_time": 166.27415292756632 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.626953125, "completions/mean_terminated_length": 79.70270538330078, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1134089101105928, "epoch": 0.1198517298187809, "frac_reward_zero_std": 0.0, "grad_norm": 0.601580023765564, "kl": 0.4884371829684824, "learning_rate": 2.996576356268617e-06, "loss": 0.0377, "num_tokens": 134646952.0, "reward": 2.277851104736328, "reward_std": 0.3502635061740875, "rewards/reward_model/mean": 2.277851104736328, "rewards/reward_model/std": 1.016486644744873, "step": 291, "step_time": 166.27507057692856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06494140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.673828125, "completions/mean_terminated_length": 79.52584838867188, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.1277775957714766, "epoch": 0.12026359143327842, "frac_reward_zero_std": 0.0, "grad_norm": 0.5489048361778259, "kl": 0.5019796285778284, "learning_rate": 2.9964291778846197e-06, "loss": 0.0364, "num_tokens": 135118828.0, "reward": 2.3384621143341064, "reward_std": 0.31429651379585266, "rewards/reward_model/mean": 2.3384621143341064, "rewards/reward_model/std": 1.041744351387024, "step": 292, "step_time": 166.99484749138355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.146484375, "completions/mean_terminated_length": 78.15272521972656, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1060659622307867, "epoch": 0.12067545304777595, "frac_reward_zero_std": 0.0, "grad_norm": 0.5651781558990479, "kl": 0.5125422339770012, "learning_rate": 2.9962789059804267e-06, "loss": 0.0363, "num_tokens": 135549976.0, "reward": 2.256876230239868, "reward_std": 0.3226175904273987, "rewards/reward_model/mean": 2.256876230239868, "rewards/reward_model/std": 0.8421558737754822, "step": 293, "step_time": 166.80462453560904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.064453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.7724609375, "completions/mean_terminated_length": 77.51879119873047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0749783692881465, "epoch": 0.12108731466227347, "frac_reward_zero_std": 0.0, "grad_norm": 0.6339244842529297, "kl": 0.5386025752522983, "learning_rate": 2.9961255408666904e-06, "loss": 0.0372, "num_tokens": 136020006.0, "reward": 2.2860939502716064, "reward_std": 0.2931899130344391, "rewards/reward_model/mean": 2.2860939502716064, "rewards/reward_model/std": 0.916381299495697, "step": 294, "step_time": 166.9460484511219 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09912109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.322265625, "completions/mean_terminated_length": 78.40650177001953, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0670697160530835, "epoch": 0.12149917627677101, "frac_reward_zero_std": 0.0, "grad_norm": 0.6845069527626038, "kl": 0.5651297419681214, "learning_rate": 2.9959690828604576e-06, "loss": 0.0495, "num_tokens": 136494682.0, "reward": 2.266152858734131, "reward_std": 0.3633003830909729, "rewards/reward_model/mean": 2.266152858734131, "rewards/reward_model/std": 0.9877234697341919, "step": 295, "step_time": 167.413200117182 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.23583984375, "completions/mean_terminated_length": 80.70237731933594, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.0983618032187223, "epoch": 0.12191103789126853, "frac_reward_zero_std": 0.0, "grad_norm": 0.5269924402236938, "kl": 0.5204845000989735, "learning_rate": 2.995809532285169e-06, "loss": 0.0473, "num_tokens": 136925917.0, "reward": 2.3026092052459717, "reward_std": 0.28458866477012634, "rewards/reward_model/mean": 2.3026092052459717, "rewards/reward_model/std": 0.8754356503486633, "step": 296, "step_time": 182.3526389640756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07861328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.72607421875, "completions/mean_terminated_length": 83.20455932617188, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.1232515673618764, "epoch": 0.12232289950576607, "frac_reward_zero_std": 0.0, "grad_norm": 0.5377551913261414, "kl": 0.5087125994032249, "learning_rate": 2.995646889470659e-06, "loss": 0.055, "num_tokens": 137430476.0, "reward": 2.4165916442871094, "reward_std": 0.3247578740119934, "rewards/reward_model/mean": 2.4165916442871094, "rewards/reward_model/std": 0.9104467034339905, "step": 297, "step_time": 183.21853946289048 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.44873046875, "completions/mean_terminated_length": 85.7113037109375, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.0935279242694378, "epoch": 0.12273476112026359, "frac_reward_zero_std": 0.0, "grad_norm": 0.6794559955596924, "kl": 0.5013321961159818, "learning_rate": 2.995481154753152e-06, "loss": 0.0418, "num_tokens": 137906947.0, "reward": 2.265456199645996, "reward_std": 0.35464298725128174, "rewards/reward_model/mean": 2.265456199645996, "rewards/reward_model/std": 0.906952440738678, "step": 298, "step_time": 181.887544546742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.84619140625, "completions/mean_terminated_length": 85.1078872680664, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.1434353357180953, "epoch": 0.12314662273476112, "frac_reward_zero_std": 0.0, "grad_norm": 0.5842140913009644, "kl": 0.4751878093229607, "learning_rate": 2.9953123284752683e-06, "loss": 0.049, "num_tokens": 138425736.0, "reward": 2.0713725090026855, "reward_std": 0.3423648476600647, "rewards/reward_model/mean": 2.0713725090026855, "rewards/reward_model/std": 0.9226396083831787, "step": 299, "step_time": 181.3867863304913 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.32470703125, "completions/mean_terminated_length": 84.77925109863281, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1648255866020918, "epoch": 0.12355848434925865, "frac_reward_zero_std": 0.0, "grad_norm": 0.53875333070755, "kl": 0.4612657963298261, "learning_rate": 2.9951404109860156e-06, "loss": 0.0535, "num_tokens": 138931809.0, "reward": 2.0180983543395996, "reward_std": 0.35618048906326294, "rewards/reward_model/mean": 2.0180983543395996, "rewards/reward_model/std": 0.8992364406585693, "step": 300, "step_time": 181.7613399666734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06298828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.86962890625, "completions/mean_terminated_length": 89.44084930419922, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.189904412953183, "epoch": 0.12397034596375618, "frac_reward_zero_std": 0.0, "grad_norm": 0.5026298761367798, "kl": 0.4417647529626265, "learning_rate": 2.9949654026407946e-06, "loss": 0.0366, "num_tokens": 139473142.0, "reward": 2.2219271659851074, "reward_std": 0.34376394748687744, "rewards/reward_model/mean": 2.2219271659851074, "rewards/reward_model/std": 0.9350811243057251, "step": 301, "step_time": 183.3938912912272 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0439453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.34814453125, "completions/mean_terminated_length": 85.47957611083984, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1584395063109696, "epoch": 0.1243822075782537, "frac_reward_zero_std": 0.0, "grad_norm": 0.7864318490028381, "kl": 0.530522993998602, "learning_rate": 2.994787303801393e-06, "loss": 0.0469, "num_tokens": 139933759.0, "reward": 2.3316164016723633, "reward_std": 0.324718177318573, "rewards/reward_model/mean": 2.3316164016723633, "rewards/reward_model/std": 1.012445330619812, "step": 302, "step_time": 181.68429006449878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07763671875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.97412109375, "completions/mean_terminated_length": 84.6050796508789, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1655622238758951, "epoch": 0.12479406919275124, "frac_reward_zero_std": 0.0, "grad_norm": 0.5984921455383301, "kl": 0.4645943237701431, "learning_rate": 2.9946061148359902e-06, "loss": 0.0511, "num_tokens": 140444106.0, "reward": 2.0499777793884277, "reward_std": 0.33987563848495483, "rewards/reward_model/mean": 2.0499777793884277, "rewards/reward_model/std": 0.9054234623908997, "step": 303, "step_time": 181.738795788493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06689453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.7626953125, "completions/mean_terminated_length": 89.16483306884766, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1909240034874529, "epoch": 0.12520593080724876, "frac_reward_zero_std": 0.0, "grad_norm": 0.520868182182312, "kl": 0.4315382630447857, "learning_rate": 2.9944218361191514e-06, "loss": 0.0452, "num_tokens": 140964676.0, "reward": 2.159797191619873, "reward_std": 0.34087467193603516, "rewards/reward_model/mean": 2.159797191619873, "rewards/reward_model/std": 0.8813415169715881, "step": 304, "step_time": 181.3012389843352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.470703125, "completions/mean_terminated_length": 84.15876770019531, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1655684576835483, "epoch": 0.12561779242174628, "frac_reward_zero_std": 0.0, "grad_norm": 0.5305781364440918, "kl": 0.47395912604406476, "learning_rate": 2.994234468031831e-06, "loss": 0.0397, "num_tokens": 141445000.0, "reward": 2.1336677074432373, "reward_std": 0.35702717304229736, "rewards/reward_model/mean": 2.1336677074432373, "rewards/reward_model/std": 0.9681492447853088, "step": 305, "step_time": 181.32029206166044 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05419921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.7705078125, "completions/mean_terminated_length": 81.23593139648438, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 1.1812542567495257, "epoch": 0.12602965403624383, "frac_reward_zero_std": 0.0, "grad_norm": 0.5398581624031067, "kl": 0.5197404274949804, "learning_rate": 2.9940440109613682e-06, "loss": 0.0461, "num_tokens": 141982194.0, "reward": 2.2467362880706787, "reward_std": 0.3366679847240448, "rewards/reward_model/mean": 2.2467362880706787, "rewards/reward_model/std": 1.079930305480957, "step": 306, "step_time": 181.78619939787313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0224609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 76.7373046875, "completions/mean_terminated_length": 75.55944061279297, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1194014376960695, "epoch": 0.12644151565074135, "frac_reward_zero_std": 0.0, "grad_norm": 0.9022649526596069, "kl": 0.5636142309522256, "learning_rate": 2.993850465301489e-06, "loss": 0.0454, "num_tokens": 142446744.0, "reward": 2.420468330383301, "reward_std": 0.28051140904426575, "rewards/reward_model/mean": 2.420468330383301, "rewards/reward_model/std": 1.001362919807434, "step": 307, "step_time": 183.12546619866043 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0400390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.76953125, "completions/mean_terminated_length": 78.79959106445312, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.149734997190535, "epoch": 0.12685337726523888, "frac_reward_zero_std": 0.0, "grad_norm": 0.580263614654541, "kl": 0.5284143775934353, "learning_rate": 2.9936538314523047e-06, "loss": 0.0511, "num_tokens": 142891328.0, "reward": 2.2082672119140625, "reward_std": 0.3452140688896179, "rewards/reward_model/mean": 2.2082672119140625, "rewards/reward_model/std": 0.9466052651405334, "step": 308, "step_time": 183.6291290926747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.78564453125, "completions/mean_terminated_length": 76.67827606201172, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0867582249920815, "epoch": 0.1272652388797364, "frac_reward_zero_std": 0.0, "grad_norm": 0.5917499661445618, "kl": 0.5569419207749888, "learning_rate": 2.99345410982031e-06, "loss": 0.0565, "num_tokens": 143353193.0, "reward": 2.357027053833008, "reward_std": 0.3116685152053833, "rewards/reward_model/mean": 2.357027053833008, "rewards/reward_model/std": 0.8548756241798401, "step": 309, "step_time": 184.83612006809562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09130859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.60107421875, "completions/mean_terminated_length": 83.5416488647461, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1834682112094015, "epoch": 0.12767710049423395, "frac_reward_zero_std": 0.0, "grad_norm": 0.5495151281356812, "kl": 0.4802495735930279, "learning_rate": 2.9932513008183822e-06, "loss": 0.0426, "num_tokens": 143824760.0, "reward": 2.1622982025146484, "reward_std": 0.3434183597564697, "rewards/reward_model/mean": 2.1622982025146484, "rewards/reward_model/std": 0.9152319431304932, "step": 310, "step_time": 182.08543810481206 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10302734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.8291015625, "completions/mean_terminated_length": 89.90419006347656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1932893018238246, "epoch": 0.12808896210873147, "frac_reward_zero_std": 0.0, "grad_norm": 0.5348262190818787, "kl": 0.4367464517126791, "learning_rate": 2.993045404865783e-06, "loss": 0.0407, "num_tokens": 144341402.0, "reward": 2.1522250175476074, "reward_std": 0.3539927005767822, "rewards/reward_model/mean": 2.1522250175476074, "rewards/reward_model/std": 0.8729134202003479, "step": 311, "step_time": 180.6585853858851 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13623046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.048828125, "completions/mean_terminated_length": 87.53646850585938, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 1.2172390951309353, "epoch": 0.128500823723229, "frac_reward_zero_std": 0.0, "grad_norm": 0.585576593875885, "kl": 0.46445279818726704, "learning_rate": 2.9928364223881534e-06, "loss": 0.0471, "num_tokens": 144782110.0, "reward": 2.2030887603759766, "reward_std": 0.38694334030151367, "rewards/reward_model/mean": 2.2030887603759766, "rewards/reward_model/std": 0.8770624399185181, "step": 312, "step_time": 179.77869805693626 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.068359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.74609375, "completions/mean_terminated_length": 79.42557525634766, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.072917009005323, "epoch": 0.1289126853377265, "frac_reward_zero_std": 0.0, "grad_norm": 0.5560229420661926, "kl": 0.5523521939176135, "learning_rate": 2.9926243538175173e-06, "loss": 0.0491, "num_tokens": 145289686.0, "reward": 2.3315746784210205, "reward_std": 0.32644104957580566, "rewards/reward_model/mean": 2.3315746784210205, "rewards/reward_model/std": 0.9419375658035278, "step": 313, "step_time": 177.75018994743004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0693359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.765625, "completions/mean_terminated_length": 78.32109069824219, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.039536531548947, "epoch": 0.12932454695222406, "frac_reward_zero_std": 0.0, "grad_norm": 1.3224843740463257, "kl": 0.690456667682156, "learning_rate": 2.9924091995922767e-06, "loss": 0.0375, "num_tokens": 145740150.0, "reward": 2.4859046936035156, "reward_std": 0.2865492105484009, "rewards/reward_model/mean": 2.4859046936035156, "rewards/reward_model/std": 0.8175839781761169, "step": 314, "step_time": 179.0033772131428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.6826171875, "completions/mean_terminated_length": 81.74661254882812, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1195802071597427, "epoch": 0.12973640856672158, "frac_reward_zero_std": 0.0, "grad_norm": 0.5611775517463684, "kl": 0.5016106746625155, "learning_rate": 2.992190960157213e-06, "loss": 0.0393, "num_tokens": 146246412.0, "reward": 2.223116159439087, "reward_std": 0.3554210662841797, "rewards/reward_model/mean": 2.223116159439087, "rewards/reward_model/std": 0.9648101329803467, "step": 315, "step_time": 179.12733222683892 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.033203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 77.45703125, "completions/mean_terminated_length": 75.7212142944336, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1000276391860098, "epoch": 0.1301482701812191, "frac_reward_zero_std": 0.0, "grad_norm": 0.6189985871315002, "kl": 0.5708128549158573, "learning_rate": 2.991969635963486e-06, "loss": 0.0342, "num_tokens": 146679156.0, "reward": 2.3533027172088623, "reward_std": 0.2834378778934479, "rewards/reward_model/mean": 2.3533027172088623, "rewards/reward_model/std": 0.8896543979644775, "step": 316, "step_time": 177.36083007231355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0244140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 74.2421875, "completions/mean_terminated_length": 72.89689636230469, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1402649169322103, "epoch": 0.13056013179571663, "frac_reward_zero_std": 0.0, "grad_norm": 0.6072295308113098, "kl": 0.5484806462191045, "learning_rate": 2.9917452274686315e-06, "loss": 0.0256, "num_tokens": 147166372.0, "reward": 2.242672920227051, "reward_std": 0.2944999635219574, "rewards/reward_model/mean": 2.242672920227051, "rewards/reward_model/std": 0.8919768929481506, "step": 317, "step_time": 178.22633884241804 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.59326171875, "completions/mean_terminated_length": 76.89845275878906, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1299983037170023, "epoch": 0.13097199341021418, "frac_reward_zero_std": 0.0, "grad_norm": 0.5376353859901428, "kl": 0.5355923104798421, "learning_rate": 2.991517735136562e-06, "loss": 0.0471, "num_tokens": 147660515.0, "reward": 2.3142805099487305, "reward_std": 0.31208235025405884, "rewards/reward_model/mean": 2.3142805099487305, "rewards/reward_model/std": 0.9066339135169983, "step": 318, "step_time": 177.2680936283432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04833984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.39892578125, "completions/mean_terminated_length": 80.08261108398438, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1427940097637475, "epoch": 0.1313838550247117, "frac_reward_zero_std": 0.0, "grad_norm": 0.5557364225387573, "kl": 0.4978455606615171, "learning_rate": 2.991287159437567e-06, "loss": 0.0589, "num_tokens": 148117364.0, "reward": 2.2578847408294678, "reward_std": 0.3306541442871094, "rewards/reward_model/mean": 2.2578847408294678, "rewards/reward_model/std": 0.8999037742614746, "step": 319, "step_time": 178.03165327990428 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05419921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.8759765625, "completions/mean_terminated_length": 89.8058853149414, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2310384036973119, "epoch": 0.13179571663920922, "frac_reward_zero_std": 0.0, "grad_norm": 0.7157785296440125, "kl": 0.4380774620221928, "learning_rate": 2.991053500848305e-06, "loss": 0.0281, "num_tokens": 148620886.0, "reward": 2.1882987022399902, "reward_std": 0.34806662797927856, "rewards/reward_model/mean": 2.1882987022399902, "rewards/reward_model/std": 0.7541354894638062, "step": 320, "step_time": 179.27065010275692 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0439453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.58447265625, "completions/mean_terminated_length": 85.72676849365234, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1994026757311076, "epoch": 0.13220757825370674, "frac_reward_zero_std": 0.0, "grad_norm": 0.541190505027771, "kl": 0.4835659723030403, "learning_rate": 2.990816759851814e-06, "loss": 0.0427, "num_tokens": 149073475.0, "reward": 2.2712597846984863, "reward_std": 0.32761114835739136, "rewards/reward_model/mean": 2.2712597846984863, "rewards/reward_model/std": 0.8332777619361877, "step": 321, "step_time": 177.734118077904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06689453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.3681640625, "completions/mean_terminated_length": 84.45526123046875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1331434089224786, "epoch": 0.1326194398682043, "frac_reward_zero_std": 0.0, "grad_norm": 0.5302863121032715, "kl": 0.474698422010988, "learning_rate": 2.9905769369374994e-06, "loss": 0.0433, "num_tokens": 149504853.0, "reward": 2.2520110607147217, "reward_std": 0.3277668356895447, "rewards/reward_model/mean": 2.2520110607147217, "rewards/reward_model/std": 0.8627861142158508, "step": 322, "step_time": 177.7586707356386 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04443359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.87646484375, "completions/mean_terminated_length": 83.91773223876953, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.185831596609205, "epoch": 0.1330313014827018, "frac_reward_zero_std": 0.0, "grad_norm": 2.509178876876831, "kl": 0.7336386083625257, "learning_rate": 2.99033403260114e-06, "loss": 0.052, "num_tokens": 149985656.0, "reward": 2.2991042137145996, "reward_std": 0.31126657128334045, "rewards/reward_model/mean": 2.2991042137145996, "rewards/reward_model/std": 0.7674990892410278, "step": 323, "step_time": 179.94766734121367 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.67578125, "completions/mean_terminated_length": 80.96580505371094, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 1.1518713545519859, "epoch": 0.13344316309719934, "frac_reward_zero_std": 0.0, "grad_norm": 0.5733713507652283, "kl": 0.5438439772697166, "learning_rate": 2.9900880473448834e-06, "loss": 0.0442, "num_tokens": 150531360.0, "reward": 2.2603042125701904, "reward_std": 0.31569212675094604, "rewards/reward_model/mean": 2.2603042125701904, "rewards/reward_model/std": 0.8451188802719116, "step": 324, "step_time": 179.65418143803254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07080078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.57568359375, "completions/mean_terminated_length": 87.72412109375, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.2464524991810322, "epoch": 0.13385502471169686, "frac_reward_zero_std": 0.0, "grad_norm": 0.5254973769187927, "kl": 0.4957517021102831, "learning_rate": 2.989838981677248e-06, "loss": 0.0524, "num_tokens": 151026011.0, "reward": 2.387697219848633, "reward_std": 0.3317372798919678, "rewards/reward_model/mean": 2.387697219848633, "rewards/reward_model/std": 0.8221855163574219, "step": 325, "step_time": 179.72920090379193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.9228515625, "completions/mean_terminated_length": 82.21795654296875, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.1433638383168727, "epoch": 0.1342668863261944, "frac_reward_zero_std": 0.0, "grad_norm": 0.5484387278556824, "kl": 0.5532796500483528, "learning_rate": 2.9895868361131194e-06, "loss": 0.058, "num_tokens": 151418365.0, "reward": 2.415696859359741, "reward_std": 0.32123619318008423, "rewards/reward_model/mean": 2.415696859359741, "rewards/reward_model/std": 0.775295078754425, "step": 326, "step_time": 177.2113542361185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.853515625, "completions/mean_terminated_length": 87.39501190185547, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.2136494934093207, "epoch": 0.13467874794069193, "frac_reward_zero_std": 0.0, "grad_norm": 0.5504549741744995, "kl": 0.5316122133517638, "learning_rate": 2.9893316111737492e-06, "loss": 0.0408, "num_tokens": 151967857.0, "reward": 2.0931055545806885, "reward_std": 0.341963529586792, "rewards/reward_model/mean": 2.0931055545806885, "rewards/reward_model/std": 1.0068964958190918, "step": 327, "step_time": 177.80469922954217 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.88427734375, "completions/mean_terminated_length": 86.31998443603516, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.1571259163320065, "epoch": 0.13509060955518945, "frac_reward_zero_std": 0.0, "grad_norm": 0.5080457329750061, "kl": 0.5161278821760789, "learning_rate": 2.9890733073867562e-06, "loss": 0.05, "num_tokens": 152471780.0, "reward": 2.2795910835266113, "reward_std": 0.35899561643600464, "rewards/reward_model/mean": 2.2795910835266113, "rewards/reward_model/std": 0.8529795408248901, "step": 328, "step_time": 178.7946647698991 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.064453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.607421875, "completions/mean_terminated_length": 89.10021209716797, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 1.2153510942589492, "epoch": 0.13550247116968697, "frac_reward_zero_std": 0.0, "grad_norm": 0.5574885606765747, "kl": 0.48955635557649657, "learning_rate": 2.988811925286124e-06, "loss": 0.0427, "num_tokens": 152969856.0, "reward": 2.108138084411621, "reward_std": 0.34064537286758423, "rewards/reward_model/mean": 2.108138084411621, "rewards/reward_model/std": 0.9116151332855225, "step": 329, "step_time": 179.73849978437647 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.65380859375, "completions/mean_terminated_length": 91.96147918701172, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.2004315832164139, "epoch": 0.13591433278418452, "frac_reward_zero_std": 0.0, "grad_norm": 0.5106313228607178, "kl": 0.4482550826505758, "learning_rate": 2.9885474654121984e-06, "loss": 0.0391, "num_tokens": 153488539.0, "reward": 2.1396079063415527, "reward_std": 0.33650410175323486, "rewards/reward_model/mean": 2.1396079063415527, "rewards/reward_model/std": 0.897406280040741, "step": 330, "step_time": 179.78078530263156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.060546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.890625, "completions/mean_terminated_length": 87.43450927734375, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 1.1696031265892088, "epoch": 0.13632619439868204, "frac_reward_zero_std": 0.0, "grad_norm": 0.5139180421829224, "kl": 0.5040780130075291, "learning_rate": 2.9882799283116894e-06, "loss": 0.0516, "num_tokens": 153893531.0, "reward": 2.337380886077881, "reward_std": 0.3156912326812744, "rewards/reward_model/mean": 2.337380886077881, "rewards/reward_model/std": 0.7420399785041809, "step": 331, "step_time": 178.8128019226715 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0673828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.27392578125, "completions/mean_terminated_length": 91.83717346191406, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.2246722686104476, "epoch": 0.13673805601317957, "frac_reward_zero_std": 0.0, "grad_norm": 0.5157548785209656, "kl": 0.476093461853452, "learning_rate": 2.9880093145376683e-06, "loss": 0.0406, "num_tokens": 154350380.0, "reward": 2.331660509109497, "reward_std": 0.34169334173202515, "rewards/reward_model/mean": 2.331660509109497, "rewards/reward_model/std": 0.8471835851669312, "step": 332, "step_time": 179.60801557265222 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0732421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.560546875, "completions/mean_terminated_length": 83.28556060791016, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.104756724787876, "epoch": 0.1371499176276771, "frac_reward_zero_std": 0.0, "grad_norm": 0.5265265107154846, "kl": 0.5674954288406298, "learning_rate": 2.9877356246495657e-06, "loss": 0.0489, "num_tokens": 154819144.0, "reward": 2.5747122764587402, "reward_std": 0.31526708602905273, "rewards/reward_model/mean": 2.5747122764587402, "rewards/reward_model/std": 0.9426965117454529, "step": 333, "step_time": 179.73973829997703 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.037109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.19921875, "completions/mean_terminated_length": 84.5882339477539, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1406109977979213, "epoch": 0.13756177924217464, "frac_reward_zero_std": 0.0, "grad_norm": 0.5683643817901611, "kl": 0.5303917661076412, "learning_rate": 2.987458859213172e-06, "loss": 0.0478, "num_tokens": 155247776.0, "reward": 2.296814203262329, "reward_std": 0.3045124411582947, "rewards/reward_model/mean": 2.296814203262329, "rewards/reward_model/std": 0.8825284242630005, "step": 334, "step_time": 177.25651026749983 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.1630859375, "completions/mean_terminated_length": 85.94536590576172, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.091474344022572, "epoch": 0.13797364085667216, "frac_reward_zero_std": 0.0, "grad_norm": 0.5543970465660095, "kl": 0.5399871490662917, "learning_rate": 2.987179018800636e-06, "loss": 0.0572, "num_tokens": 155717582.0, "reward": 2.385404348373413, "reward_std": 0.3782488703727722, "rewards/reward_model/mean": 2.385404348373413, "rewards/reward_model/std": 0.959280788898468, "step": 335, "step_time": 179.73958875564858 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07861328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.634765625, "completions/mean_terminated_length": 88.53205871582031, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1590190418064594, "epoch": 0.13838550247116968, "frac_reward_zero_std": 0.0, "grad_norm": 0.5160994529724121, "kl": 0.5000830187927932, "learning_rate": 2.986896103990463e-06, "loss": 0.0494, "num_tokens": 156205762.0, "reward": 2.336515426635742, "reward_std": 0.3470662832260132, "rewards/reward_model/mean": 2.336515426635742, "rewards/reward_model/std": 0.9487416744232178, "step": 336, "step_time": 179.17238640319556 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.89013671875, "completions/mean_terminated_length": 83.52403259277344, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 1.0843422173056751, "epoch": 0.1387973640856672, "frac_reward_zero_std": 0.0, "grad_norm": 0.5337826609611511, "kl": 0.5413748703431338, "learning_rate": 2.9866101153675126e-06, "loss": 0.0407, "num_tokens": 156686721.0, "reward": 2.3108198642730713, "reward_std": 0.33316195011138916, "rewards/reward_model/mean": 2.3108198642730713, "rewards/reward_model/std": 0.9103125333786011, "step": 337, "step_time": 180.0918151801452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 79.5009765625, "completions/mean_terminated_length": 76.2137680053711, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0434326599352062, "epoch": 0.13920922570016475, "frac_reward_zero_std": 0.0, "grad_norm": 0.6374076008796692, "kl": 0.6402807212434709, "learning_rate": 2.986321053523001e-06, "loss": 0.0519, "num_tokens": 157152259.0, "reward": 2.4411561489105225, "reward_std": 0.3202677369117737, "rewards/reward_model/mean": 2.4411561489105225, "rewards/reward_model/std": 0.9377383589744568, "step": 338, "step_time": 179.4927121209912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06298828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.73486328125, "completions/mean_terminated_length": 89.29702758789062, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1900427804794163, "epoch": 0.13962108731466227, "frac_reward_zero_std": 0.0, "grad_norm": 0.5223175883293152, "kl": 0.4704223853768781, "learning_rate": 2.9860289190544963e-06, "loss": 0.0322, "num_tokens": 157674724.0, "reward": 2.083465099334717, "reward_std": 0.3120734989643097, "rewards/reward_model/mean": 2.083465099334717, "rewards/reward_model/std": 0.8356510400772095, "step": 339, "step_time": 178.68514743354172 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.31689453125, "completions/mean_terminated_length": 84.3088607788086, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.115970968035981, "epoch": 0.1400329489291598, "frac_reward_zero_std": 0.0, "grad_norm": 0.5332046747207642, "kl": 0.5464606848545372, "learning_rate": 2.9857337125659183e-06, "loss": 0.0352, "num_tokens": 158132397.0, "reward": 2.3391377925872803, "reward_std": 0.33263468742370605, "rewards/reward_model/mean": 2.3391377925872803, "rewards/reward_model/std": 0.9403417706489563, "step": 340, "step_time": 176.27032817946747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0537109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.861328125, "completions/mean_terminated_length": 84.52632141113281, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1279544085264206, "epoch": 0.14044481054365734, "frac_reward_zero_std": 0.0, "grad_norm": 0.5842037200927734, "kl": 0.5285600468050689, "learning_rate": 2.9854354346675384e-06, "loss": 0.0378, "num_tokens": 158629137.0, "reward": 2.3534529209136963, "reward_std": 0.3193182945251465, "rewards/reward_model/mean": 2.3534529209136963, "rewards/reward_model/std": 0.9178069233894348, "step": 341, "step_time": 178.7187624592334 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05810546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.6044921875, "completions/mean_terminated_length": 85.11249542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 1.0778781943954527, "epoch": 0.14085667215815487, "frac_reward_zero_std": 0.0, "grad_norm": 0.7371928691864014, "kl": 0.5926275659585372, "learning_rate": 2.9851340859759767e-06, "loss": 0.045, "num_tokens": 159087303.0, "reward": 2.485408306121826, "reward_std": 0.3070271611213684, "rewards/reward_model/mean": 2.485408306121826, "rewards/reward_model/std": 0.8073623180389404, "step": 342, "step_time": 177.88028611475602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06591796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.61962890625, "completions/mean_terminated_length": 86.9111328125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1835390287451446, "epoch": 0.1412685337726524, "frac_reward_zero_std": 0.0, "grad_norm": 0.5398038625717163, "kl": 0.4825705448165536, "learning_rate": 2.9848296671142016e-06, "loss": 0.0355, "num_tokens": 159571996.0, "reward": 2.22139835357666, "reward_std": 0.34988173842430115, "rewards/reward_model/mean": 2.22139835357666, "rewards/reward_model/std": 0.9210228323936462, "step": 343, "step_time": 180.209929341916 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10205078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.94091796875, "completions/mean_terminated_length": 96.75204467773438, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.2730968266259879, "epoch": 0.1416803953871499, "frac_reward_zero_std": 0.0, "grad_norm": 0.5108513236045837, "kl": 0.4061113327043131, "learning_rate": 2.984522178711529e-06, "loss": 0.0467, "num_tokens": 160035363.0, "reward": 2.0125880241394043, "reward_std": 0.40472501516342163, "rewards/reward_model/mean": 2.0125880241394043, "rewards/reward_model/std": 0.8265287280082703, "step": 344, "step_time": 177.52404638472944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.77978515625, "completions/mean_terminated_length": 86.8891830444336, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1638474613428116, "epoch": 0.14209225700164746, "frac_reward_zero_std": 0.0, "grad_norm": 0.4806898236274719, "kl": 0.5140834733610973, "learning_rate": 2.9842116214036195e-06, "loss": 0.0373, "num_tokens": 160511040.0, "reward": 2.2891035079956055, "reward_std": 0.3146075904369354, "rewards/reward_model/mean": 2.2891035079956055, "rewards/reward_model/std": 1.0442883968353271, "step": 345, "step_time": 177.42380552878603 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07373046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.87353515625, "completions/mean_terminated_length": 85.75909423828125, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.148594735423103, "epoch": 0.14250411861614498, "frac_reward_zero_std": 0.0, "grad_norm": 0.6340553164482117, "kl": 0.5861271159956232, "learning_rate": 2.9838979958324788e-06, "loss": 0.0438, "num_tokens": 160990013.0, "reward": 2.3169190883636475, "reward_std": 0.3422526717185974, "rewards/reward_model/mean": 2.3169190883636475, "rewards/reward_model/std": 0.9172117710113525, "step": 346, "step_time": 179.3430458502844 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.2685546875, "completions/mean_terminated_length": 83.93343353271484, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.160749185597524, "epoch": 0.1429159802306425, "frac_reward_zero_std": 0.0, "grad_norm": 0.5243058204650879, "kl": 0.5383644141256809, "learning_rate": 2.9835813026464537e-06, "loss": 0.0486, "num_tokens": 161462115.0, "reward": 2.3683254718780518, "reward_std": 0.3717167377471924, "rewards/reward_model/mean": 2.3683254718780518, "rewards/reward_model/std": 0.9474873542785645, "step": 347, "step_time": 178.22654771106318 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.58056640625, "completions/mean_terminated_length": 86.9781494140625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1957304971292615, "epoch": 0.14332784184514002, "frac_reward_zero_std": 0.0, "grad_norm": 0.5304787755012512, "kl": 0.49395692243706435, "learning_rate": 2.983261542500235e-06, "loss": 0.0359, "num_tokens": 161999816.0, "reward": 2.249234676361084, "reward_std": 0.3472321033477783, "rewards/reward_model/mean": 2.249234676361084, "rewards/reward_model/std": 0.9096790552139282, "step": 348, "step_time": 177.99551344383508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.037109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.31787109375, "completions/mean_terminated_length": 86.78853607177734, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.197805259609595, "epoch": 0.14373970345963757, "frac_reward_zero_std": 0.0, "grad_norm": 0.5113887190818787, "kl": 0.49214744940400124, "learning_rate": 2.9829387160548527e-06, "loss": 0.0368, "num_tokens": 162504339.0, "reward": 2.250112771987915, "reward_std": 0.33230942487716675, "rewards/reward_model/mean": 2.250112771987915, "rewards/reward_model/std": 0.9004961848258972, "step": 349, "step_time": 179.55271911527961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0498046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.7275390625, "completions/mean_terminated_length": 84.56423950195312, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.148556930013001, "epoch": 0.1441515650741351, "frac_reward_zero_std": 0.0, "grad_norm": 0.5116400122642517, "kl": 0.5200912128202617, "learning_rate": 2.982612823977675e-06, "loss": 0.037, "num_tokens": 162969797.0, "reward": 2.4369401931762695, "reward_std": 0.30880752205848694, "rewards/reward_model/mean": 2.4369401931762695, "rewards/reward_model/std": 0.8948498964309692, "step": 350, "step_time": 178.23102125991136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04931640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.15478515625, "completions/mean_terminated_length": 86.08782196044922, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1142600453458726, "epoch": 0.14456342668863262, "frac_reward_zero_std": 0.0, "grad_norm": 0.4922683835029602, "kl": 0.5358123979531229, "learning_rate": 2.9822838669424084e-06, "loss": 0.0329, "num_tokens": 163429794.0, "reward": 2.39753794670105, "reward_std": 0.30952227115631104, "rewards/reward_model/mean": 2.39753794670105, "rewards/reward_model/std": 0.9243928790092468, "step": 351, "step_time": 179.88818115461618 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07080078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.7568359375, "completions/mean_terminated_length": 82.53810119628906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.112268062774092, "epoch": 0.14497528830313014, "frac_reward_zero_std": 0.0, "grad_norm": 0.5688492655754089, "kl": 0.5405389621155336, "learning_rate": 2.9819518456290953e-06, "loss": 0.0515, "num_tokens": 163885840.0, "reward": 2.2547385692596436, "reward_std": 0.35960260033607483, "rewards/reward_model/mean": 2.2547385692596436, "rewards/reward_model/std": 0.8486311435699463, "step": 352, "step_time": 178.39603649405763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08251953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.28173828125, "completions/mean_terminated_length": 85.79936218261719, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1427024435251951, "epoch": 0.1453871499176277, "frac_reward_zero_std": 0.0, "grad_norm": 0.5504512190818787, "kl": 0.5753144897753373, "learning_rate": 2.9816167607241126e-06, "loss": 0.0508, "num_tokens": 164356817.0, "reward": 2.4262845516204834, "reward_std": 0.3433650732040405, "rewards/reward_model/mean": 2.4262845516204834, "rewards/reward_model/std": 0.8902410268783569, "step": 353, "step_time": 177.5910886824131 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0771484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 82.59765625, "completions/mean_terminated_length": 78.80211639404297, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1155586063396186, "epoch": 0.1457990115321252, "frac_reward_zero_std": 0.0, "grad_norm": 0.5517716407775879, "kl": 0.5654806508682668, "learning_rate": 2.981278612920171e-06, "loss": 0.0551, "num_tokens": 164834009.0, "reward": 2.2812585830688477, "reward_std": 0.3289134204387665, "rewards/reward_model/mean": 2.2812585830688477, "rewards/reward_model/std": 0.9483779668807983, "step": 354, "step_time": 178.1212548930198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0927734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.57861328125, "completions/mean_terminated_length": 91.16092681884766, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1689230327028781, "epoch": 0.14621087314662273, "frac_reward_zero_std": 0.0, "grad_norm": 0.5392249822616577, "kl": 0.48778879980091006, "learning_rate": 2.9809374029163123e-06, "loss": 0.0453, "num_tokens": 165352474.0, "reward": 2.209052562713623, "reward_std": 0.3627263307571411, "rewards/reward_model/mean": 2.209052562713623, "rewards/reward_model/std": 0.8795516490936279, "step": 355, "step_time": 178.82629200676456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0986328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.2587890625, "completions/mean_terminated_length": 90.5666275024414, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1824185741133988, "epoch": 0.14662273476112025, "frac_reward_zero_std": 0.0, "grad_norm": 0.5126945972442627, "kl": 0.5040855963015929, "learning_rate": 2.9805931314179096e-06, "loss": 0.0413, "num_tokens": 165847020.0, "reward": 2.251725673675537, "reward_std": 0.365298867225647, "rewards/reward_model/mean": 2.251725673675537, "rewards/reward_model/std": 0.9735676646232605, "step": 356, "step_time": 178.07318962551653 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0751953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.322265625, "completions/mean_terminated_length": 85.09609985351562, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 1.1268312393222004, "epoch": 0.1470345963756178, "frac_reward_zero_std": 0.0, "grad_norm": 0.599510669708252, "kl": 0.5479696299880743, "learning_rate": 2.9802457991366633e-06, "loss": 0.042, "num_tokens": 166308448.0, "reward": 2.2650880813598633, "reward_std": 0.3318103551864624, "rewards/reward_model/mean": 2.2650880813598633, "rewards/reward_model/std": 0.9420972466468811, "step": 357, "step_time": 177.16204105550423 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06591796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 83.40576171875, "completions/mean_terminated_length": 80.25875091552734, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.103062795009464, "epoch": 0.14744645799011533, "frac_reward_zero_std": 0.0, "grad_norm": 0.5064690113067627, "kl": 0.5855860275914893, "learning_rate": 2.9798954067906033e-06, "loss": 0.0443, "num_tokens": 166788191.0, "reward": 2.4571127891540527, "reward_std": 0.29924511909484863, "rewards/reward_model/mean": 2.4571127891540527, "rewards/reward_model/std": 0.8980612754821777, "step": 358, "step_time": 178.72926516924053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 80.7333984375, "completions/mean_terminated_length": 79.01113891601562, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0899577154777944, "epoch": 0.14785831960461285, "frac_reward_zero_std": 0.0, "grad_norm": 0.5409436821937561, "kl": 0.5860746707767248, "learning_rate": 2.9795419551040837e-06, "loss": 0.0446, "num_tokens": 167225693.0, "reward": 2.363490104675293, "reward_std": 0.29864785075187683, "rewards/reward_model/mean": 2.363490104675293, "rewards/reward_model/std": 0.8716721534729004, "step": 359, "step_time": 178.14612354105338 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02294921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.90283203125, "completions/mean_terminated_length": 84.91404724121094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1397253011818975, "epoch": 0.14827018121911037, "frac_reward_zero_std": 0.0, "grad_norm": 0.5434765815734863, "kl": 0.4996351352892816, "learning_rate": 2.9791854448077843e-06, "loss": 0.0381, "num_tokens": 167725750.0, "reward": 2.180931568145752, "reward_std": 0.32154789566993713, "rewards/reward_model/mean": 2.180931568145752, "rewards/reward_model/std": 0.8739709258079529, "step": 360, "step_time": 177.8121801377274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03955078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.17724609375, "completions/mean_terminated_length": 86.53736114501953, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1015683440491557, "epoch": 0.14868204283360792, "frac_reward_zero_std": 0.0, "grad_norm": 0.9227505922317505, "kl": 0.6062664500204846, "learning_rate": 2.9788258766387073e-06, "loss": 0.0424, "num_tokens": 168154241.0, "reward": 2.4231786727905273, "reward_std": 0.30548059940338135, "rewards/reward_model/mean": 2.4231786727905273, "rewards/reward_model/std": 0.8246005177497864, "step": 361, "step_time": 178.50632599601522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05322265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.0859375, "completions/mean_terminated_length": 86.89840698242188, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.1470173054840416, "epoch": 0.14909390444810544, "frac_reward_zero_std": 0.0, "grad_norm": 0.5660985708236694, "kl": 0.5425310720456764, "learning_rate": 2.9784632513401775e-06, "loss": 0.0374, "num_tokens": 168627409.0, "reward": 2.3287858963012695, "reward_std": 0.3059350848197937, "rewards/reward_model/mean": 2.3287858963012695, "rewards/reward_model/std": 0.8679665327072144, "step": 362, "step_time": 177.6265059551224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.59326171875, "completions/mean_terminated_length": 89.44725799560547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1611525951884687, "epoch": 0.14950576606260296, "frac_reward_zero_std": 0.0, "grad_norm": 0.5426596403121948, "kl": 0.5089211564045399, "learning_rate": 2.9780975696618367e-06, "loss": 0.0347, "num_tokens": 169080880.0, "reward": 2.072262763977051, "reward_std": 0.34514063596725464, "rewards/reward_model/mean": 2.072262763977051, "rewards/reward_model/std": 0.8410583734512329, "step": 363, "step_time": 177.04447367601097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.40478515625, "completions/mean_terminated_length": 90.21279907226562, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1758178700692952, "epoch": 0.14991762767710048, "frac_reward_zero_std": 0.0, "grad_norm": 0.524529755115509, "kl": 0.4698503875406459, "learning_rate": 2.9777288323596484e-06, "loss": 0.0192, "num_tokens": 169575149.0, "reward": 2.2351765632629395, "reward_std": 0.36411774158477783, "rewards/reward_model/mean": 2.2351765632629395, "rewards/reward_model/std": 1.0176758766174316, "step": 364, "step_time": 179.14810549514368 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.13720703125, "completions/mean_terminated_length": 87.56346893310547, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1091436152346432, "epoch": 0.15032948929159803, "frac_reward_zero_std": 0.0, "grad_norm": 0.5425158143043518, "kl": 0.556012088782154, "learning_rate": 2.977357040195891e-06, "loss": 0.0389, "num_tokens": 170077030.0, "reward": 2.468992233276367, "reward_std": 0.3592677116394043, "rewards/reward_model/mean": 2.468992233276367, "rewards/reward_model/std": 0.9123467206954956, "step": 365, "step_time": 177.56812796927989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07470703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.84716796875, "completions/mean_terminated_length": 91.08970642089844, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.2040003568399698, "epoch": 0.15074135090609556, "frac_reward_zero_std": 0.0, "grad_norm": 0.5132448077201843, "kl": 0.5246874364675023, "learning_rate": 2.976982193939158e-06, "loss": 0.0374, "num_tokens": 170532173.0, "reward": 2.2929229736328125, "reward_std": 0.3226976990699768, "rewards/reward_model/mean": 2.2929229736328125, "rewards/reward_model/std": 0.812001645565033, "step": 366, "step_time": 176.60000207601115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08740234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.33349609375, "completions/mean_terminated_length": 85.63027954101562, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.1354411500506103, "epoch": 0.15115321252059308, "frac_reward_zero_std": 0.0, "grad_norm": 0.5214349627494812, "kl": 0.5545413340441883, "learning_rate": 2.976604294364358e-06, "loss": 0.0473, "num_tokens": 170984536.0, "reward": 2.264357805252075, "reward_std": 0.34315481781959534, "rewards/reward_model/mean": 2.264357805252075, "rewards/reward_model/std": 0.927959680557251, "step": 367, "step_time": 177.56931630847976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.8876953125, "completions/mean_terminated_length": 89.42167663574219, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1453675739467144, "epoch": 0.1515650741350906, "frac_reward_zero_std": 0.0, "grad_norm": 0.49507108330726624, "kl": 0.523668595473282, "learning_rate": 2.97622334225271e-06, "loss": 0.0398, "num_tokens": 171481938.0, "reward": 2.256321430206299, "reward_std": 0.36627641320228577, "rewards/reward_model/mean": 2.256321430206299, "rewards/reward_model/std": 0.8239408731460571, "step": 368, "step_time": 177.0141043085605 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.58203125, "completions/mean_terminated_length": 92.64875793457031, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.1576440229546279, "epoch": 0.15197693574958815, "frac_reward_zero_std": 0.0, "grad_norm": 0.5009844303131104, "kl": 0.4823580157244578, "learning_rate": 2.975839338391745e-06, "loss": 0.0366, "num_tokens": 171945562.0, "reward": 2.346864700317383, "reward_std": 0.3248952627182007, "rewards/reward_model/mean": 2.346864700317383, "rewards/reward_model/std": 0.8989713788032532, "step": 369, "step_time": 177.23541718255728 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.10498046875, "completions/mean_terminated_length": 83.63546752929688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1101096351630986, "epoch": 0.15238879736408567, "frac_reward_zero_std": 0.0, "grad_norm": 0.5538142323493958, "kl": 0.5714477442088537, "learning_rate": 2.9754522835753005e-06, "loss": 0.0437, "num_tokens": 172426993.0, "reward": 2.3575704097747803, "reward_std": 0.3203357458114624, "rewards/reward_model/mean": 2.3575704097747803, "rewards/reward_model/std": 1.006217360496521, "step": 370, "step_time": 176.18894119095057 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05712890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.4365234375, "completions/mean_terminated_length": 91.3423080444336, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.131627653958276, "epoch": 0.1528006589785832, "frac_reward_zero_std": 0.0, "grad_norm": 0.5207405090332031, "kl": 0.4991882311878726, "learning_rate": 2.975062178603524e-06, "loss": 0.0394, "num_tokens": 172884783.0, "reward": 2.2432644367218018, "reward_std": 0.3401308059692383, "rewards/reward_model/mean": 2.2432644367218018, "rewards/reward_model/std": 0.9848742485046387, "step": 371, "step_time": 177.03785067796707 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0341796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.6083984375, "completions/mean_terminated_length": 86.17897033691406, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1297472438309342, "epoch": 0.15321252059308071, "frac_reward_zero_std": 0.0, "grad_norm": 0.5180069208145142, "kl": 0.5400325213558972, "learning_rate": 2.974669024282866e-06, "loss": 0.0376, "num_tokens": 173371757.0, "reward": 2.2407922744750977, "reward_std": 0.3197630047798157, "rewards/reward_model/mean": 2.2407922744750977, "rewards/reward_model/std": 0.8413589000701904, "step": 372, "step_time": 180.56042950646952 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06787109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.3662109375, "completions/mean_terminated_length": 84.40754699707031, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.0851728890556842, "epoch": 0.15362438220757826, "frac_reward_zero_std": 0.0, "grad_norm": 0.49752500653266907, "kl": 0.6113036054885015, "learning_rate": 2.974272821426083e-06, "loss": 0.0492, "num_tokens": 173857371.0, "reward": 2.4817733764648438, "reward_std": 0.3162037134170532, "rewards/reward_model/mean": 2.4817733764648438, "rewards/reward_model/std": 0.9889266490936279, "step": 373, "step_time": 178.22315939189866 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10009765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.990234375, "completions/mean_terminated_length": 94.6521987915039, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.189962183125317, "epoch": 0.15403624382207579, "frac_reward_zero_std": 0.0, "grad_norm": 0.5456772446632385, "kl": 0.47029789409134537, "learning_rate": 2.9738735708522313e-06, "loss": 0.0503, "num_tokens": 174372103.0, "reward": 2.0360536575317383, "reward_std": 0.38842496275901794, "rewards/reward_model/mean": 2.0360536575317383, "rewards/reward_model/std": 0.7656176686286926, "step": 374, "step_time": 178.9142449963838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07373046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.626953125, "completions/mean_terminated_length": 93.05007934570312, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1439739849884063, "epoch": 0.1544481054365733, "frac_reward_zero_std": 0.0, "grad_norm": 0.5723274350166321, "kl": 0.6108489506877959, "learning_rate": 2.9734712733866687e-06, "loss": 0.0336, "num_tokens": 174893803.0, "reward": 2.401510238647461, "reward_std": 0.29750198125839233, "rewards/reward_model/mean": 2.401510238647461, "rewards/reward_model/std": 0.7927289605140686, "step": 375, "step_time": 179.0465439679101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.107421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.5341796875, "completions/mean_terminated_length": 96.10831451416016, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.192191269947216, "epoch": 0.15485996705107083, "frac_reward_zero_std": 0.0, "grad_norm": 0.48471301794052124, "kl": 0.4877035020617768, "learning_rate": 2.973065929861054e-06, "loss": 0.0417, "num_tokens": 175408113.0, "reward": 2.1799428462982178, "reward_std": 0.3564319312572479, "rewards/reward_model/mean": 2.1799428462982178, "rewards/reward_model/std": 0.8470654487609863, "step": 376, "step_time": 176.60565100703388 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0849609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.8544921875, "completions/mean_terminated_length": 84.12699890136719, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.0992068788036704, "epoch": 0.15527182866556838, "frac_reward_zero_std": 0.0, "grad_norm": 0.5215189456939697, "kl": 0.621023507672362, "learning_rate": 2.972657541113338e-06, "loss": 0.0585, "num_tokens": 175866375.0, "reward": 2.4900102615356445, "reward_std": 0.33789998292922974, "rewards/reward_model/mean": 2.4900102615356445, "rewards/reward_model/std": 0.9949961304664612, "step": 377, "step_time": 178.08895701449364 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0654296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.3671875, "completions/mean_terminated_length": 95.22256469726562, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2169749648310244, "epoch": 0.1556836902800659, "frac_reward_zero_std": 0.0, "grad_norm": 0.5098894238471985, "kl": 0.5038701184093952, "learning_rate": 2.9722461079877715e-06, "loss": 0.0393, "num_tokens": 176383511.0, "reward": 2.2236218452453613, "reward_std": 0.33536750078201294, "rewards/reward_model/mean": 2.2236218452453613, "rewards/reward_model/std": 0.7749165892601013, "step": 378, "step_time": 177.30160689540207 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0947265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.64453125, "completions/mean_terminated_length": 91.15425872802734, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1403646562248468, "epoch": 0.15609555189456342, "frac_reward_zero_std": 0.0, "grad_norm": 0.5310172438621521, "kl": 0.5397502090781927, "learning_rate": 2.971831631334897e-06, "loss": 0.0474, "num_tokens": 176863839.0, "reward": 2.3736839294433594, "reward_std": 0.34396374225616455, "rewards/reward_model/mean": 2.3736839294433594, "rewards/reward_model/std": 0.9688174724578857, "step": 379, "step_time": 178.4916840638034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08251953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.58056640625, "completions/mean_terminated_length": 93.7546615600586, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.1766100558452308, "epoch": 0.15650741350906094, "frac_reward_zero_std": 0.0, "grad_norm": 0.7297463417053223, "kl": 0.5337443093303591, "learning_rate": 2.971414112011548e-06, "loss": 0.0438, "num_tokens": 177341892.0, "reward": 2.310980796813965, "reward_std": 0.34860295057296753, "rewards/reward_model/mean": 2.310980796813965, "rewards/reward_model/std": 0.8974020481109619, "step": 380, "step_time": 178.17074899980798 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06201171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.55712890625, "completions/mean_terminated_length": 82.75117492675781, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1021537743508816, "epoch": 0.1569192751235585, "frac_reward_zero_std": 0.0, "grad_norm": 0.5430254936218262, "kl": 0.5842929161153734, "learning_rate": 2.9709935508808496e-06, "loss": 0.0447, "num_tokens": 177784185.0, "reward": 2.2781543731689453, "reward_std": 0.3159905672073364, "rewards/reward_model/mean": 2.2781543731689453, "rewards/reward_model/std": 0.9647110104560852, "step": 381, "step_time": 177.03299723379314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05029296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.65771484375, "completions/mean_terminated_length": 86.57429504394531, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 1.1540411075111479, "epoch": 0.15733113673805602, "frac_reward_zero_std": 0.0, "grad_norm": 0.5368109345436096, "kl": 0.5893488918663934, "learning_rate": 2.970569948812214e-06, "loss": 0.0527, "num_tokens": 178240924.0, "reward": 2.368800640106201, "reward_std": 0.32643285393714905, "rewards/reward_model/mean": 2.368800640106201, "rewards/reward_model/std": 0.851567804813385, "step": 382, "step_time": 177.5798663329333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09521484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.76708984375, "completions/mean_terminated_length": 91.26982879638672, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1531663357745856, "epoch": 0.15774299835255354, "frac_reward_zero_std": 0.0, "grad_norm": 0.5390238165855408, "kl": 0.5587490777252242, "learning_rate": 2.9701433066813405e-06, "loss": 0.0401, "num_tokens": 178711743.0, "reward": 2.1919569969177246, "reward_std": 0.368083655834198, "rewards/reward_model/mean": 2.1919569969177246, "rewards/reward_model/std": 0.9435523152351379, "step": 383, "step_time": 178.15146172884852 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06884765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.60546875, "completions/mean_terminated_length": 81.39695739746094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.11506907036528, "epoch": 0.15815485996705106, "frac_reward_zero_std": 0.0, "grad_norm": 0.5449233055114746, "kl": 0.6226521099451929, "learning_rate": 2.9697136253702126e-06, "loss": 0.0387, "num_tokens": 179224311.0, "reward": 2.423664093017578, "reward_std": 0.3127518892288208, "rewards/reward_model/mean": 2.423664093017578, "rewards/reward_model/std": 0.8909543752670288, "step": 384, "step_time": 178.18525339430198 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1025390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.8671875, "completions/mean_terminated_length": 91.08161163330078, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 1.1719313994981349, "epoch": 0.1585667215815486, "frac_reward_zero_std": 0.0, "grad_norm": 0.548802375793457, "kl": 0.553286608075723, "learning_rate": 2.9692809057670965e-06, "loss": 0.0443, "num_tokens": 179713095.0, "reward": 2.313905954360962, "reward_std": 0.35927480459213257, "rewards/reward_model/mean": 2.313905954360962, "rewards/reward_model/std": 0.9519342184066772, "step": 385, "step_time": 175.52173027023673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.7763671875, "completions/mean_terminated_length": 92.65350341796875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.15960702043958, "epoch": 0.15897858319604613, "frac_reward_zero_std": 0.0, "grad_norm": 0.5294508337974548, "kl": 0.5067053996026516, "learning_rate": 2.96884514876654e-06, "loss": 0.0289, "num_tokens": 180241277.0, "reward": 2.2298755645751953, "reward_std": 0.3278700113296509, "rewards/reward_model/mean": 2.2298755645751953, "rewards/reward_model/std": 0.8828072547912598, "step": 386, "step_time": 177.65252877306193 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.029296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.97705078125, "completions/mean_terminated_length": 83.6785659790039, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0917152806650847, "epoch": 0.15939044481054365, "frac_reward_zero_std": 0.0, "grad_norm": 0.5437387824058533, "kl": 0.6648489931831136, "learning_rate": 2.968406355269369e-06, "loss": 0.0532, "num_tokens": 180746990.0, "reward": 2.4890732765197754, "reward_std": 0.27386534214019775, "rewards/reward_model/mean": 2.4890732765197754, "rewards/reward_model/std": 0.8845586180686951, "step": 387, "step_time": 176.9386322265491 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.9140625, "completions/mean_terminated_length": 88.52632141113281, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1001179211307317, "epoch": 0.15980230642504117, "frac_reward_zero_std": 0.0, "grad_norm": 0.5501876473426819, "kl": 0.6239889877615497, "learning_rate": 2.967964526182688e-06, "loss": 0.0365, "num_tokens": 181240862.0, "reward": 2.4321987628936768, "reward_std": 0.2832774221897125, "rewards/reward_model/mean": 2.4321987628936768, "rewards/reward_model/std": 0.9067803025245667, "step": 388, "step_time": 178.29235074063763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.56884765625, "completions/mean_terminated_length": 88.1436996459961, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 1.0769016318954527, "epoch": 0.16021416803953872, "frac_reward_zero_std": 0.0, "grad_norm": 0.5125551223754883, "kl": 0.6245637289248407, "learning_rate": 2.967519662419876e-06, "loss": 0.0359, "num_tokens": 181763307.0, "reward": 2.4167962074279785, "reward_std": 0.3256126344203949, "rewards/reward_model/mean": 2.4167962074279785, "rewards/reward_model/std": 0.819508969783783, "step": 389, "step_time": 177.810865778476 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.095703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.99560546875, "completions/mean_terminated_length": 89.29103088378906, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.1027363415341824, "epoch": 0.16062602965403625, "frac_reward_zero_std": 0.0, "grad_norm": 0.5195670127868652, "kl": 0.6121240209322423, "learning_rate": 2.9670717649005853e-06, "loss": 0.0379, "num_tokens": 182231458.0, "reward": 2.436800003051758, "reward_std": 0.3338165283203125, "rewards/reward_model/mean": 2.436800003051758, "rewards/reward_model/std": 0.9554334878921509, "step": 390, "step_time": 177.86740992916748 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1484375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 102.00927734375, "completions/mean_terminated_length": 97.4787826538086, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.180186995305121, "epoch": 0.16103789126853377, "frac_reward_zero_std": 0.0, "grad_norm": 0.5072348713874817, "kl": 0.4997855545952916, "learning_rate": 2.9666208345507405e-06, "loss": 0.0304, "num_tokens": 182735445.0, "reward": 2.3285670280456543, "reward_std": 0.35755306482315063, "rewards/reward_model/mean": 2.3285670280456543, "rewards/reward_model/std": 0.8138447403907776, "step": 391, "step_time": 176.45818348089233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.15283203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 101.00390625, "completions/mean_terminated_length": 96.13372039794922, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 1.1727967918850482, "epoch": 0.1614497528830313, "frac_reward_zero_std": 0.0, "grad_norm": 0.7219176888465881, "kl": 0.5146724105579779, "learning_rate": 2.9661668723025345e-06, "loss": 0.0433, "num_tokens": 183239293.0, "reward": 2.046259880065918, "reward_std": 0.3783193528652191, "rewards/reward_model/mean": 2.046259880065918, "rewards/reward_model/std": 0.9237948656082153, "step": 392, "step_time": 177.52231694059446 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.18017578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 102.7744140625, "completions/mean_terminated_length": 97.2304916381836, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.1685084947384894, "epoch": 0.16186161449752884, "frac_reward_zero_std": 0.0, "grad_norm": 0.5364360213279724, "kl": 0.5117919066688046, "learning_rate": 2.9657098790944304e-06, "loss": 0.0338, "num_tokens": 183786767.0, "reward": 2.1263389587402344, "reward_std": 0.3857016861438751, "rewards/reward_model/mean": 2.1263389587402344, "rewards/reward_model/std": 0.9210972785949707, "step": 393, "step_time": 176.74113190779462 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12841796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.0859375, "completions/mean_terminated_length": 91.38375854492188, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1625521839596331, "epoch": 0.16227347611202636, "frac_reward_zero_std": 0.0, "grad_norm": 0.5917537808418274, "kl": 0.5666783119086176, "learning_rate": 2.965249855871155e-06, "loss": 0.0491, "num_tokens": 184283039.0, "reward": 2.4056830406188965, "reward_std": 0.34639692306518555, "rewards/reward_model/mean": 2.4056830406188965, "rewards/reward_model/std": 0.8865830302238464, "step": 394, "step_time": 178.58091470925137 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09619140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.87158203125, "completions/mean_terminated_length": 93.5586166381836, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1989740673452616, "epoch": 0.16268533772652388, "frac_reward_zero_std": 0.0, "grad_norm": 0.5081303119659424, "kl": 0.5465071906801313, "learning_rate": 2.9647868035836998e-06, "loss": 0.0489, "num_tokens": 184752408.0, "reward": 2.2463488578796387, "reward_std": 0.32901906967163086, "rewards/reward_model/mean": 2.2463488578796387, "rewards/reward_model/std": 0.9013258218765259, "step": 395, "step_time": 175.39229744207114 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03955078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.15966796875, "completions/mean_terminated_length": 93.80731964111328, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2094175051897764, "epoch": 0.1630971993410214, "frac_reward_zero_std": 0.0, "grad_norm": 0.4846594035625458, "kl": 0.5309065574547276, "learning_rate": 2.964320723189317e-06, "loss": 0.0398, "num_tokens": 185278367.0, "reward": 2.152841806411743, "reward_std": 0.31126904487609863, "rewards/reward_model/mean": 2.152841806411743, "rewards/reward_model/std": 0.9445822834968567, "step": 396, "step_time": 177.67668041959405 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0361328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.7578125, "completions/mean_terminated_length": 91.43668365478516, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.186940818093717, "epoch": 0.16350906095551895, "frac_reward_zero_std": 0.0, "grad_norm": 0.5278573036193848, "kl": 0.5498459930531681, "learning_rate": 2.963851615651521e-06, "loss": 0.046, "num_tokens": 185781487.0, "reward": 2.449389696121216, "reward_std": 0.323286235332489, "rewards/reward_model/mean": 2.449389696121216, "rewards/reward_model/std": 0.9318462610244751, "step": 397, "step_time": 178.20849479036406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0283203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.21826171875, "completions/mean_terminated_length": 89.11708068847656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1913757142610848, "epoch": 0.16392092257001648, "frac_reward_zero_std": 0.0, "grad_norm": 0.6481385231018066, "kl": 0.5765298299957067, "learning_rate": 2.9633794819400815e-06, "loss": 0.0395, "num_tokens": 186239598.0, "reward": 2.354376792907715, "reward_std": 0.28786200284957886, "rewards/reward_model/mean": 2.354376792907715, "rewards/reward_model/std": 0.8109849095344543, "step": 398, "step_time": 177.42671499820426 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.2216796875, "completions/mean_terminated_length": 85.90625, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.1610781620256603, "epoch": 0.164332784184514, "frac_reward_zero_std": 0.0, "grad_norm": 0.5104904770851135, "kl": 0.6208906166721135, "learning_rate": 2.9629043230310274e-06, "loss": 0.0435, "num_tokens": 186720308.0, "reward": 2.409971237182617, "reward_std": 0.29173052310943604, "rewards/reward_model/mean": 2.409971237182617, "rewards/reward_model/std": 0.8872696161270142, "step": 399, "step_time": 177.13015271630138 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 87.64599609375, "completions/mean_terminated_length": 85.94147491455078, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1466283753979951, "epoch": 0.16474464579901152, "frac_reward_zero_std": 0.0, "grad_norm": 0.5326036810874939, "kl": 0.6419826528290287, "learning_rate": 2.9624261399066377e-06, "loss": 0.0336, "num_tokens": 187282431.0, "reward": 2.321812629699707, "reward_std": 0.3022409677505493, "rewards/reward_model/mean": 2.321812629699707, "rewards/reward_model/std": 0.9636738300323486, "step": 400, "step_time": 177.51840636087582 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.119140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.00634765625, "completions/mean_terminated_length": 92.81430053710938, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1952906486112624, "epoch": 0.16515650741350907, "frac_reward_zero_std": 0.0, "grad_norm": 0.506132960319519, "kl": 0.5479367685038596, "learning_rate": 2.9619449335554455e-06, "loss": 0.044, "num_tokens": 187803788.0, "reward": 2.1437668800354004, "reward_std": 0.3961408734321594, "rewards/reward_model/mean": 2.1437668800354004, "rewards/reward_model/std": 0.9888077974319458, "step": 401, "step_time": 175.43097314983606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.16015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.26318359375, "completions/mean_terminated_length": 87.82965087890625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1123762743081897, "epoch": 0.1655683690280066, "frac_reward_zero_std": 0.0, "grad_norm": 0.5745705962181091, "kl": 0.5608791208942421, "learning_rate": 2.961460704972234e-06, "loss": 0.0476, "num_tokens": 188290663.0, "reward": 2.397477626800537, "reward_std": 0.36399635672569275, "rewards/reward_model/mean": 2.397477626800537, "rewards/reward_model/std": 1.1395490169525146, "step": 402, "step_time": 175.5078289047815 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.4619140625, "completions/mean_terminated_length": 91.95647430419922, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2134921576362103, "epoch": 0.1659802306425041, "frac_reward_zero_std": 0.0, "grad_norm": 0.5102027654647827, "kl": 0.5554043365409598, "learning_rate": 2.9609734551580327e-06, "loss": 0.0384, "num_tokens": 188830041.0, "reward": 2.2964675426483154, "reward_std": 0.3560657799243927, "rewards/reward_model/mean": 2.2964675426483154, "rewards/reward_model/std": 0.9160341620445251, "step": 403, "step_time": 176.9802961065434 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.13720703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.39501953125, "completions/mean_terminated_length": 92.52801513671875, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.1722146437969059, "epoch": 0.16639209225700163, "frac_reward_zero_std": 0.0, "grad_norm": 0.5073086619377136, "kl": 0.54943755234126, "learning_rate": 2.9604831851201175e-06, "loss": 0.0439, "num_tokens": 189337378.0, "reward": 2.306377410888672, "reward_std": 0.35895848274230957, "rewards/reward_model/mean": 2.306377410888672, "rewards/reward_model/std": 0.9279192090034485, "step": 404, "step_time": 175.7343806955032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0576171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.845703125, "completions/mean_terminated_length": 91.75751495361328, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.2383166246581823, "epoch": 0.16680395387149918, "frac_reward_zero_std": 0.0, "grad_norm": 0.527592122554779, "kl": 0.5337270497111604, "learning_rate": 2.959989895872009e-06, "loss": 0.0395, "num_tokens": 189884646.0, "reward": 2.1534171104431152, "reward_std": 0.32572656869888306, "rewards/reward_model/mean": 2.1534171104431152, "rewards/reward_model/std": 0.9540998935699463, "step": 405, "step_time": 177.02214229060337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06103515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.44580078125, "completions/mean_terminated_length": 85.87467193603516, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1009101041126996, "epoch": 0.1672158154859967, "frac_reward_zero_std": 0.0, "grad_norm": 0.4913785457611084, "kl": 0.617920774850063, "learning_rate": 2.959493588433467e-06, "loss": 0.0444, "num_tokens": 190320119.0, "reward": 2.4706740379333496, "reward_std": 0.3032488226890564, "rewards/reward_model/mean": 2.4706740379333496, "rewards/reward_model/std": 0.8560939431190491, "step": 406, "step_time": 176.57884306134656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06201171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.86474609375, "completions/mean_terminated_length": 91.60801696777344, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 1.166981672635302, "epoch": 0.16762767710049423, "frac_reward_zero_std": 0.0, "grad_norm": 0.49378469586372375, "kl": 0.6059474772773683, "learning_rate": 2.9589942638304934e-06, "loss": 0.0411, "num_tokens": 190795138.0, "reward": 2.4793567657470703, "reward_std": 0.31056612730026245, "rewards/reward_model/mean": 2.4793567657470703, "rewards/reward_model/std": 0.8931427597999573, "step": 407, "step_time": 178.11406488483772 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 84.67578125, "completions/mean_terminated_length": 82.16941833496094, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0551256611943245, "epoch": 0.16803953871499178, "frac_reward_zero_std": 0.0, "grad_norm": 0.5458740592002869, "kl": 0.6690047752344981, "learning_rate": 2.9584919230953244e-06, "loss": 0.0481, "num_tokens": 191263818.0, "reward": 2.430650472640991, "reward_std": 0.28612712025642395, "rewards/reward_model/mean": 2.430650472640991, "rewards/reward_model/std": 0.9574082493782043, "step": 408, "step_time": 177.65550036821514 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06787109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.05712890625, "completions/mean_terminated_length": 87.29439544677734, "completions/min_length": 2.0, "completions/min_terminated_length": 2.0, "entropy": 1.1133707312401384, "epoch": 0.1684514003294893, "frac_reward_zero_std": 0.0, "grad_norm": 0.5644468665122986, "kl": 0.5948376578744501, "learning_rate": 2.9579865672664338e-06, "loss": 0.041, "num_tokens": 191764799.0, "reward": 2.4271719455718994, "reward_std": 0.3046577572822571, "rewards/reward_model/mean": 2.4271719455718994, "rewards/reward_model/std": 0.9240416884422302, "step": 409, "step_time": 177.7455269023776 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10595703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 101.0869140625, "completions/mean_terminated_length": 97.8973159790039, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.188851453596726, "epoch": 0.16886326194398682, "frac_reward_zero_std": 0.0, "grad_norm": 0.5382643938064575, "kl": 0.5169988783309236, "learning_rate": 2.9574781973885277e-06, "loss": 0.0468, "num_tokens": 192298481.0, "reward": 2.30741548538208, "reward_std": 0.3707348704338074, "rewards/reward_model/mean": 2.30741548538208, "rewards/reward_model/std": 0.9600353240966797, "step": 410, "step_time": 177.55136401578784 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.76123046875, "completions/mean_terminated_length": 90.34103393554688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1256985717918724, "epoch": 0.16927512355848434, "frac_reward_zero_std": 0.0, "grad_norm": 0.4733363389968872, "kl": 0.5666498588398099, "learning_rate": 2.956966814512543e-06, "loss": 0.0401, "num_tokens": 192806856.0, "reward": 2.275219440460205, "reward_std": 0.3202829360961914, "rewards/reward_model/mean": 2.275219440460205, "rewards/reward_model/std": 0.8682781457901001, "step": 411, "step_time": 178.21866449015215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10595703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.5224609375, "completions/mean_terminated_length": 91.67340087890625, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 1.1405959548428655, "epoch": 0.1696869851729819, "frac_reward_zero_std": 0.0, "grad_norm": 0.4965401291847229, "kl": 0.5722325802780688, "learning_rate": 2.9564524196956443e-06, "loss": 0.0416, "num_tokens": 193248150.0, "reward": 2.197939157485962, "reward_std": 0.33323246240615845, "rewards/reward_model/mean": 2.197939157485962, "rewards/reward_model/std": 0.9109729528427124, "step": 412, "step_time": 178.2320651030168 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10302734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.953125, "completions/mean_terminated_length": 96.73162078857422, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.1993136450182647, "epoch": 0.1700988467874794, "frac_reward_zero_std": 0.0, "grad_norm": 0.5145481824874878, "kl": 0.5244667251827195, "learning_rate": 2.955935014001224e-06, "loss": 0.0356, "num_tokens": 193706070.0, "reward": 2.3130741119384766, "reward_std": 0.35740673542022705, "rewards/reward_model/mean": 2.3130741119384766, "rewards/reward_model/std": 0.8100281953811646, "step": 413, "step_time": 177.37916512275115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.15380859375, "completions/mean_terminated_length": 96.70921325683594, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.1724783945828676, "epoch": 0.17051070840197693, "frac_reward_zero_std": 0.0, "grad_norm": 0.49485906958580017, "kl": 0.5356933635193855, "learning_rate": 2.955414598498899e-06, "loss": 0.0383, "num_tokens": 194167313.0, "reward": 2.3801841735839844, "reward_std": 0.3082989752292633, "rewards/reward_model/mean": 2.3801841735839844, "rewards/reward_model/std": 0.8933265209197998, "step": 414, "step_time": 180.22052889876068 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.3154296875, "completions/mean_terminated_length": 92.10718536376953, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1516162871848792, "epoch": 0.17092257001647446, "frac_reward_zero_std": 0.0, "grad_norm": 0.4951915442943573, "kl": 0.5319540449418128, "learning_rate": 2.954891174264507e-06, "loss": 0.0269, "num_tokens": 194646071.0, "reward": 2.3047561645507812, "reward_std": 0.3134269714355469, "rewards/reward_model/mean": 2.3047561645507812, "rewards/reward_model/std": 0.799450159072876, "step": 415, "step_time": 178.1219038334675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06298828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.76318359375, "completions/mean_terminated_length": 92.52891540527344, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1199587755836546, "epoch": 0.171334431630972, "frac_reward_zero_std": 0.0, "grad_norm": 0.4877195358276367, "kl": 0.5455110664479434, "learning_rate": 2.9543647423801063e-06, "loss": 0.0294, "num_tokens": 195132946.0, "reward": 2.3860585689544678, "reward_std": 0.32239750027656555, "rewards/reward_model/mean": 2.3860585689544678, "rewards/reward_model/std": 0.9772287607192993, "step": 416, "step_time": 177.29199483664706 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0712890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.169921875, "completions/mean_terminated_length": 88.34278869628906, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1120513607747853, "epoch": 0.17174629324546953, "frac_reward_zero_std": 0.0, "grad_norm": 0.6118646860122681, "kl": 0.6378167612710968, "learning_rate": 2.9538353039339722e-06, "loss": 0.0471, "num_tokens": 195575054.0, "reward": 2.4343907833099365, "reward_std": 0.31432047486305237, "rewards/reward_model/mean": 2.4343907833099365, "rewards/reward_model/std": 0.8919382691383362, "step": 417, "step_time": 178.19161146692932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.099609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.5791015625, "completions/mean_terminated_length": 90.88178253173828, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1288885136600584, "epoch": 0.17215815485996705, "frac_reward_zero_std": 0.0, "grad_norm": 0.5038954019546509, "kl": 0.5964081999845803, "learning_rate": 2.953302860020597e-06, "loss": 0.0388, "num_tokens": 196054096.0, "reward": 2.349412679672241, "reward_std": 0.3208439350128174, "rewards/reward_model/mean": 2.349412679672241, "rewards/reward_model/std": 0.9101561903953552, "step": 418, "step_time": 177.92830974142998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.15966796875, "completions/mean_terminated_length": 95.26673889160156, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.1778133569750935, "epoch": 0.17257001647446457, "frac_reward_zero_std": 0.0, "grad_norm": 0.47825726866722107, "kl": 0.5348276777658612, "learning_rate": 2.9527674117406837e-06, "loss": 0.0352, "num_tokens": 196534263.0, "reward": 2.217905044555664, "reward_std": 0.3482544422149658, "rewards/reward_model/mean": 2.217905044555664, "rewards/reward_model/std": 0.9416971206665039, "step": 419, "step_time": 177.1391870630905 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 81.6259765625, "completions/mean_terminated_length": 78.58585357666016, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.9905391852371395, "epoch": 0.17298187808896212, "frac_reward_zero_std": 0.0, "grad_norm": 0.5390706658363342, "kl": 0.6956530390307307, "learning_rate": 2.9522289602011486e-06, "loss": 0.0408, "num_tokens": 197010873.0, "reward": 2.6095123291015625, "reward_std": 0.2852742671966553, "rewards/reward_model/mean": 2.6095123291015625, "rewards/reward_model/std": 1.021905541419983, "step": 420, "step_time": 177.74983442714438 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.4638671875, "completions/mean_terminated_length": 93.31364440917969, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1362889125011861, "epoch": 0.17339373970345964, "frac_reward_zero_std": 0.0, "grad_norm": 0.49332642555236816, "kl": 0.5611731224926189, "learning_rate": 2.951687506515114e-06, "loss": 0.0368, "num_tokens": 197510735.0, "reward": 2.4012043476104736, "reward_std": 0.32887279987335205, "rewards/reward_model/mean": 2.4012043476104736, "rewards/reward_model/std": 0.802983283996582, "step": 421, "step_time": 178.92538169259205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0810546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.93505859375, "completions/mean_terminated_length": 89.8421859741211, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.078607544535771, "epoch": 0.17380560131795716, "frac_reward_zero_std": 0.0, "grad_norm": 0.4767504036426544, "kl": 0.597627462586388, "learning_rate": 2.951143051801911e-06, "loss": 0.0397, "num_tokens": 197994282.0, "reward": 2.4638566970825195, "reward_std": 0.3232494592666626, "rewards/reward_model/mean": 2.4638566970825195, "rewards/reward_model/std": 1.0042797327041626, "step": 422, "step_time": 176.72820660145953 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.9638671875, "completions/mean_terminated_length": 98.48507690429688, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1821262266021222, "epoch": 0.1742174629324547, "frac_reward_zero_std": 0.0, "grad_norm": 0.5219224095344543, "kl": 0.5636481594992802, "learning_rate": 2.9505955971870733e-06, "loss": 0.0269, "num_tokens": 198496096.0, "reward": 2.2261011600494385, "reward_std": 0.3572756052017212, "rewards/reward_model/mean": 2.2261011600494385, "rewards/reward_model/std": 0.9350313544273376, "step": 423, "step_time": 178.598951802589 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0673828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.94970703125, "completions/mean_terminated_length": 87.20052337646484, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1196602610871196, "epoch": 0.17462932454695224, "frac_reward_zero_std": 0.0, "grad_norm": 0.5144994854927063, "kl": 0.6399828335270286, "learning_rate": 2.950045143802336e-06, "loss": 0.038, "num_tokens": 198955865.0, "reward": 2.451430559158325, "reward_std": 0.3187256157398224, "rewards/reward_model/mean": 2.451430559158325, "rewards/reward_model/std": 0.9397474527359009, "step": 424, "step_time": 179.72773122834042 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.74951171875, "completions/mean_terminated_length": 91.17847442626953, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.1110601257532835, "epoch": 0.17504118616144976, "frac_reward_zero_std": 0.0, "grad_norm": 0.5179552435874939, "kl": 0.5934415927622467, "learning_rate": 2.949491692785633e-06, "loss": 0.0343, "num_tokens": 199430392.0, "reward": 2.494533061981201, "reward_std": 0.3300376534461975, "rewards/reward_model/mean": 2.494533061981201, "rewards/reward_model/std": 0.9257327318191528, "step": 425, "step_time": 178.6376907993108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08447265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.46044921875, "completions/mean_terminated_length": 96.82720184326172, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1673854449763894, "epoch": 0.17545304777594728, "frac_reward_zero_std": 0.0, "grad_norm": 0.5084400177001953, "kl": 0.5602261393796653, "learning_rate": 2.9489352452810977e-06, "loss": 0.0336, "num_tokens": 199956903.0, "reward": 2.416677951812744, "reward_std": 0.3193284273147583, "rewards/reward_model/mean": 2.416677951812744, "rewards/reward_model/std": 0.8601666688919067, "step": 426, "step_time": 177.77451831940562 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1220703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.29150390625, "completions/mean_terminated_length": 95.29977416992188, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1528692690189928, "epoch": 0.1758649093904448, "frac_reward_zero_std": 0.0, "grad_norm": 0.5529664754867554, "kl": 0.5827847815817222, "learning_rate": 2.948375802439055e-06, "loss": 0.027, "num_tokens": 200428060.0, "reward": 2.440098524093628, "reward_std": 0.35225480794906616, "rewards/reward_model/mean": 2.440098524093628, "rewards/reward_model/std": 0.913156270980835, "step": 427, "step_time": 178.14818801917136 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.400390625, "completions/mean_terminated_length": 94.15271759033203, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.1740019149146974, "epoch": 0.17627677100494235, "frac_reward_zero_std": 0.0, "grad_norm": 0.5385034680366516, "kl": 0.6034399388590828, "learning_rate": 2.9478133654160233e-06, "loss": 0.0373, "num_tokens": 200948688.0, "reward": 2.448856830596924, "reward_std": 0.3294673562049866, "rewards/reward_model/mean": 2.448856830596924, "rewards/reward_model/std": 0.9103710651397705, "step": 428, "step_time": 178.2354981545359 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06591796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.2626953125, "completions/mean_terminated_length": 82.2467269897461, "completions/min_length": 13.0, "completions/min_terminated_length": 13.0, "entropy": 1.0678621237166226, "epoch": 0.17668863261943987, "frac_reward_zero_std": 0.0, "grad_norm": 0.5149862170219421, "kl": 0.7372386472998187, "learning_rate": 2.9472479353747103e-06, "loss": 0.0388, "num_tokens": 201377002.0, "reward": 2.63199782371521, "reward_std": 0.2893105745315552, "rewards/reward_model/mean": 2.63199782371521, "rewards/reward_model/std": 0.7980876564979553, "step": 429, "step_time": 177.70909517724067 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.099609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.70556640625, "completions/mean_terminated_length": 97.6860122680664, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.1862491094507277, "epoch": 0.1771004942339374, "frac_reward_zero_std": 0.0, "grad_norm": 0.5579699277877808, "kl": 0.5715408079558983, "learning_rate": 2.9466795134840126e-06, "loss": 0.0411, "num_tokens": 201889295.0, "reward": 2.358752727508545, "reward_std": 0.33939802646636963, "rewards/reward_model/mean": 2.358752727508545, "rewards/reward_model/std": 0.8526293635368347, "step": 430, "step_time": 176.758549827151 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0673828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.16552734375, "completions/mean_terminated_length": 85.28743743896484, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 1.1023415883537382, "epoch": 0.17751235584843492, "frac_reward_zero_std": 0.0, "grad_norm": 0.5216069221496582, "kl": 0.6755480580031872, "learning_rate": 2.9461081009190098e-06, "loss": 0.0443, "num_tokens": 202364258.0, "reward": 2.686612844467163, "reward_std": 0.29735130071640015, "rewards/reward_model/mean": 2.686612844467163, "rewards/reward_model/std": 0.8905471563339233, "step": 431, "step_time": 176.51525117084384 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05712890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.8232421875, "completions/mean_terminated_length": 95.99481964111328, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.1819677515886724, "epoch": 0.17792421746293247, "frac_reward_zero_std": 0.0, "grad_norm": 0.485084593296051, "kl": 0.5605144231813028, "learning_rate": 2.945533698860965e-06, "loss": 0.0345, "num_tokens": 202860792.0, "reward": 2.4848527908325195, "reward_std": 0.3100086450576782, "rewards/reward_model/mean": 2.4848527908325195, "rewards/reward_model/std": 0.8651890754699707, "step": 432, "step_time": 176.5795973110944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04833984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.44580078125, "completions/mean_terminated_length": 86.43663787841797, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.119892367394641, "epoch": 0.17833607907743, "frac_reward_zero_std": 0.0, "grad_norm": 0.7239587306976318, "kl": 0.6481211216887459, "learning_rate": 2.944956308497321e-06, "loss": 0.0609, "num_tokens": 203316233.0, "reward": 2.469818592071533, "reward_std": 0.319928377866745, "rewards/reward_model/mean": 2.469818592071533, "rewards/reward_model/std": 0.8600060343742371, "step": 433, "step_time": 176.51235378254205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08056640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.146484375, "completions/mean_terminated_length": 97.7057876586914, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.2105460944585502, "epoch": 0.1787479406919275, "frac_reward_zero_std": 0.0, "grad_norm": 0.48540621995925903, "kl": 0.5203440812183544, "learning_rate": 2.944375931021699e-06, "loss": 0.0365, "num_tokens": 203865781.0, "reward": 2.1294150352478027, "reward_std": 0.37899887561798096, "rewards/reward_model/mean": 2.1294150352478027, "rewards/reward_model/std": 0.9112393856048584, "step": 434, "step_time": 176.66102132527158 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.583984375, "completions/mean_terminated_length": 89.15625762939453, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1091220492962748, "epoch": 0.17915980230642503, "frac_reward_zero_std": 0.0, "grad_norm": 0.5116854310035706, "kl": 0.65462797798682, "learning_rate": 2.9437925676338944e-06, "loss": 0.0615, "num_tokens": 204363905.0, "reward": 2.5014491081237793, "reward_std": 0.3244835138320923, "rewards/reward_model/mean": 2.5014491081237793, "rewards/reward_model/std": 0.8409462571144104, "step": 435, "step_time": 176.92669505951926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.205078125, "completions/mean_terminated_length": 91.66824340820312, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.183810024522245, "epoch": 0.17957166392092258, "frac_reward_zero_std": 0.0, "grad_norm": 0.507256031036377, "kl": 0.5834380444139242, "learning_rate": 2.9432062195398753e-06, "loss": 0.0381, "num_tokens": 204843429.0, "reward": 2.407453775405884, "reward_std": 0.33924970030784607, "rewards/reward_model/mean": 2.407453775405884, "rewards/reward_model/std": 0.9802740216255188, "step": 436, "step_time": 178.91840605437756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12841796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 102.6591796875, "completions/mean_terminated_length": 98.92549133300781, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.142559586558491, "epoch": 0.1799835255354201, "frac_reward_zero_std": 0.0, "grad_norm": 0.48775413632392883, "kl": 0.5294286153512076, "learning_rate": 2.9426168879517812e-06, "loss": 0.038, "num_tokens": 205302891.0, "reward": 2.2659778594970703, "reward_std": 0.37942689657211304, "rewards/reward_model/mean": 2.2659778594970703, "rewards/reward_model/std": 0.8682555556297302, "step": 437, "step_time": 174.95479731122032 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08544921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.23046875, "completions/mean_terminated_length": 97.63587951660156, "completions/min_length": 9.0, "completions/min_terminated_length": 9.0, "entropy": 1.2018397455103695, "epoch": 0.18039538714991762, "frac_reward_zero_std": 0.0, "grad_norm": 0.5316077470779419, "kl": 0.5363015634939075, "learning_rate": 2.9420245740879177e-06, "loss": 0.031, "num_tokens": 205829411.0, "reward": 2.2885236740112305, "reward_std": 0.3580632209777832, "rewards/reward_model/mean": 2.2885236740112305, "rewards/reward_model/std": 0.90251225233078, "step": 438, "step_time": 176.55790072493255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.1484375, "completions/mean_terminated_length": 93.30258178710938, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1315481034107506, "epoch": 0.18080724876441515, "frac_reward_zero_std": 0.0, "grad_norm": 0.4738367199897766, "kl": 0.5569514802191406, "learning_rate": 2.941429279172757e-06, "loss": 0.0418, "num_tokens": 206318131.0, "reward": 2.3975930213928223, "reward_std": 0.3421768546104431, "rewards/reward_model/mean": 2.3975930213928223, "rewards/reward_model/std": 0.7411666512489319, "step": 439, "step_time": 176.7504616943188 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14697265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.8681640625, "completions/mean_terminated_length": 95.02117919921875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1570600166451186, "epoch": 0.1812191103789127, "frac_reward_zero_std": 0.0, "grad_norm": 0.4971511662006378, "kl": 0.5430542938411236, "learning_rate": 2.9408310044369323e-06, "loss": 0.0404, "num_tokens": 206859813.0, "reward": 2.3325583934783936, "reward_std": 0.38505664467811584, "rewards/reward_model/mean": 2.3325583934783936, "rewards/reward_model/std": 0.9573835134506226, "step": 440, "step_time": 177.64647138258442 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11669921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.47705078125, "completions/mean_terminated_length": 91.18020629882812, "completions/min_length": 24.0, "completions/min_terminated_length": 24.0, "entropy": 1.0904387107584625, "epoch": 0.18163097199341022, "frac_reward_zero_std": 0.0, "grad_norm": 0.5499387383460999, "kl": 0.6065284563228488, "learning_rate": 2.940229751117239e-06, "loss": 0.032, "num_tokens": 207325526.0, "reward": 2.5178890228271484, "reward_std": 0.3385392427444458, "rewards/reward_model/mean": 2.5178890228271484, "rewards/reward_model/std": 0.8110549449920654, "step": 441, "step_time": 176.5419153519906 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09228515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.67724609375, "completions/mean_terminated_length": 87.9843978881836, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0984761593863368, "epoch": 0.18204283360790774, "frac_reward_zero_std": 0.0, "grad_norm": 0.4781702756881714, "kl": 0.6365486000431702, "learning_rate": 2.939625520456628e-06, "loss": 0.0402, "num_tokens": 207855585.0, "reward": 2.623962879180908, "reward_std": 0.3231748044490814, "rewards/reward_model/mean": 2.623962879180908, "rewards/reward_model/std": 1.0030767917633057, "step": 442, "step_time": 176.57954967115074 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.32470703125, "completions/mean_terminated_length": 90.20227813720703, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0942314735148102, "epoch": 0.18245469522240526, "frac_reward_zero_std": 0.0, "grad_norm": 0.4861615300178528, "kl": 0.5833023431478068, "learning_rate": 2.9390183137042067e-06, "loss": 0.04, "num_tokens": 208365498.0, "reward": 2.4050230979919434, "reward_std": 0.3347085118293762, "rewards/reward_model/mean": 2.4050230979919434, "rewards/reward_model/std": 0.7748783230781555, "step": 443, "step_time": 177.43503043008968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03857421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 93.82177734375, "completions/mean_terminated_length": 92.45048522949219, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "entropy": 1.1592300543561578, "epoch": 0.1828665568369028, "frac_reward_zero_std": 0.0, "grad_norm": 0.514148473739624, "kl": 0.5543227386660874, "learning_rate": 2.9384081321152333e-06, "loss": 0.0296, "num_tokens": 208869741.0, "reward": 2.1709766387939453, "reward_std": 0.31762227416038513, "rewards/reward_model/mean": 2.1709766387939453, "rewards/reward_model/std": 0.852414608001709, "step": 444, "step_time": 175.31828605988994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 85.93896484375, "completions/mean_terminated_length": 84.5821533203125, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0829803219530731, "epoch": 0.18327841845140033, "frac_reward_zero_std": 0.0, "grad_norm": 0.6001779437065125, "kl": 0.6650931317126378, "learning_rate": 2.9377949769511173e-06, "loss": 0.0353, "num_tokens": 209343568.0, "reward": 2.472573757171631, "reward_std": 0.2634936273097992, "rewards/reward_model/mean": 2.472573757171631, "rewards/reward_model/std": 0.8676891326904297, "step": 445, "step_time": 177.0883853463456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04345703125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.35791015625, "completions/mean_terminated_length": 84.46605682373047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0314271703828126, "epoch": 0.18369028006589785, "frac_reward_zero_std": 0.0, "grad_norm": 0.5127878785133362, "kl": 0.6577366165583953, "learning_rate": 2.9371788494794146e-06, "loss": 0.0428, "num_tokens": 209788429.0, "reward": 2.4889237880706787, "reward_std": 0.28512921929359436, "rewards/reward_model/mean": 2.4889237880706787, "rewards/reward_model/std": 0.9370663166046143, "step": 446, "step_time": 175.78170171938837 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0673828125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.3876953125, "completions/mean_terminated_length": 89.81465911865234, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.128923305310309, "epoch": 0.18410214168039538, "frac_reward_zero_std": 0.0, "grad_norm": 0.5161535143852234, "kl": 0.57824415655341, "learning_rate": 2.9365597509738264e-06, "loss": 0.0385, "num_tokens": 210312391.0, "reward": 2.297618865966797, "reward_std": 0.31075942516326904, "rewards/reward_model/mean": 2.297618865966797, "rewards/reward_model/std": 0.8569238781929016, "step": 447, "step_time": 178.10298594087362 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.14111328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 103.72998046875, "completions/mean_terminated_length": 99.74246978759766, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.098568452289328, "epoch": 0.18451400329489293, "frac_reward_zero_std": 0.0, "grad_norm": 0.5501395463943481, "kl": 0.5400044284760952, "learning_rate": 2.9359376827141947e-06, "loss": 0.0331, "num_tokens": 210798814.0, "reward": 2.30403733253479, "reward_std": 0.34964519739151, "rewards/reward_model/mean": 2.30403733253479, "rewards/reward_model/std": 0.8943761587142944, "step": 448, "step_time": 176.38012149278075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10009765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.8125, "completions/mean_terminated_length": 94.45469665527344, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.0876766440924257, "epoch": 0.18492586490939045, "frac_reward_zero_std": 0.0, "grad_norm": 0.48033761978149414, "kl": 0.5915502562420443, "learning_rate": 2.935312645986501e-06, "loss": 0.036, "num_tokens": 211312926.0, "reward": 2.447918653488159, "reward_std": 0.31356704235076904, "rewards/reward_model/mean": 2.447918653488159, "rewards/reward_model/std": 0.8935808539390564, "step": 449, "step_time": 177.1732467082329 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07568359375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.86328125, "completions/mean_terminated_length": 95.39566802978516, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1029284975957125, "epoch": 0.18533772652388797, "frac_reward_zero_std": 0.0, "grad_norm": 0.482118159532547, "kl": 0.5952668602112681, "learning_rate": 2.934684642082864e-06, "loss": 0.0359, "num_tokens": 211852134.0, "reward": 2.3744916915893555, "reward_std": 0.32203009724617004, "rewards/reward_model/mean": 2.3744916915893555, "rewards/reward_model/std": 0.8777472376823425, "step": 450, "step_time": 177.8922571069561 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0537109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.150390625, "completions/mean_terminated_length": 93.2858657836914, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1415106805507094, "epoch": 0.1857495881383855, "frac_reward_zero_std": 0.0, "grad_norm": 0.5331604480743408, "kl": 0.5654863331001252, "learning_rate": 2.9340536723015366e-06, "loss": 0.0356, "num_tokens": 212406682.0, "reward": 2.3145880699157715, "reward_std": 0.33033740520477295, "rewards/reward_model/mean": 2.3145880699157715, "rewards/reward_model/std": 0.8525269031524658, "step": 451, "step_time": 179.43042960343882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09130859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.0166015625, "completions/mean_terminated_length": 92.80279541015625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0888551205862314, "epoch": 0.18616144975288304, "frac_reward_zero_std": 0.0, "grad_norm": 0.5692293643951416, "kl": 0.6042150420835242, "learning_rate": 2.9334197379469016e-06, "loss": 0.0279, "num_tokens": 212923452.0, "reward": 2.3162081241607666, "reward_std": 0.3242701292037964, "rewards/reward_model/mean": 2.3162081241607666, "rewards/reward_model/std": 1.0072386264801025, "step": 452, "step_time": 177.41770586324856 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.58203125, "completions/mean_terminated_length": 95.82231140136719, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1837097681127489, "epoch": 0.18657331136738056, "frac_reward_zero_std": 0.0, "grad_norm": 0.5043319463729858, "kl": 0.5810586004517972, "learning_rate": 2.9327828403294722e-06, "loss": 0.0278, "num_tokens": 213503492.0, "reward": 2.3591485023498535, "reward_std": 0.318893700838089, "rewards/reward_model/mean": 2.3591485023498535, "rewards/reward_model/std": 0.9125985503196716, "step": 453, "step_time": 179.33237711060792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08837890625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 102.17578125, "completions/mean_terminated_length": 99.67220306396484, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1589810342993587, "epoch": 0.18698517298187808, "frac_reward_zero_std": 0.0, "grad_norm": 0.46748653054237366, "kl": 0.5620730358641595, "learning_rate": 2.932142980765886e-06, "loss": 0.0303, "num_tokens": 214035244.0, "reward": 2.3349175453186035, "reward_std": 0.3226012587547302, "rewards/reward_model/mean": 2.3349175453186035, "rewards/reward_model/std": 0.7850995063781738, "step": 454, "step_time": 178.44097527628765 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06494140625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.00439453125, "completions/mean_terminated_length": 91.64334106445312, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1147995276842266, "epoch": 0.1873970345963756, "frac_reward_zero_std": 0.0, "grad_norm": 0.6195225715637207, "kl": 0.6642492334358394, "learning_rate": 2.9315001605789038e-06, "loss": 0.0355, "num_tokens": 214489717.0, "reward": 2.5178701877593994, "reward_std": 0.3080519735813141, "rewards/reward_model/mean": 2.5178701877593994, "rewards/reward_model/std": 0.8761839270591736, "step": 455, "step_time": 178.27048154268414 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10205078125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.75439453125, "completions/mean_terminated_length": 96.54431915283203, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1883398399222642, "epoch": 0.18780889621087316, "frac_reward_zero_std": 0.0, "grad_norm": 0.510930597782135, "kl": 0.5764352440601215, "learning_rate": 2.9308543810974077e-06, "loss": 0.0462, "num_tokens": 215003902.0, "reward": 2.2514212131500244, "reward_std": 0.3160772919654846, "rewards/reward_model/mean": 2.2514212131500244, "rewards/reward_model/std": 0.8512153029441833, "step": 456, "step_time": 178.02870258502662 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11474609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.42333984375, "completions/mean_terminated_length": 93.46000671386719, "completions/min_length": 31.0, "completions/min_terminated_length": 31.0, "entropy": 1.16164245409891, "epoch": 0.18822075782537068, "frac_reward_zero_std": 0.0, "grad_norm": 0.6683105826377869, "kl": 0.6036664441926405, "learning_rate": 2.9302056436563965e-06, "loss": 0.048, "num_tokens": 215475681.0, "reward": 2.2650651931762695, "reward_std": 0.3349018096923828, "rewards/reward_model/mean": 2.2650651931762695, "rewards/reward_model/std": 0.842054545879364, "step": 457, "step_time": 177.5618844004348 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.8203125, "completions/mean_terminated_length": 94.9468994140625, "completions/min_length": 22.0, "completions/min_terminated_length": 22.0, "entropy": 1.1730834019836038, "epoch": 0.1886326194398682, "frac_reward_zero_std": 0.0, "grad_norm": 0.513211190700531, "kl": 0.5681351118255407, "learning_rate": 2.9295539495969854e-06, "loss": 0.0387, "num_tokens": 215972465.0, "reward": 2.4165258407592773, "reward_std": 0.3427693843841553, "rewards/reward_model/mean": 2.4165258407592773, "rewards/reward_model/std": 0.9288755655288696, "step": 458, "step_time": 180.19577173981816 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0634765625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.59521484375, "completions/mean_terminated_length": 96.60218811035156, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1844341268297285, "epoch": 0.18904448105436572, "frac_reward_zero_std": 0.0, "grad_norm": 0.49073705077171326, "kl": 0.5458073208574206, "learning_rate": 2.9288993002664e-06, "loss": 0.0398, "num_tokens": 216469588.0, "reward": 2.2943968772888184, "reward_std": 0.32561981678009033, "rewards/reward_model/mean": 2.2943968772888184, "rewards/reward_model/std": 0.9535711407661438, "step": 459, "step_time": 178.42609012499452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0322265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.24609375, "completions/mean_terminated_length": 91.05549621582031, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 1.1161227691918612, "epoch": 0.18945634266886327, "frac_reward_zero_std": 0.0, "grad_norm": 0.5149496793746948, "kl": 0.6099860027898103, "learning_rate": 2.928241697017976e-06, "loss": 0.0235, "num_tokens": 216948076.0, "reward": 2.4459617137908936, "reward_std": 0.27860504388809204, "rewards/reward_model/mean": 2.4459617137908936, "rewards/reward_model/std": 0.8823313117027283, "step": 460, "step_time": 176.95573361217976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02392578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 89.01220703125, "completions/mean_terminated_length": 88.05653381347656, "completions/min_length": 20.0, "completions/min_terminated_length": 20.0, "entropy": 1.1527804306242615, "epoch": 0.1898682042833608, "frac_reward_zero_std": 0.0, "grad_norm": 0.5291562676429749, "kl": 0.6793452298734337, "learning_rate": 2.927581141211155e-06, "loss": 0.0377, "num_tokens": 217400997.0, "reward": 2.5573511123657227, "reward_std": 0.2717992663383484, "rewards/reward_model/mean": 2.5573511123657227, "rewards/reward_model/std": 0.8602915406227112, "step": 461, "step_time": 177.6904992805794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0498046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.7861328125, "completions/mean_terminated_length": 93.04522705078125, "completions/min_length": 17.0, "completions/min_terminated_length": 17.0, "entropy": 1.2037953287363052, "epoch": 0.1902800658978583, "frac_reward_zero_std": 0.0, "grad_norm": 0.5041398406028748, "kl": 0.5906063452130184, "learning_rate": 2.926917634211484e-06, "loss": 0.0313, "num_tokens": 217954511.0, "reward": 2.3205208778381348, "reward_std": 0.3215928077697754, "rewards/reward_model/mean": 2.3205208778381348, "rewards/reward_model/std": 0.9974476099014282, "step": 462, "step_time": 178.31891357479617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0380859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.42236328125, "completions/mean_terminated_length": 84.7761459350586, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.0918651034589857, "epoch": 0.19069192751235584, "frac_reward_zero_std": 0.0, "grad_norm": 0.5197232365608215, "kl": 0.6783080940367654, "learning_rate": 2.926251177390609e-06, "loss": 0.0402, "num_tokens": 218384944.0, "reward": 2.5142483711242676, "reward_std": 0.28619104623794556, "rewards/reward_model/mean": 2.5142483711242676, "rewards/reward_model/std": 0.8500559329986572, "step": 463, "step_time": 177.18723722640425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0615234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.7880859375, "completions/mean_terminated_length": 86.21748352050781, "completions/min_length": 21.0, "completions/min_terminated_length": 21.0, "entropy": 1.0844496260397136, "epoch": 0.19110378912685339, "frac_reward_zero_std": 0.0, "grad_norm": 0.4965822398662567, "kl": 0.662226699758321, "learning_rate": 2.9255817721262743e-06, "loss": 0.0356, "num_tokens": 218857694.0, "reward": 2.510044574737549, "reward_std": 0.27605462074279785, "rewards/reward_model/mean": 2.510044574737549, "rewards/reward_model/std": 0.8087391257286072, "step": 464, "step_time": 177.06814434053376 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.11376953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.59326171875, "completions/mean_terminated_length": 92.56143951416016, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0997457699850202, "epoch": 0.1915156507413509, "frac_reward_zero_std": 0.0, "grad_norm": 0.5444568395614624, "kl": 0.6190934079932049, "learning_rate": 2.924909419802322e-06, "loss": 0.0427, "num_tokens": 219348061.0, "reward": 2.439056396484375, "reward_std": 0.32993441820144653, "rewards/reward_model/mean": 2.439056396484375, "rewards/reward_model/std": 0.7995099425315857, "step": 465, "step_time": 177.1582528813742 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08935546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.20703125, "completions/mean_terminated_length": 87.5967788696289, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 1.1058951758313924, "epoch": 0.19192751235584843, "frac_reward_zero_std": 0.0, "grad_norm": 0.5266746878623962, "kl": 0.6608873887453228, "learning_rate": 2.924234121808683e-06, "loss": 0.04, "num_tokens": 219892485.0, "reward": 2.427839994430542, "reward_std": 0.304832398891449, "rewards/reward_model/mean": 2.427839994430542, "rewards/reward_model/std": 0.8829104900360107, "step": 466, "step_time": 176.96438163286075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0478515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.83447265625, "completions/mean_terminated_length": 97.36871337890625, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1630366663448513, "epoch": 0.19233937397034595, "frac_reward_zero_std": 0.0, "grad_norm": 0.5191803574562073, "kl": 0.590993246762082, "learning_rate": 2.92355587954138e-06, "loss": 0.0311, "num_tokens": 220448434.0, "reward": 2.4503188133239746, "reward_std": 0.2877051532268524, "rewards/reward_model/mean": 2.4503188133239746, "rewards/reward_model/std": 0.7933539748191833, "step": 467, "step_time": 177.20374609716237 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05224609375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.2099609375, "completions/mean_terminated_length": 94.4574966430664, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 1.15341454022564, "epoch": 0.1927512355848435, "frac_reward_zero_std": 0.0, "grad_norm": 0.5067684054374695, "kl": 0.5940121430903673, "learning_rate": 2.922874694402521e-06, "loss": 0.0388, "num_tokens": 220935424.0, "reward": 2.4387810230255127, "reward_std": 0.3035017251968384, "rewards/reward_model/mean": 2.4387810230255127, "rewards/reward_model/std": 0.9896895885467529, "step": 468, "step_time": 175.75350676244125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0556640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 101.8916015625, "completions/mean_terminated_length": 100.3526382446289, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1894471587147564, "epoch": 0.19316309719934102, "frac_reward_zero_std": 0.0, "grad_norm": 0.5075152516365051, "kl": 0.5726111725671217, "learning_rate": 2.922190567800298e-06, "loss": 0.0317, "num_tokens": 221446978.0, "reward": 2.3622727394104004, "reward_std": 0.30957335233688354, "rewards/reward_model/mean": 2.3622727394104004, "rewards/reward_model/std": 0.8335116505622864, "step": 469, "step_time": 176.19587636506185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 97.76513671875, "completions/mean_terminated_length": 95.61454010009766, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 1.1029500795993954, "epoch": 0.19357495881383854, "frac_reward_zero_std": 0.0, "grad_norm": 0.46881791949272156, "kl": 0.5983435865491629, "learning_rate": 2.921503501148985e-06, "loss": 0.0315, "num_tokens": 221963201.0, "reward": 2.403939723968506, "reward_std": 0.2895151674747467, "rewards/reward_model/mean": 2.403939723968506, "rewards/reward_model/std": 0.7417267560958862, "step": 470, "step_time": 177.0896590379998 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0439453125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 88.3759765625, "completions/mean_terminated_length": 86.55464935302734, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 1.0182313050609082, "epoch": 0.19398682042833607, "frac_reward_zero_std": 0.0, "grad_norm": 0.9269804954528809, "kl": 0.7156516662798822, "learning_rate": 2.920813495868932e-06, "loss": 0.0454, "num_tokens": 222473187.0, "reward": 2.5426406860351562, "reward_std": 0.27168920636177063, "rewards/reward_model/mean": 2.5426406860351562, "rewards/reward_model/std": 0.8641615509986877, "step": 471, "step_time": 178.64879790926352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08544921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.33056640625, "completions/mean_terminated_length": 97.74533081054688, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.143377386033535, "epoch": 0.19439868204283361, "frac_reward_zero_std": 0.0, "grad_norm": 0.48133528232574463, "kl": 0.538433846202679, "learning_rate": 2.9201205533865656e-06, "loss": 0.0362, "num_tokens": 222958600.0, "reward": 2.36783766746521, "reward_std": 0.3661612570285797, "rewards/reward_model/mean": 2.36783766746521, "rewards/reward_model/std": 0.8611352443695068, "step": 472, "step_time": 176.99345774995163 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1005859375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.02978515625, "completions/mean_terminated_length": 96.90174102783203, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 1.082069953205064, "epoch": 0.19481054365733114, "frac_reward_zero_std": 0.0, "grad_norm": 0.4831677973270416, "kl": 0.5806642937241122, "learning_rate": 2.9194246751343832e-06, "loss": 0.0346, "num_tokens": 223414501.0, "reward": 2.543872356414795, "reward_std": 0.3163744807243347, "rewards/reward_model/mean": 2.543872356414795, "rewards/reward_model/std": 0.8579828143119812, "step": 473, "step_time": 177.16888101864606 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.083984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.10546875, "completions/mean_terminated_length": 97.5479736328125, "completions/min_length": 30.0, "completions/min_terminated_length": 30.0, "entropy": 1.147307124454528, "epoch": 0.19522240527182866, "frac_reward_zero_std": 0.0, "grad_norm": 0.5124398469924927, "kl": 0.5504329709801823, "learning_rate": 2.918725862550952e-06, "loss": 0.048, "num_tokens": 223904925.0, "reward": 2.325784683227539, "reward_std": 0.354381799697876, "rewards/reward_model/mean": 2.325784683227539, "rewards/reward_model/std": 0.8341200947761536, "step": 474, "step_time": 178.86953710624948 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0517578125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 90.12646484375, "completions/mean_terminated_length": 88.05921936035156, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.0577353781554848, "epoch": 0.1956342668863262, "frac_reward_zero_std": 0.0, "grad_norm": 0.5797446370124817, "kl": 0.6813627502415329, "learning_rate": 2.9180241170809048e-06, "loss": 0.0439, "num_tokens": 224357696.0, "reward": 2.6658530235290527, "reward_std": 0.2722157835960388, "rewards/reward_model/mean": 2.6658530235290527, "rewards/reward_model/std": 0.7896630764007568, "step": 475, "step_time": 176.36151596717536 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.056640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.48681640625, "completions/mean_terminated_length": 96.71480560302734, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1672656913287938, "epoch": 0.19604612850082373, "frac_reward_zero_std": 0.0, "grad_norm": 0.8208109736442566, "kl": 0.5693298900732771, "learning_rate": 2.917319440174937e-06, "loss": 0.0419, "num_tokens": 224799941.0, "reward": 2.2602474689483643, "reward_std": 0.33485889434814453, "rewards/reward_model/mean": 2.2602474689483643, "rewards/reward_model/std": 0.7517103552818298, "step": 476, "step_time": 175.88794818613678 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0478515625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.3701171875, "completions/mean_terminated_length": 92.68000030517578, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "entropy": 1.1173612410202622, "epoch": 0.19645799011532125, "frac_reward_zero_std": 0.0, "grad_norm": 1.0202687978744507, "kl": 0.6552785804960877, "learning_rate": 2.9166118332898053e-06, "loss": 0.0387, "num_tokens": 225307675.0, "reward": 2.4912514686584473, "reward_std": 0.30000853538513184, "rewards/reward_model/mean": 2.4912514686584473, "rewards/reward_model/std": 0.8663033246994019, "step": 477, "step_time": 176.8035656195134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0654296875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.90478515625, "completions/mean_terminated_length": 89.37773895263672, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 1.136641243007034, "epoch": 0.19686985172981877, "frac_reward_zero_std": 0.0, "grad_norm": 0.6772795915603638, "kl": 0.6525858750101179, "learning_rate": 2.9159012978883227e-06, "loss": 0.0469, "num_tokens": 225807352.0, "reward": 2.4790232181549072, "reward_std": 0.3305346369743347, "rewards/reward_model/mean": 2.4790232181549072, "rewards/reward_model/std": 1.0394783020019531, "step": 478, "step_time": 176.85282629868016 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08740234375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.20068359375, "completions/mean_terminated_length": 96.4424819946289, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1933749543968588, "epoch": 0.19728171334431632, "frac_reward_zero_std": 0.0, "grad_norm": 0.49781715869903564, "kl": 0.568764234543778, "learning_rate": 2.9151878354393565e-06, "loss": 0.0356, "num_tokens": 226391187.0, "reward": 2.4199905395507812, "reward_std": 0.34556785225868225, "rewards/reward_model/mean": 2.4199905395507812, "rewards/reward_model/std": 1.0211058855056763, "step": 479, "step_time": 177.6960162757896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.08251953125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.57958984375, "completions/mean_terminated_length": 98.11336517333984, "completions/min_length": 14.0, "completions/min_terminated_length": 14.0, "entropy": 1.2039740288164467, "epoch": 0.19769357495881384, "frac_reward_zero_std": 0.0, "grad_norm": 1.0587437152862549, "kl": 0.601815638015978, "learning_rate": 2.914471447417824e-06, "loss": 0.0317, "num_tokens": 226931542.0, "reward": 2.3412864208221436, "reward_std": 0.3626915216445923, "rewards/reward_model/mean": 2.3412864208221436, "rewards/reward_model/std": 0.8987504839897156, "step": 480, "step_time": 176.73450627736747 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.12109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.47412109375, "completions/mean_terminated_length": 92.13055419921875, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.2335775888059288, "epoch": 0.19810543657331137, "frac_reward_zero_std": 0.0, "grad_norm": 0.5525914430618286, "kl": 0.5698116646381095, "learning_rate": 2.913752135304692e-06, "loss": 0.0444, "num_tokens": 227445345.0, "reward": 2.282869338989258, "reward_std": 0.38537949323654175, "rewards/reward_model/mean": 2.282869338989258, "rewards/reward_model/std": 0.8601107001304626, "step": 481, "step_time": 177.06750200130045 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.3134765625, "completions/mean_terminated_length": 96.07064819335938, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 1.160321586066857, "epoch": 0.1985172981878089, "frac_reward_zero_std": 0.0, "grad_norm": 0.4909352660179138, "kl": 0.5540991665329784, "learning_rate": 2.913029900586971e-06, "loss": 0.0476, "num_tokens": 227892995.0, "reward": 2.3108367919921875, "reward_std": 0.3486390709877014, "rewards/reward_model/mean": 2.3108367919921875, "rewards/reward_model/std": 0.9318093657493591, "step": 482, "step_time": 178.26090911217034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.10546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.53564453125, "completions/mean_terminated_length": 97.29749298095703, "completions/min_length": 27.0, "completions/min_terminated_length": 27.0, "entropy": 1.2190581287723035, "epoch": 0.19892915980230644, "frac_reward_zero_std": 0.0, "grad_norm": 0.5173491835594177, "kl": 0.5541561709251255, "learning_rate": 2.912304744757714e-06, "loss": 0.0495, "num_tokens": 228359148.0, "reward": 2.25392484664917, "reward_std": 0.38485991954803467, "rewards/reward_model/mean": 2.25392484664917, "rewards/reward_model/std": 0.886904239654541, "step": 483, "step_time": 175.91856609657407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07958984375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 98.875, "completions/mean_terminated_length": 96.35649871826172, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.239135185489431, "epoch": 0.19934102141680396, "frac_reward_zero_std": 0.0, "grad_norm": 0.5492594838142395, "kl": 0.5564993016887456, "learning_rate": 2.911576669316013e-06, "loss": 0.0456, "num_tokens": 228912012.0, "reward": 2.2495884895324707, "reward_std": 0.36551016569137573, "rewards/reward_model/mean": 2.2495884895324707, "rewards/reward_model/std": 0.834344744682312, "step": 484, "step_time": 178.1594290942885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04931640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.5234375, "completions/mean_terminated_length": 93.8387222290039, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1909776257816702, "epoch": 0.19975288303130148, "frac_reward_zero_std": 0.0, "grad_norm": 0.5581647753715515, "kl": 0.5739589168224484, "learning_rate": 2.9108456757669938e-06, "loss": 0.0518, "num_tokens": 229373948.0, "reward": 2.439542770385742, "reward_std": 0.32610630989074707, "rewards/reward_model/mean": 2.439542770385742, "rewards/reward_model/std": 0.8306021690368652, "step": 485, "step_time": 178.3928206982091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.046875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.81201171875, "completions/mean_terminated_length": 90.03227233886719, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.2118659724947065, "epoch": 0.200164744645799, "frac_reward_zero_std": 0.0, "grad_norm": 0.5383432507514954, "kl": 0.6172555118100718, "learning_rate": 2.9101117656218167e-06, "loss": 0.065, "num_tokens": 229837339.0, "reward": 2.469618797302246, "reward_std": 0.331695556640625, "rewards/reward_model/mean": 2.469618797302246, "rewards/reward_model/std": 0.9376615285873413, "step": 486, "step_time": 180.3623530426994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04736328125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 95.98681640625, "completions/mean_terminated_length": 94.39518737792969, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1846132616046816, "epoch": 0.20057660626029655, "frac_reward_zero_std": 0.0, "grad_norm": 0.47990092635154724, "kl": 0.595566791947931, "learning_rate": 2.9093749403976707e-06, "loss": 0.044, "num_tokens": 230317632.0, "reward": 2.479060649871826, "reward_std": 0.3090110421180725, "rewards/reward_model/mean": 2.479060649871826, "rewards/reward_model/std": 0.8331393003463745, "step": 487, "step_time": 178.39384605595842 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.06982421875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.640625, "completions/mean_terminated_length": 88.91128540039062, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1726260157302022, "epoch": 0.20098846787479407, "frac_reward_zero_std": 0.0, "grad_norm": 0.5010887384414673, "kl": 0.5850812633289024, "learning_rate": 2.9086352016177717e-06, "loss": 0.0457, "num_tokens": 230801024.0, "reward": 2.2932491302490234, "reward_std": 0.3312646448612213, "rewards/reward_model/mean": 2.2932491302490234, "rewards/reward_model/std": 0.781287670135498, "step": 488, "step_time": 179.38535980135202 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0537109375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.75537109375, "completions/mean_terminated_length": 92.86842346191406, "completions/min_length": 28.0, "completions/min_terminated_length": 28.0, "entropy": 1.131763206096366, "epoch": 0.2014003294892916, "frac_reward_zero_std": 0.0, "grad_norm": 0.5697361826896667, "kl": 0.57810703816358, "learning_rate": 2.9078925508113568e-06, "loss": 0.0476, "num_tokens": 231275499.0, "reward": 2.415783643722534, "reward_std": 0.30520525574684143, "rewards/reward_model/mean": 2.415783643722534, "rewards/reward_model/std": 0.8585343956947327, "step": 489, "step_time": 179.02824277989566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0419921875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 86.10546875, "completions/mean_terminated_length": 84.26911163330078, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 1.1569622193928808, "epoch": 0.20181219110378912, "frac_reward_zero_std": 0.0, "grad_norm": 0.5144662857055664, "kl": 0.6677741960156709, "learning_rate": 2.9071469895136846e-06, "loss": 0.049, "num_tokens": 231825379.0, "reward": 2.524040937423706, "reward_std": 0.29400721192359924, "rewards/reward_model/mean": 2.524040937423706, "rewards/reward_model/std": 1.012975811958313, "step": 490, "step_time": 176.841063328553 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.09326171875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 101.3359375, "completions/mean_terminated_length": 98.59342193603516, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1706854968797415, "epoch": 0.20222405271828667, "frac_reward_zero_std": 0.0, "grad_norm": 0.5214114189147949, "kl": 0.5280423930380493, "learning_rate": 2.9063985192660307e-06, "loss": 0.0328, "num_tokens": 232351443.0, "reward": 2.2841615676879883, "reward_std": 0.37312573194503784, "rewards/reward_model/mean": 2.2841615676879883, "rewards/reward_model/std": 0.8823071718215942, "step": 491, "step_time": 178.4695579907857 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0791015625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.037109375, "completions/mean_terminated_length": 96.54931640625, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1619540557730943, "epoch": 0.2026359143327842, "frac_reward_zero_std": 0.0, "grad_norm": 0.49944233894348145, "kl": 0.5264173087198287, "learning_rate": 2.905647141615683e-06, "loss": 0.0425, "num_tokens": 232823807.0, "reward": 2.2317872047424316, "reward_std": 0.34733903408050537, "rewards/reward_model/mean": 2.2317872047424316, "rewards/reward_model/std": 0.9077332019805908, "step": 492, "step_time": 176.85156889958307 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05810546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 99.494140625, "completions/mean_terminated_length": 97.73561096191406, "completions/min_length": 4.0, "completions/min_terminated_length": 4.0, "entropy": 1.1311972457915545, "epoch": 0.2030477759472817, "frac_reward_zero_std": 0.0, "grad_norm": 0.4777950942516327, "kl": 0.5227718978421763, "learning_rate": 2.904892858115941e-06, "loss": 0.0346, "num_tokens": 233303827.0, "reward": 2.434147834777832, "reward_std": 0.320200651884079, "rewards/reward_model/mean": 2.434147834777832, "rewards/reward_model/std": 0.8256375789642334, "step": 493, "step_time": 176.74407581565902 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.07275390625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 100.75390625, "completions/mean_terminated_length": 98.6161117553711, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1435500592924654, "epoch": 0.20345963756177923, "frac_reward_zero_std": 0.0, "grad_norm": 0.4735333025455475, "kl": 0.5233036724384874, "learning_rate": 2.904135670326111e-06, "loss": 0.0295, "num_tokens": 233764315.0, "reward": 2.353809356689453, "reward_std": 0.34140223264694214, "rewards/reward_model/mean": 2.353809356689453, "rewards/reward_model/std": 0.8551777601242065, "step": 494, "step_time": 176.53644647775218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04052734375, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 91.654296875, "completions/mean_terminated_length": 90.11908721923828, "completions/min_length": 23.0, "completions/min_terminated_length": 23.0, "entropy": 1.1197186093777418, "epoch": 0.20387149917627678, "frac_reward_zero_std": 0.0, "grad_norm": 0.5277080535888672, "kl": 0.6185391843318939, "learning_rate": 2.903375579811502e-06, "loss": 0.033, "num_tokens": 234208919.0, "reward": 2.6142148971557617, "reward_std": 0.258402943611145, "rewards/reward_model/mean": 2.6142148971557617, "rewards/reward_model/std": 0.8695737719535828, "step": 495, "step_time": 176.3124782810919 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.04931640625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.85302734375, "completions/mean_terminated_length": 93.13353729248047, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.105817890027538, "epoch": 0.2042833607907743, "frac_reward_zero_std": 0.0, "grad_norm": 0.49831780791282654, "kl": 0.5880182490218431, "learning_rate": 2.9026125881434263e-06, "loss": 0.0371, "num_tokens": 234689450.0, "reward": 2.496224880218506, "reward_std": 0.29817432165145874, "rewards/reward_model/mean": 2.496224880218506, "rewards/reward_model/std": 0.7783900499343872, "step": 496, "step_time": 176.80578273162246 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0810546875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 94.0751953125, "completions/mean_terminated_length": 91.0828857421875, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.0922315237112343, "epoch": 0.20469522240527183, "frac_reward_zero_std": 0.0, "grad_norm": 0.48617762327194214, "kl": 0.5914126335410401, "learning_rate": 2.9018466968991914e-06, "loss": 0.0318, "num_tokens": 235194628.0, "reward": 2.4226419925689697, "reward_std": 0.31893834471702576, "rewards/reward_model/mean": 2.4226419925689697, "rewards/reward_model/std": 0.9487303495407104, "step": 497, "step_time": 179.51883107610047 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0908203125, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.0986328125, "completions/mean_terminated_length": 92.91192626953125, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1081214109435678, "epoch": 0.20510708401976935, "frac_reward_zero_std": 0.0, "grad_norm": 0.5024422407150269, "kl": 0.6213067708304152, "learning_rate": 2.9010779076621e-06, "loss": 0.0366, "num_tokens": 235644526.0, "reward": 2.5506255626678467, "reward_std": 0.307564377784729, "rewards/reward_model/mean": 2.5506255626678467, "rewards/reward_model/std": 0.866729736328125, "step": 498, "step_time": 177.24243266135454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05322265625, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 96.5458984375, "completions/mean_terminated_length": 94.77772521972656, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 1.1355702604632825, "epoch": 0.2055189456342669, "frac_reward_zero_std": 0.0, "grad_norm": 0.4873970150947571, "kl": 0.6060064556077123, "learning_rate": 2.900306222021445e-06, "loss": 0.0359, "num_tokens": 236101228.0, "reward": 2.4689903259277344, "reward_std": 0.3167894780635834, "rewards/reward_model/mean": 2.4689903259277344, "rewards/reward_model/std": 0.7992905378341675, "step": 499, "step_time": 177.44218201190233 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.03466796875, "completions/max_length": 128.0, "completions/max_terminated_length": 128.0, "completions/mean_length": 92.22998046875, "completions/mean_terminated_length": 90.94537353515625, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 1.1435975283384323, "epoch": 0.20593080724876442, "frac_reward_zero_std": 0.0, "grad_norm": 1.4175333976745605, "kl": 0.6452224206877872, "learning_rate": 2.899531641572508e-06, "loss": 0.0294, "num_tokens": 236602435.0, "reward": 2.2785120010375977, "reward_std": 0.30327603220939636, "rewards/reward_model/mean": 2.2785120010375977, "rewards/reward_model/std": 0.9206061363220215, "step": 500, "step_time": 178.54687940515578 } ], "logging_steps": 1, "max_steps": 2428, "num_input_tokens_seen": 236602435, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }